ch246 — Bayes’ Theorem

import numpy as np
import matplotlib.pyplot as plt

def bayes_update(prior: float, likelihood: float, p_evidence: float) -> float:
    """
    Compute posterior P(H|E) given:
      prior      = P(H)
      likelihood = P(E|H)
      p_evidence = P(E) — marginal probability of evidence
    """
    return (likelihood * prior) / p_evidence


# Spam filter from the Part VIII intro
p_spam = 0.30              # P(spam)
p_legit = 1 - p_spam       # P(legit)
p_offer_given_spam = 0.40  # P('offer' | spam)
p_offer_given_legit = 0.05 # P('offer' | legit)

# Law of total probability: P('offer') = P('offer'|spam)P(spam) + P('offer'|legit)P(legit)
p_offer = p_offer_given_spam * p_spam + p_offer_given_legit * p_legit

# Bayes' theorem
p_spam_given_offer = bayes_update(p_spam, p_offer_given_spam, p_offer)

print("Spam Filter — Bayes' Theorem")
print(f"  Prior P(spam) = {p_spam:.4f}")
print(f"  P('offer' | spam) = {p_offer_given_spam:.4f}")
print(f"  P('offer' | legit) = {p_offer_given_legit:.4f}")
print(f"  P('offer') = {p_offer:.4f}  [law of total probability]")
print(f"  Posterior P(spam | 'offer') = {p_spam_given_offer:.4f}")

def bayes_two_class(prior_h: float, likelihood_h: float, likelihood_not_h: float) -> float:
    """
    P(H|E) using full Bayes with expanded denominator.
    No need to pre-compute P(E).
    """
    numerator = likelihood_h * prior_h
    denominator = likelihood_h * prior_h + likelihood_not_h * (1 - prior_h)
    return numerator / denominator


# Medical test example
# Disease prevalence: 1%
# Test sensitivity (true positive rate): 99%
# Test specificity (true negative rate): 95% → false positive rate: 5%

prior_disease = 0.01
sensitivity = 0.99   # P(positive | disease)
false_positive = 0.05  # P(positive | no disease)

posterior = bayes_two_class(prior_disease, sensitivity, false_positive)

print("Medical Test Example")
print(f"  Disease prevalence: {prior_disease:.0%}")
print(f"  Test sensitivity: {sensitivity:.0%}")
print(f"  False positive rate: {false_positive:.0%}")
print(f"  P(disease | positive test) = {posterior:.4f} ({posterior:.1%})")
print()
print("  Despite 99% sensitivity, a positive test on a low-prevalence")
print(f"  disease means only {posterior:.1%} chance of actually having it.")
print("  This is the base rate fallacy — Bayes corrects for it.")

# Sequential Bayesian updating
# Each email is independently scanned for spam words.
# Words: 'offer', 'free', 'win'

likelihoods_spam  = {'offer': 0.40, 'free': 0.60, 'win': 0.50}
likelihoods_legit = {'offer': 0.05, 'free': 0.10, 'win': 0.08}

def sequential_bayes(prior, words_found):
    posterior = prior
    history = [prior]
    for word in words_found:
        posterior = bayes_two_class(
            posterior,
            likelihoods_spam[word],
            likelihoods_legit[word]
        )
        history.append(posterior)
    return posterior, history

words = ['offer', 'free', 'win']
final_p, history = sequential_bayes(prior=0.30, words_found=words)

print("Sequential updates after each spam word:")
labels = ['Prior'] + [f"After '{w}'" for w in words]
for label, p in zip(labels, history):
    bar = '█' * int(p * 40)
    print(f"  {label:<18}: {p:.4f}  {bar}")

# Plot
fig, ax = plt.subplots(figsize=(7, 3))
ax.plot(range(len(history)), history, 'o-', color='steelblue', linewidth=2, markersize=8)
ax.axhline(0.5, color='gray', linestyle='--', alpha=0.5, label='Decision boundary')
ax.set_xticks(range(len(history)))
ax.set_xticklabels(labels, rotation=15)
ax.set_ylabel('P(spam)')
ax.set_title('Bayesian Belief Update — Email Spam Filter')
ax.set_ylim(0, 1)
ax.legend()
plt.tight_layout()
plt.show()

# Minimal Naive Bayes classifier from scratch
# Training data: (word_count_offer, label)

class NaiveBayesBinaryClassifier:
    """Naive Bayes for binary classification from counts."""
    
    def __init__(self):
        self.prior = {}
        self.likelihoods = {}  # {class: {feature: P(feature|class)}}
    
    def fit(self, X, y, feature_names):
        """
        X: shape (n_samples, n_features) — binary feature presence
        y: labels (0 or 1)
        """
        X, y = np.array(X), np.array(y)
        classes = np.unique(y)
        n = len(y)
        
        for c in classes:
            mask = y == c
            self.prior[c] = mask.sum() / n
            # Laplace smoothing: add 1 to numerator and denominator
            self.likelihoods[c] = {
                feat: (X[mask, i].sum() + 1) / (mask.sum() + 2)
                for i, feat in enumerate(feature_names)
            }
        self.feature_names = feature_names
        return self
    
    def predict_proba(self, x, feature_names):
        """Return P(class=1 | x) using Bayes' theorem."""
        scores = {}
        for c, prior in self.prior.items():
            log_prob = np.log(prior)
            for i, feat in enumerate(feature_names):
                p = self.likelihoods[c][feat]
                log_prob += np.log(p) if x[i] == 1 else np.log(1 - p)
            scores[c] = log_prob
        # Convert log-probabilities to normalized probabilities
        max_log = max(scores.values())
        exp_scores = {c: np.exp(v - max_log) for c, v in scores.items()}
        total = sum(exp_scores.values())
        return {c: v / total for c, v in exp_scores.items()}


# Tiny dataset: [has_offer, has_free, has_win] → spam(1) or legit(0)
features = ['offer', 'free', 'win']
X_train = [
    [1, 1, 0],  # spam
    [1, 0, 1],  # spam
    [0, 1, 1],  # spam
    [0, 0, 0],  # legit
    [1, 0, 0],  # legit
    [0, 0, 0],  # legit
]
y_train = [1, 1, 1, 0, 0, 0]

clf = NaiveBayesBinaryClassifier().fit(X_train, y_train, features)

# Predict a new email: contains 'offer' and 'free'
x_new = [1, 1, 0]
proba = clf.predict_proba(x_new, features)
print(f"New email [offer=1, free=1, win=0]:")
print(f"  P(spam | features) = {proba[1]:.4f}")
print(f"  P(legit | features) = {proba[0]:.4f}")