Chapter 131 — Dot Product Intuition

# --- Visualization: Dot product as alignment ---
# We draw several pairs of vectors and display their dot products.
# The sign and magnitude of the dot product reveals the alignment.

import numpy as np
import matplotlib.pyplot as plt

plt.style.use('seaborn-v0_8-whitegrid')

ORIGIN = np.array([0, 0])

# Three test cases: aligned, perpendicular, opposing
cases = [
    (np.array([2, 1]), np.array([1, 2]), "Mostly aligned"),
    (np.array([2, 0]), np.array([0, 2]), "Perpendicular"),
    (np.array([2, 1]), np.array([-1, -2]), "Opposing"),
]

fig, axes = plt.subplots(1, 3, figsize=(14, 4))

for ax, (a, b, label) in zip(axes, cases):
    dot = np.dot(a, b)
    ax.quiver(*ORIGIN, *a, angles='xy', scale_units='xy', scale=1,
               color='steelblue', label=f'a = {a}')
    ax.quiver(*ORIGIN, *b, angles='xy', scale_units='xy', scale=1,
               color='tomato', label=f'b = {b}')
    ax.set_xlim(-3, 3)
    ax.set_ylim(-3, 3)
    ax.set_aspect('equal')
    ax.axhline(0, color='gray', linewidth=0.5)
    ax.axvline(0, color='gray', linewidth=0.5)
    ax.set_title(f'{label}\na·b = {dot}', fontsize=11)
    ax.legend(fontsize=8)
    ax.set_xlabel('x')
    ax.set_ylabel('y')

plt.suptitle('Dot Product as Alignment Signal', fontsize=13, fontweight='bold')
plt.tight_layout()
plt.show()

# --- Visualization: Dot product vs angle sweep ---
# Fix vector a = [1, 0] and rotate vector b through 360 degrees.
# Plot how the dot product changes — this reveals the cosine relationship.

a = np.array([1.0, 0.0])  # fixed reference vector
angles_deg = np.linspace(0, 360, 360)
angles_rad = np.radians(angles_deg)

b_vectors = np.column_stack([np.cos(angles_rad), np.sin(angles_rad)])  # unit circle
dot_products = b_vectors @ a  # equivalent to [np.dot(a, b) for b in b_vectors]

fig, ax = plt.subplots(figsize=(9, 4))
ax.plot(angles_deg, dot_products, color='steelblue', linewidth=2)
ax.axhline(0, color='gray', linewidth=0.8, linestyle='--')
ax.fill_between(angles_deg, dot_products, 0,
                where=(dot_products > 0), alpha=0.2, color='green', label='Positive (same side)')
ax.fill_between(angles_deg, dot_products, 0,
                where=(dot_products < 0), alpha=0.2, color='red', label='Negative (opposite side)')
ax.set_xlabel('Angle of b (degrees)')
ax.set_ylabel('a · b')
ax.set_title('Dot product of [1,0] with unit vector at angle θ')
ax.legend()
plt.tight_layout()
plt.show()

print("The dot product traces a cosine curve — the formal link is proven in ch132.")

# Worked numeric example: step by step
a = np.array([3, -1, 2])
b = np.array([1, 4, -2])

print("Vectors:")
print(f"  a = {a}")
print(f"  b = {b}")

# Component-wise multiplication
products = a * b
print(f"\nComponent products a_i * b_i: {products}")

# Sum
dot_manual = np.sum(products)
print(f"Sum (dot product): {dot_manual}")

# Verify with numpy
dot_numpy = np.dot(a, b)
print(f"np.dot result:     {dot_numpy}")
print(f"Match: {dot_manual == dot_numpy}")

# --- Implementation: Dot product from scratch ---

def dot_product(a, b):
    """
    Compute the dot product of two vectors.

    Args:
        a: array-like, shape (n,)
        b: array-like, shape (n,)

    Returns:
        float: scalar dot product

    Raises:
        ValueError: if dimensions don't match
    """
    a, b = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
    if a.shape != b.shape:
        raise ValueError(f"Shape mismatch: {a.shape} vs {b.shape}")
    return float(np.sum(a * b))  # element-wise multiply then sum


def is_orthogonal(a, b, tol=1e-10):
    """
    Check if two vectors are orthogonal (dot product is zero).

    Args:
        a, b: array-like vectors
        tol: numerical tolerance for zero comparison

    Returns:
        bool
    """
    return abs(dot_product(a, b)) < tol


# Tests
print("dot_product([1,2,3], [4,5,6]) =", dot_product([1,2,3], [4,5,6]))
print("Expected: 1*4 + 2*5 + 3*6 =", 1*4 + 2*5 + 3*6)
print()
print("is_orthogonal([1,0], [0,1]):", is_orthogonal([1,0], [0,1]))  # True
print("is_orthogonal([1,1], [1,0]):", is_orthogonal([1,1], [1,0]))  # False

# Validate against numpy
a = np.random.randn(100)
b = np.random.randn(100)
assert abs(dot_product(a, b) - np.dot(a, b)) < 1e-10, "Mismatch!"
print("\n100-dimensional validation against np.dot: PASSED")

# --- Neural network neuron: dot product in action ---
# A single neuron computes: output = dot(weights, input) + bias
# This is the fundamental operation in deep learning.

def neuron(weights, inputs, bias=0.0):
    """
    Simulate a single linear neuron.

    Args:
        weights: array (n,), learned parameters
        inputs:  array (n,), input features
        bias:    float, offset term

    Returns:
        float: pre-activation output
    """
    return dot_product(weights, inputs) + bias


# Example: 3-feature input, learned weights
w = np.array([0.5, -1.2, 0.8])   # weights
x = np.array([1.0,  0.5, 2.0])   # input features
b = 0.1                            # bias

output = neuron(w, x, b)
print(f"Weights:  {w}")
print(f"Inputs:   {x}")
print(f"Bias:     {b}")
print(f"Output:   {output:.4f}")
print(f"(This reappears in ch176 — Linear Layers in Deep Learning)")

# --- Experiment 1: Effect of scaling on dot product ---
# Hypothesis: scaling a vector by c scales the dot product by c.
# Try changing: the scale factor and vector dimensions.

a = np.array([1.0, 2.0, 3.0])
b = np.array([4.0, 5.0, 6.0])
SCALE = 3.0  # <-- modify this

print(f"a·b = {np.dot(a, b):.2f}")
print(f"(SCALE*a)·b = {np.dot(SCALE*a, b):.2f}")
print(f"SCALE * (a·b) = {SCALE * np.dot(a, b):.2f}")
print(f"Equal: {np.isclose(np.dot(SCALE*a, b), SCALE * np.dot(a, b))}")

# --- Experiment 2: Dot product in high dimensions ---
# Hypothesis: random unit vectors in high dimensions have near-zero dot products.
# This is the curse of dimensionality — most directions become orthogonal.
# Try changing: N_DIMS and N_SAMPLES.

N_DIMS   = 300  # <-- modify this
N_SAMPLES = 2000

# Generate random unit vectors
vecs = np.random.randn(N_SAMPLES, N_DIMS)
norms = np.linalg.norm(vecs, axis=1, keepdims=True)
unit_vecs = vecs / norms

# Compute pairwise dot products (first vector vs all others)
dots = unit_vecs[1:] @ unit_vecs[0]

print(f"Dimensions: {N_DIMS}")
print(f"Mean |dot product|: {np.mean(np.abs(dots)):.4f}")
print(f"Std of dot products: {np.std(dots):.4f}")
print(f"\nIn {N_DIMS} dimensions, random unit vectors are nearly orthogonal.")
print("(Revisited in ch129 distance concentration, and ch182 PCA intuition.)")

# --- Experiment 3: Dot product as similarity ---
# Use raw dot product to find which word vectors are most similar to a query.
# (Simplified: random embeddings. Real embeddings used in ch180.)
# Try changing: QUERY_IDX

np.random.seed(42)
N_WORDS = 20
DIM     = 50  # embedding dimension

# Simulate word embeddings as random unit vectors
embeddings = np.random.randn(N_WORDS, DIM)
embeddings /= np.linalg.norm(embeddings, axis=1, keepdims=True)

QUERY_IDX = 0  # <-- modify this (0 to 19)
query = embeddings[QUERY_IDX]

# Dot product with all embeddings = similarity scores
scores = embeddings @ query

ranked = np.argsort(scores)[::-1]
print(f"Word {QUERY_IDX} similarity ranking (top 5):")
for rank, idx in enumerate(ranked[:5]):
    print(f"  Rank {rank+1}: word {idx:2d}, score = {scores[idx]:.4f}")

# --- Mini Project: Document Similarity via Dot Product ---
# Problem: Represent text documents as word-frequency vectors,
#          then use the dot product to rank documents by similarity to a query.
# Dataset: Manually constructed bag-of-words vectors.
# Task: Complete the similarity_rank function and analyze results.

# Vocabulary and documents (bag-of-words representation)
vocabulary = ['math', 'python', 'machine', 'learning', 'matrix', 'vector',
               'data', 'model', 'code', 'statistics']

# Each row is a document; each column is a word count from vocabulary
documents = np.array([
    [5, 3, 1, 1, 4, 4, 0, 0, 2, 0],  # doc 0: linear algebra tutorial
    [0, 4, 3, 4, 0, 0, 5, 4, 5, 1],  # doc 1: ML engineering blog
    [2, 1, 0, 0, 0, 0, 6, 0, 0, 8],  # doc 2: statistics textbook
    [3, 5, 2, 2, 2, 3, 1, 3, 6, 0],  # doc 3: computational math course
    [0, 0, 5, 5, 0, 0, 4, 6, 0, 2],  # doc 4: deep learning paper
], dtype=float)

# Query: user is searching for content about vectors and math
query = np.array([3, 2, 0, 0, 2, 5, 0, 0, 1, 0], dtype=float)

def similarity_rank(documents, query):
    """
    Rank documents by raw dot product with query.

    Args:
        documents: array (m, n) — m documents, n-dim vectors
        query:     array (n,)   — query vector

    Returns:
        ranked_indices: array of document indices, best first
        scores:         corresponding dot product scores
    """
    # TODO: compute dot product of each document with query
    scores = None  # replace with your implementation

    # TODO: return indices sorted by score (descending)
    ranked_indices = None  # replace

    return ranked_indices, scores


# --- Test your implementation ---
# ranked, scores = similarity_rank(documents, query)
# doc_labels = ['Linear Algebra Tutorial', 'ML Engineering Blog',
#               'Statistics Textbook', 'Computational Math Course', 'Deep Learning Paper']
# print("Ranking for query:", query)
# for i, idx in enumerate(ranked):
#     print(f"  {i+1}. {doc_labels[idx]}: score = {scores[idx]:.1f}")

# --- Reflection question ---
# The raw dot product favors longer documents (higher word counts).
# How would you modify this to be length-independent?
# (Answer: normalize each vector — this is cosine similarity, formalized in ch132.)