# examples/liger-swiglu/swiglu.py
"""CPU teaching reference for Liger's SiLU-and-multiply fusion boundary.

Uses only Python's standard library. Run with --json for deterministic checks.
This is binary64 arithmetic, not a GPU kernel, low-precision emulator, or benchmark.
"""

import argparse
import json
import math


def _validate_vectors(*vectors):
    """Require equally sized, nonempty, finite vectors without broadcasting."""
    if not vectors[0] or any(len(v) != len(vectors[0]) for v in vectors):
        raise ValueError("Expected nonempty vectors of equal length")
    if any(not math.isfinite(x) for vector in vectors for x in vector):
        raise ValueError("Expected finite vector entries")


# region: forward
def sigmoid(value):
    """Evaluate sigmoid without exponentiating a large positive number."""
    if value >= 0:
        return 1.0 / (1.0 + math.exp(-value))
    exp_value = math.exp(value)
    return exp_value / (1.0 + exp_value)


def swiglu(gate, up):
    """Map two projected vectors to SiLU(gate) * up, element by element."""
    _validate_vectors(gate, up)
    return [a * sigmoid(a) * b for a, b in zip(gate, up)]
# endregion: forward


# region: backward
def swiglu_backward(gate, up, upstream):
    """Recompute SiLU from saved inputs; return both vector-Jacobian products."""
    _validate_vectors(gate, up, upstream)
    d_gate, d_up = [], []
    for a, b, gradient in zip(gate, up, upstream):
        probability = sigmoid(a)
        silu = a * probability
        slope = probability + silu * (1.0 - probability)
        d_gate.append(gradient * b * slope)
        d_up.append(gradient * silu)
    return d_gate, d_up
# endregion: backward


# region: traffic
def traffic(tokens, hidden, bytes_per_element=2):
    """Count logical forward bytes for two passes versus one fused pass.

    Excludes projections, backward, caching, copies, and launch overhead.
    The saved activation size is incremental storage, not peak GPU memory.
    """
    if any(type(v) is not int or v <= 0 for v in (tokens, hidden)):
        raise ValueError("Expected positive integer dimensions")
    if bytes_per_element not in (2, 4):
        raise ValueError("Expected 2 or 4 storage bytes per element")
    tensor_bytes = tokens * hidden * bytes_per_element
    return {
        "separateBytes": 5 * tensor_bytes,
        "fusedBytes": 3 * tensor_bytes,
        "avoidedBytes": 2 * tensor_bytes,
        "savedSiluBytes": tensor_bytes,
    }
# endregion: traffic


def finite_difference(gate, up, upstream, delta=1e-5):
    """Differentiate the scalar loss sum(upstream * output) numerically."""
    derivatives = []
    for branch, values in enumerate((gate, up)):
        result = []
        for index in range(len(values)):
            losses = []
            for sign in (-1, 1):
                inputs = [list(gate), list(up)]
                inputs[branch][index] += sign * delta
                output = swiglu(*inputs)
                losses.append(math.fsum(g * y for g, y in zip(upstream, output)))
            result.append((losses[1] - losses[0]) / (2 * delta))
        derivatives.append(result)
    return derivatives


def self_check():
    """Check signs, limits, independent finite differences, and input contracts."""
    gate = [-8.0, -2.0, -1.0, 0.0, 1.0, 4.0]
    up = [2.0, -1.5, 0.0, 3.0, -2.0, 0.5]
    upstream = [1.0, -0.5, 2.0, 1.0, 0.25, -1.0]
    output = swiglu(gate, up)
    analytical = swiglu_backward(gate, up, upstream)
    numerical = finite_difference(gate, up, upstream)
    for exact_values, approximate_values in zip(analytical, numerical):
        for exact, approximate in zip(exact_values, approximate_values):
            assert math.isclose(exact, approximate, rel_tol=1e-7, abs_tol=1e-8)
    assert swiglu([0.0], [7.0]) == [0.0]
    assert swiglu_backward([0.0], [7.0], [2.0]) == ([7.0], [0.0])
    assert swiglu_backward([1.0], [0.0], [1.0])[1][0] > 0
    assert swiglu_backward([-2.0], [1.0], [1.0])[0][0] < 0
    assert swiglu([-1000.0, 1000.0], [1.0, 1.0]) == [-0.0, 1000.0]
    assert traffic(1, 1) == {
        "separateBytes": 10, "fusedBytes": 6,
        "avoidedBytes": 4, "savedSiluBytes": 2,
    }
    for invalid in (([], []), ([1.0], [1.0, 2.0]), ([math.nan], [1.0])):
        try:
            swiglu(*invalid)
        except ValueError:
            pass
        else:
            raise AssertionError("Invalid vectors were accepted")
    assert gate == [-8.0, -2.0, -1.0, 0.0, 1.0, 4.0]
    return {
        "gate": gate, "up": up, "upstream": upstream, "output": output,
        "dGate": analytical[0], "dUp": analytical[1],
        "finiteDifference": numerical,
        "traffic": traffic(1024, 4096),
    }


def main():
    """Print deterministic arithmetic after running correctness checks."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--json", action="store_true")
    args = parser.parse_args()
    result = self_check()
    if args.json:
        print(json.dumps(result))
    else:
        print("SwiGLU CPU checks passed: forward, both gradients, and traffic.")
        print(json.dumps(result, indent=2))


if __name__ == "__main__":
    main()
