# examples/liger-rmsnorm/reference.py
"""Educational RMSNorm forward/backward; Python standard library only.

Real-valued equations from Liger Kernel, arXiv:2410.10989v3, equations 1-2.
Python floats use binary64. This is neither a Triton kernel nor an emulation of
Liger's mixed-precision rounding, reduction order, device scheduling or speed.
"""

import argparse
import json
import math


def validate_row(values):
    """Require a nonempty finite vector before using its row statistics."""
    if not values or not all(math.isfinite(value) for value in values):
        raise ValueError("A nonempty finite row is required")


# region: normalize
def normalize(x, epsilon=1e-6):
    """Return normalized features and the cached reciprocal RMS of one row."""
    validate_row(x)
    if not math.isfinite(epsilon) or epsilon <= 0:
        raise ValueError("Epsilon must be finite and positive")
    mean_square = math.fsum(value * value for value in x) / len(x)
    if not math.isfinite(mean_square):
        raise ValueError("Squared inputs exceed the reference's numeric range")
    rstd = 1.0 / math.sqrt(mean_square + epsilon)
    return [value * rstd for value in x], rstd
# endregion: normalize


# region: forward
def forward(x, weight, epsilon=1e-6, offset=0.0):
    """Normalize the final dimension and apply a learned feature scale."""
    validate_row(weight)
    if len(x) != len(weight) or not math.isfinite(offset):
        raise ValueError("Weight must match the row; offset must be finite")
    x_hat, rstd = normalize(x, epsilon)
    y = [value * (scale + offset)
         for value, scale in zip(x_hat, weight)]
    return y, x_hat, rstd
# endregion: forward


# region: backward
def backward(x, weight, dy, epsilon=1e-6, offset=0.0):
    """Differentiate one row, including dependence of its RMS on every x."""
    validate_row(dy)
    if len(dy) != len(x):
        raise ValueError("Upstream gradient must match the row")
    _, x_hat, rstd = forward(x, weight, epsilon, offset)
    q = [grad * (scale + offset) for grad, scale in zip(dy, weight)]
    projection = math.fsum(a * b for a, b in zip(q, x_hat)) / len(x)
    dx = [rstd * (grad - normalized * projection)
          for grad, normalized in zip(q, x_hat)]
    dw = [grad * normalized for grad, normalized in zip(dy, x_hat)]
    return dx, dw
# endregion: backward


# region: weights
def weight_gradient(rows, weight, upstream, epsilon=1e-6, offset=0.0):
    """Sum the contributions to the weight shared by all token rows."""
    if not rows or len(rows) != len(upstream):
        raise ValueError("Input rows and upstream rows must match")
    partials = [backward(x, weight, dy, epsilon, offset)[1]
                for x, dy in zip(rows, upstream)]
    return [math.fsum(row[column] for row in partials)
            for column in range(len(weight))]
# endregion: weights


def main():
    """Print a small deterministic trace; tests live in test_reference.py."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--json", action="store_true")
    args = parser.parse_args()
    x, weight, dy = [1.0, 2.0, 3.0, 4.0], [1.0, 0.5, 1.5, 1.0], [1.0, 0.0, 0.0, 0.0]
    y, x_hat, rstd = forward(x, weight)
    dx, dw = backward(x, weight, dy)
    result = dict(x=x, weight=weight, dy=dy, y=y, normalized=x_hat,
                  rstd=rstd, dx=dx, dw=dw)
    print(json.dumps(result, indent=None if args.json else 2))


if __name__ == "__main__":
    main()
