"""Scalar-feature BatchNorm, Ioffe and Szegedy ICML 2015. No model training.
Run python examples/batchnorm.py [--json]. Standard library only.
"""
import argparse
import json
import math


# region: training
def batch_norm(x, gamma=1.0, beta=0.0, epsilon=1e-5):
    """Algorithm 1: biased mini-batch variance (divide by m), then affine."""
    if not x or not all(math.isfinite(v) for v in [*x, gamma, beta, epsilon]):
        raise ValueError("Nonempty finite values required")
    if epsilon <= 0:
        raise ValueError("Positive epsilon required")
    mean = sum(x) / len(x)
    variance = sum((v - mean) ** 2 for v in x) / len(x)
    normalized = [(v - mean) / math.sqrt(variance + epsilon) for v in x]
    return dict(mean=mean, variance=variance, normalized=normalized,
                output=[gamma * v + beta for v in normalized])
# endregion: training


# region: backward
def backward(x, dy, gamma=1.0, epsilon=1e-5):
    """Differentiate through mean AND variance; vector-Jacobian product."""
    if len(dy) != len(x) or not all(math.isfinite(v) for v in dy):
        raise ValueError("Finite gradient of matching shape required")
    f = batch_norm(x, gamma=gamma, epsilon=epsilon)
    n, xhat = len(x), f['normalized']
    dbeta = sum(dy)
    dgamma = sum(d * h for d, h in zip(dy, xhat))
    dx = [gamma / math.sqrt(f['variance'] + epsilon) / n
          * (n * d - dbeta - h * dgamma) for d, h in zip(dy, xhat)]
    return dict(dx=dx, dGamma=dgamma, dBeta=dbeta)
# endregion: backward


# region: inference
def population_stats(batches):
    """Algorithm 2 estimate: equal-sized batches, m/(m-1) variance correction."""
    if not batches or len(batches[0]) < 2:
        raise ValueError("Batches must contain at least two samples")
    m = len(batches[0])
    if any(len(b) != m for b in batches):
        raise ValueError("Equal batch sizes required")
    stats = [batch_norm(b) for b in batches]
    return dict(mean=sum(s['mean'] for s in stats) / len(stats),
                variance=m / (m - 1) * sum(s['variance'] for s in stats) / len(stats))


def inference(x, mean, variance, gamma=1.0, beta=0.0, epsilon=1e-5):
    """Frozen statistics: each result is independent of other test examples."""
    if not x or not all(math.isfinite(v) for v in [*x, mean, variance, gamma, beta, epsilon]):
        raise ValueError("Nonempty finite values required")
    if variance < 0 or epsilon <= 0:
        raise ValueError("Nonnegative variance and positive epsilon required")
    return [gamma * (v - mean) / math.sqrt(variance + epsilon) + beta for v in x]
# endregion: inference


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--json', action='store_true')
    args = parser.parse_args()
    x, dy = [1, 2, 4, 8], [1, -2, 3, -1]
    result = dict(forward=batch_norm(x), backward=backward(x, dy),
                  population=population_stats([[1, 2, 4, 8], [2, 3, 5, 9]]),
                  inference=inference([2, 5], mean=3, variance=4))
    dx = result['backward']['dx']
    for i in range(len(x)):
        left, right, h = x.copy(), x.copy(), 1e-5
        left[i] -= h
        right[i] += h
        loss = lambda a: sum(y * d for y, d in zip(batch_norm(a)['output'], dy))
        assert math.isclose(dx[i], (loss(right) - loss(left)) / (2 * h), abs_tol=1e-8)
    assert batch_norm([3, 3])['output'] == [0.0, 0.0]
    print(json.dumps(result) if args.json else 'BatchNorm gradient checks passed: ' + str(result))


if __name__ == '__main__':
    main()
