"""Exhaustive small-mask dropout arithmetic; JMLR 2014, Sections 4 and 5.
Python standard library only. This does not train or evaluate a neural network.
"""
import argparse
import json
import math


def validate(x, p):
    if not x or not all(math.isfinite(v) for v in [*x, p]) or not 0 <= p <= 1:
        raise ValueError("Finite inputs and retention p in [0,1] required")


# region: mask
def apply_mask(x, mask, keep=0.5, inverted=False):
    """Independent Bernoulli masks are supplied explicitly, not sampled here."""
    validate(x, keep)
    if len(mask) != len(x) or any(v not in (0, 1) for v in mask):
        raise ValueError("Binary mask of matching shape required")
    if inverted and keep == 0:
        raise ValueError("Inverted dropout requires positive retention")
    return [v * r / (keep if inverted else 1) for v, r in zip(x, mask)]
# endregion: mask


# region: scaling
def moments(x, keep, inverted=False):
    """Exact coordinate means/variances; original p is KEEP probability."""
    validate(x, keep)
    if inverted and keep == 0:
        raise ValueError("Inverted dropout requires positive retention")
    scale = 1 / keep if inverted else 1
    return dict(mean=[keep * v * scale for v in x],
                variance=[keep * (1 - keep) * v * v * scale * scale for v in x],
                test=[v * (1 if inverted else keep) for v in x])
# endregion: scaling


# region: ensemble
def enumerate_masks(x, weights, keep, bias=0.25, nonlinear=True):
    """Enumerate an authored tiny network; nonlinear expectation is not exact scaling."""
    validate(x, keep)
    if len(x) > 12 or len(weights) != len(x) or not all(math.isfinite(v) for v in [bias, *weights]):
        raise ValueError("Matching finite weights and at most 12 units required")
    def activate(z):
        if not nonlinear:
            return z
        e = math.exp(-z if z >= 0 else z)
        return 1 / (1 + e) if z >= 0 else e / (1 + e)
    rows = []
    for bits in range(2 ** len(x)):
        mask = [(bits >> i) & 1 for i in range(len(x))]
        count = sum(mask)
        probability = keep ** count * (1 - keep) ** (len(x) - count)
        z = bias + sum(v * w for v, w in zip(apply_mask(x, mask, keep), weights))
        rows.append(dict(mask=mask, probability=probability, z=z, output=activate(z)))
    expected_logit = bias + sum(keep * v * w for v, w in zip(x, weights))
    return dict(rows=rows, expectedLogit=expected_logit,
                expectedOutput=sum(r['probability'] * r['output'] for r in rows),
                scaledOutput=activate(expected_logit))
# endregion: ensemble


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--json', action='store_true')
    args = parser.parse_args()
    result = enumerate_masks([1, 2, -1], [2, -1, 1.5], 0.6)
    assert math.isclose(sum(r['probability'] for r in result['rows']), 1)
    linear = enumerate_masks([1, 2, -1], [2, -1, 1.5], 0.6, nonlinear=False)
    assert math.isclose(linear['expectedOutput'], linear['scaledOutput'])
    assert abs(result['expectedOutput'] - result['scaledOutput']) > 1e-3
    assert apply_mask([1, 2], [0, 0]) == [0, 0]
    assert moments([2], 0.5, inverted=True)['mean'] == [2]
    print(json.dumps(result) if args.json else 'Dropout exact expectation checks passed: ' + str(result))


if __name__ == '__main__':
    main()
