# examples/vjepa2_training.py
"""Offline, reduced V-JEPA 2 training mechanics in native PyTorch.

This random-weight reference uses 8-frame 32-pixel clips, two masks, two-layer
encoders, standard pairwise 3D RoPE, and AdamW. It is not the full training
recipe or a pretrained model. Original released checkpoints retain a documented
RoPE frequency-repeat bug; the separate native export preserves that behavior.
"""

import argparse
import copy
import json

import torch
from torch import nn
from torch.nn import functional as F


# region: tubelets
class Tubelets(nn.Module):
    """Projects non-overlapping video tubelets in time-height-width order."""

    def __init__(self, width=48, tubelet=2, patch=8):
        super().__init__()
        if min(width, tubelet, patch) <= 0:
            raise ValueError('Tubelet dimensions and width must be positive.')
        self.size = (tubelet, patch, patch)
        self.proj = nn.Conv3d(3, width, self.size, stride=self.size)

    def forward(self, video):
        if video.ndim != 5 or video.shape[1] != 3:
            raise ValueError('Expected [batch, 3, time, height, width].')
        if any(n <= 0 or n % p for n, p in zip(video.shape[2:], self.size)):
            raise ValueError('Video dimensions must be positive tubelet multiples.')
        grid = tuple(n // p for n, p in zip(video.shape[2:], self.size))
        return self.proj(video).flatten(2).transpose(1, 2), grid
# endregion: tubelets


# region: masks
def make_masks(grid, generator, count=2):
    """Unions spatial rectangles across ALL times; returns ordered complements."""
    time, height, width = grid
    if min(time, height, width) < 2 or count < 1:
        raise ValueError('Mask grid axes must be >= 2; count must be positive.')
    pairs = []
    for _ in range(count):
        spatial = torch.zeros(height, width, dtype=torch.bool)
        for _ in range(2):
            h, w = max(1, height // 2), max(1, width // 2)
            top = int(torch.randint(height - h + 1, (), generator=generator))
            left = int(torch.randint(width - w + 1, (), generator=generator))
            spatial[top:top + h, left:left + w] = True
        target = spatial.expand(time, -1, -1).flatten()
        # nonzero preserves chronological, row-major tubelet ids.
        pairs.append(((~target).nonzero().flatten(), target.nonzero().flatten()))
    return pairs
# endregion: masks


# region: rope
def rope3d(values, ids, grid):
    """Rotates Q/K pairs separately for time, row and column positions."""
    axis_width = 2 * ((values.shape[-1] // 3) // 2)
    if axis_width < 2:
        raise ValueError('Attention head width must be at least six.')
    _, height, width = grid
    positions = (ids // (height * width), (ids // width) % height, ids % width)
    frequencies = 10000.0 ** (-torch.arange(
        0, axis_width, 2, device=values.device, dtype=values.dtype) / axis_width)
    parts = []
    for axis, position in enumerate(positions):
        angle = position[:, None] * frequencies[None, :]
        pairs = values[..., axis * axis_width:(axis + 1) * axis_width]
        even, odd = pairs[..., 0::2], pairs[..., 1::2]
        rotated = torch.stack((even * angle.cos() - odd * angle.sin(),
                               even * angle.sin() + odd * angle.cos()), dim=-1)
        parts.append(rotated.flatten(-2))
    return torch.cat(parts + [values[..., 3 * axis_width:]], dim=-1)
# endregion: rope


class Block(nn.Module):
    """Pre-norm bidirectional attention and a GELU feedforward residual."""

    def __init__(self, width=48, heads=4):
        super().__init__()
        if min(width, heads) <= 0 or width % heads or width // heads < 6:
            raise ValueError('Width must divide heads, with head width >= 6.')
        self.heads = heads
        self.norm1 = nn.LayerNorm(width)
        self.qkv = nn.Linear(width, width * 3)
        self.proj = nn.Linear(width, width)
        self.norm2 = nn.LayerNorm(width)
        self.mlp = nn.Sequential(nn.Linear(width, width * 4), nn.GELU(),
                                 nn.Linear(width * 4, width))

    def forward(self, tokens, ids, grid):
        batch, length, width = tokens.shape
        qkv = self.qkv(self.norm1(tokens)).reshape(
            batch, length, 3, self.heads, width // self.heads)
        q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(0)
        attended = F.scaled_dot_product_attention(
            rope3d(q, ids, grid), rope3d(k, ids, grid), v, is_causal=False)
        tokens = tokens + self.proj(attended.transpose(1, 2).reshape(
            batch, length, width))
        return tokens + self.mlp(self.norm2(tokens))


# region: encoder
class Encoder(nn.Module):
    """Encodes retained tubelets while preserving their original positions."""

    def __init__(self, width=48):
        super().__init__()
        self.tubelets = Tubelets(width)
        self.blocks = nn.ModuleList([Block(width) for _ in range(2)])
        self.norm = nn.LayerNorm(width)

    def forward(self, video, ids=None):
        tokens, grid = self.tubelets(video)
        if ids is None:
            ids = torch.arange(tokens.shape[1], device=video.device)
        tokens = tokens[:, ids]  # Drop masked tubelets before self-attention.
        for block in self.blocks:
            tokens = block(tokens, ids, grid)
        return self.norm(tokens)
# endregion: encoder


# region: teacher
def make_teacher(encoder):
    """Starts the momentum target at the student weights, without gradients."""
    return copy.deepcopy(encoder).requires_grad_(False).eval()


@torch.no_grad()
def teacher_targets(teacher, video):
    """Encodes the full video, then normalizes each target feature vector."""
    targets = teacher(video)
    return F.layer_norm(targets, (targets.shape[-1],)).detach()
# endregion: teacher


# region: predictor
class Predictor(nn.Module):
    """Combines encoded context with learned queries at missing positions."""

    def __init__(self, encoder_width=48, width=24):
        super().__init__()
        self.input = nn.Linear(encoder_width, width)
        self.query = nn.Parameter(torch.zeros(1, 1, width))
        self.block = Block(width, heads=2)
        self.norm = nn.LayerNorm(width)
        self.output = nn.Linear(width, encoder_width)

    def forward(self, context, context_ids, target_ids, grid):
        queries = self.query.expand(context.shape[0], len(target_ids), -1)
        tokens = torch.cat((self.input(context), queries), dim=1)
        ids = torch.cat((context_ids, target_ids))
        # RoPE uses original ids even though context and targets are concatenated.
        tokens = self.block(tokens, ids, grid)
        return self.output(self.norm(tokens[:, len(context_ids):]))
# endregion: predictor


# region: loss
def masked_loss(encoder, predictor, teacher, video, masks):
    """Averages masked-only L1 across masks, never reconstructing visible tokens."""
    if not masks:
        raise ValueError('At least one mask is required.')
    targets = teacher_targets(teacher, video)
    grid = tuple(n // p for n, p in zip(video.shape[2:], encoder.tubelets.size))
    terms = []
    for context_ids, target_ids in masks:
        all_ids = torch.cat((context_ids, target_ids))
        expected = torch.arange(targets.shape[1], device=all_ids.device)
        if not len(context_ids) or not len(target_ids) or not torch.equal(
                all_ids.sort().values, expected):
            raise ValueError('Context and target ids must partition the video.')
        context = encoder(video, context_ids)
        prediction = predictor(context, context_ids, target_ids, grid)
        terms.append(F.l1_loss(prediction, targets[:, target_ids]))
    return torch.stack(terms).mean()
# endregion: loss


# region: ema
@torch.no_grad()
def update_teacher(encoder, teacher, momentum=0.99):
    """Updates the target AFTER the online optimizer step, with fixed momentum."""
    if not 0 <= momentum <= 1:
        raise ValueError('EMA momentum must lie in [0, 1].')
    for online, target in zip(encoder.parameters(), teacher.parameters()):
        target.mul_(momentum).add_(online, alpha=1 - momentum)


def train_step(encoder, predictor, teacher, optimizer, video, masks):
    optimizer.zero_grad(set_to_none=True)
    loss = masked_loss(encoder, predictor, teacher, video, masks)
    loss.backward()
    optimizer.step()
    update_teacher(encoder, teacher)
    return float(loss.detach())
# endregion: ema


def smoke():
    """Executes a small, reproducible, offline video-shaped training step."""
    torch.set_num_threads(2)
    torch.manual_seed(7)
    video = torch.randn(2, 3, 8, 32, 32)
    encoder, predictor = Encoder(), Predictor()
    teacher = make_teacher(encoder)
    masks = make_masks((4, 4, 4), torch.Generator().manual_seed(17))
    optimizer = torch.optim.AdamW(
        list(encoder.parameters()) + list(predictor.parameters()), lr=1e-3)
    losses = [train_step(encoder, predictor, teacher, optimizer, video, masks)
              for _ in range(3)]
    with torch.no_grad():
        features = encoder.eval()(video)
    return {'scope': 'Reduced random-weight numerical training; no pretrained semantics.',
            'inputShape': list(video.shape), 'featureShape': list(features.shape),
            'targetCounts': [len(target) for _, target in masks], 'losses': losses,
            'teacherHasGradients': any(p.grad is not None for p in teacher.parameters())}


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--smoke', action='store_true', required=True)
    parser.parse_args()
    print(json.dumps(smoke(), indent=2))
