"""Lightweight event simulator: standard video / image sequence -> event voxel grids. Provides an accessible input path for the LongE2V demo. Instead of requiring pre-computed event-camera voxel `.npz` files (which most users won't have), this module synthesizes an event stream from an ordinary video clip using the classic frame-difference / log-intensity contrast-threshold model (the core model used by ESIM / v2e), then voxelizes each consecutive-frame interval into a 3-bin voxel grid using the SAME `events_to_voxel_torch` accumulation LongE2V uses to build its own training/eval voxels (see cdfan0627/LongE2V `prepare_voxels.py`). For each pair of consecutive frames we: 1. convert to grayscale log-intensity, 2. threshold the per-pixel log-intensity difference at contrast C to emit ON (+1) / OFF (-1) events, one event per crossed contrast level, 3. spread the events' timestamps uniformly across the inter-frame interval, 4. accumulate them into a `num_bins`-bin voxel with bilinear temporal weighting. The resulting per-interval voxel grids `(num_bins, H, W)` are exactly the format the LongE2V pipeline consumes (the demo then resizes + normalizes them just like it does for uploaded `.npz` voxels). """ from __future__ import annotations import numpy as np import torch def _events_to_image_torch(xs, ys, ps, sensor_size, device): """Accumulate signed event weights into a (H, W) image (LongE2V-compatible).""" img = torch.zeros(list(sensor_size), device=device, dtype=ps.dtype) xs = xs.long().to(device) ys = ys.long().to(device) ps = ps.to(device) img.index_put_((ys, xs), ps, accumulate=True) return img def events_to_voxel_torch(xs, ys, ts, ps, num_bins, sensor_size, device=None): """Build a num_bins-bin voxel grid from events. Ported 1:1 from cdfan0627/LongE2V `prepare_voxels.py::events_to_voxel_torch` so simulated events are voxelized identically to the model's real data. """ if device is None: device = xs.device if len(xs) == 0: return torch.zeros((num_bins, *sensor_size), device=device, dtype=torch.float32) dt = ts[-1] - ts[0] if abs(float(dt)) < 1e-9: t_norm = torch.linspace(0, num_bins - 1, steps=len(ts), device=device) else: t_norm = (ts - ts[0]) / dt * (num_bins - 1) zeros = torch.zeros(t_norm.size(), device=device) bins = [] for bin_idx in range(num_bins): bilinear_weights = torch.max(zeros, 1.0 - torch.abs(t_norm - bin_idx)) weights = ps * bilinear_weights bins.append(_events_to_image_torch(xs, ys, weights, sensor_size, device)) return torch.stack(bins) def _to_log_intensity(frame_gray: np.ndarray, threshold_c: float) -> np.ndarray: """Map an 8-bit grayscale frame to log intensity, quantized into contrast levels.""" # log(intensity) with a small offset for numerical stability (v2e-style) log = np.log(frame_gray.astype(np.float32) / 255.0 + 1e-3) # Quantize into integer numbers of contrast steps so event counts are exact. return log / threshold_c def simulate_events_between_frames( prev_gray: np.ndarray, curr_gray: np.ndarray, t_start: float, t_end: float, threshold_c: float, ): """Simulate ON/OFF events between two consecutive grayscale frames. Uses the standard contrast-threshold model: a pixel emits one event per crossed contrast level of log-intensity change (sign = polarity). Event timestamps are spread uniformly across [t_start, t_end). Returns numpy arrays (xs, ys, ts, ps) with ps in {+1, -1}. """ prev_q = _to_log_intensity(prev_gray, threshold_c) curr_q = _to_log_intensity(curr_gray, threshold_c) # Number of contrast levels crossed per pixel (signed). diff_levels = np.floor(curr_q).astype(np.int32) - np.floor(prev_q).astype(np.int32) ys, xs = np.nonzero(diff_levels) if len(xs) == 0: empty = np.array([], dtype=np.float32) return empty, empty, empty, empty counts = diff_levels[ys, xs] # signed number of events at each pixel abs_counts = np.abs(counts) polarity = np.sign(counts).astype(np.float32) # Expand each pixel into |count| events. xs_e = np.repeat(xs, abs_counts).astype(np.float32) ys_e = np.repeat(ys, abs_counts).astype(np.float32) ps_e = np.repeat(polarity, abs_counts).astype(np.float32) n = len(xs_e) # Spread timestamps uniformly across the interval (order within a pixel is arbitrary). ts_e = (t_start + (t_end - t_start) * (np.arange(n, dtype=np.float32) + 0.5) / n).astype(np.float32) return xs_e, ys_e, ts_e, ps_e def video_frames_to_voxels( frames_gray: list[np.ndarray], num_bins: int = 3, threshold_c: float = 0.2, device: str | torch.device = "cpu", ): """Convert a list of consecutive grayscale frames into per-interval voxel grids. Args: frames_gray: list of HxW uint8 grayscale numpy arrays (>= 2 frames). num_bins: temporal voxel bins (LongE2V uses 3). threshold_c: contrast threshold for event generation (lower => more events). device: torch device for accumulation. Returns: torch.FloatTensor of shape (N-1, num_bins, H, W) — one voxel per frame interval, matching the raw voxel format the LongE2V pipeline consumes. """ if len(frames_gray) < 2: raise ValueError("Need at least 2 frames to simulate events.") h, w = frames_gray[0].shape[:2] sensor_size = (h, w) device = torch.device(device) voxels = [] n_frames = len(frames_gray) for i in range(n_frames - 1): t_start = float(i) t_end = float(i + 1) xs, ys, ts, ps = simulate_events_between_frames( frames_gray[i], frames_gray[i + 1], t_start, t_end, threshold_c ) if len(xs) == 0: voxels.append(torch.zeros((num_bins, h, w), dtype=torch.float32, device=device)) continue xs_t = torch.from_numpy(xs.astype(np.int64)).to(device) ys_t = torch.from_numpy(ys.astype(np.int64)).to(device) ts_t = torch.from_numpy(ts.astype(np.float32)).to(device) ps_t = torch.from_numpy(ps.astype(np.float32)).to(device) voxel = events_to_voxel_torch(xs_t, ys_t, ts_t, ps_t, num_bins, sensor_size, device) voxels.append(voxel) return torch.stack(voxels, dim=0) # (N-1, num_bins, H, W)