Spaces:
Running on Zero
Running on Zero
Download event_simulator.py from fansam39/longe2v-event-video: direct link, hf CLI and curl.
- Browser
- Download file 6.4 kB
-
https://huggingface.co/spaces/fansam39/longe2v-event-video/resolve/main/event_simulator.py
- Command line
-
hf download hf://spaces/fansam39/longe2v-event-video/event_simulator.py
-
curl -L -o event_simulator.py https://huggingface.co/spaces/fansam39/longe2v-event-video/resolve/main/event_simulator.py
6.4 kB
| """Lightweight event simulator: standard video / image sequence -> event voxel grids. | |
| Provides an accessible input path for the LongE2V demo. Instead of requiring | |
| pre-computed event-camera voxel `.npz` files (which most users won't have), this | |
| module synthesizes an event stream from an ordinary video clip using the classic | |
| frame-difference / log-intensity contrast-threshold model (the core model used by | |
| ESIM / v2e), then voxelizes each consecutive-frame interval into a 3-bin voxel | |
| grid using the SAME `events_to_voxel_torch` accumulation LongE2V uses to build its | |
| own training/eval voxels (see cdfan0627/LongE2V `prepare_voxels.py`). | |
| For each pair of consecutive frames we: | |
| 1. convert to grayscale log-intensity, | |
| 2. threshold the per-pixel log-intensity difference at contrast C to emit ON | |
| (+1) / OFF (-1) events, one event per crossed contrast level, | |
| 3. spread the events' timestamps uniformly across the inter-frame interval, | |
| 4. accumulate them into a `num_bins`-bin voxel with bilinear temporal weighting. | |
| The resulting per-interval voxel grids `(num_bins, H, W)` are exactly the format | |
| the LongE2V pipeline consumes (the demo then resizes + normalizes them just like | |
| it does for uploaded `.npz` voxels). | |
| """ | |
| from __future__ import annotations | |
| import numpy as np | |
| import torch | |
| def _events_to_image_torch(xs, ys, ps, sensor_size, device): | |
| """Accumulate signed event weights into a (H, W) image (LongE2V-compatible).""" | |
| img = torch.zeros(list(sensor_size), device=device, dtype=ps.dtype) | |
| xs = xs.long().to(device) | |
| ys = ys.long().to(device) | |
| ps = ps.to(device) | |
| img.index_put_((ys, xs), ps, accumulate=True) | |
| return img | |
| def events_to_voxel_torch(xs, ys, ts, ps, num_bins, sensor_size, device=None): | |
| """Build a num_bins-bin voxel grid from events. | |
| Ported 1:1 from cdfan0627/LongE2V `prepare_voxels.py::events_to_voxel_torch` | |
| so simulated events are voxelized identically to the model's real data. | |
| """ | |
| if device is None: | |
| device = xs.device | |
| if len(xs) == 0: | |
| return torch.zeros((num_bins, *sensor_size), device=device, dtype=torch.float32) | |
| dt = ts[-1] - ts[0] | |
| if abs(float(dt)) < 1e-9: | |
| t_norm = torch.linspace(0, num_bins - 1, steps=len(ts), device=device) | |
| else: | |
| t_norm = (ts - ts[0]) / dt * (num_bins - 1) | |
| zeros = torch.zeros(t_norm.size(), device=device) | |
| bins = [] | |
| for bin_idx in range(num_bins): | |
| bilinear_weights = torch.max(zeros, 1.0 - torch.abs(t_norm - bin_idx)) | |
| weights = ps * bilinear_weights | |
| bins.append(_events_to_image_torch(xs, ys, weights, sensor_size, device)) | |
| return torch.stack(bins) | |
| def _to_log_intensity(frame_gray: np.ndarray, threshold_c: float) -> np.ndarray: | |
| """Map an 8-bit grayscale frame to log intensity, quantized into contrast levels.""" | |
| # log(intensity) with a small offset for numerical stability (v2e-style) | |
| log = np.log(frame_gray.astype(np.float32) / 255.0 + 1e-3) | |
| # Quantize into integer numbers of contrast steps so event counts are exact. | |
| return log / threshold_c | |
| def simulate_events_between_frames( | |
| prev_gray: np.ndarray, | |
| curr_gray: np.ndarray, | |
| t_start: float, | |
| t_end: float, | |
| threshold_c: float, | |
| ): | |
| """Simulate ON/OFF events between two consecutive grayscale frames. | |
| Uses the standard contrast-threshold model: a pixel emits one event per | |
| crossed contrast level of log-intensity change (sign = polarity). Event | |
| timestamps are spread uniformly across [t_start, t_end). | |
| Returns numpy arrays (xs, ys, ts, ps) with ps in {+1, -1}. | |
| """ | |
| prev_q = _to_log_intensity(prev_gray, threshold_c) | |
| curr_q = _to_log_intensity(curr_gray, threshold_c) | |
| # Number of contrast levels crossed per pixel (signed). | |
| diff_levels = np.floor(curr_q).astype(np.int32) - np.floor(prev_q).astype(np.int32) | |
| ys, xs = np.nonzero(diff_levels) | |
| if len(xs) == 0: | |
| empty = np.array([], dtype=np.float32) | |
| return empty, empty, empty, empty | |
| counts = diff_levels[ys, xs] # signed number of events at each pixel | |
| abs_counts = np.abs(counts) | |
| polarity = np.sign(counts).astype(np.float32) | |
| # Expand each pixel into |count| events. | |
| xs_e = np.repeat(xs, abs_counts).astype(np.float32) | |
| ys_e = np.repeat(ys, abs_counts).astype(np.float32) | |
| ps_e = np.repeat(polarity, abs_counts).astype(np.float32) | |
| n = len(xs_e) | |
| # Spread timestamps uniformly across the interval (order within a pixel is arbitrary). | |
| ts_e = (t_start + (t_end - t_start) * (np.arange(n, dtype=np.float32) + 0.5) / n).astype(np.float32) | |
| return xs_e, ys_e, ts_e, ps_e | |
| def video_frames_to_voxels( | |
| frames_gray: list[np.ndarray], | |
| num_bins: int = 3, | |
| threshold_c: float = 0.2, | |
| device: str | torch.device = "cpu", | |
| ): | |
| """Convert a list of consecutive grayscale frames into per-interval voxel grids. | |
| Args: | |
| frames_gray: list of HxW uint8 grayscale numpy arrays (>= 2 frames). | |
| num_bins: temporal voxel bins (LongE2V uses 3). | |
| threshold_c: contrast threshold for event generation (lower => more events). | |
| device: torch device for accumulation. | |
| Returns: | |
| torch.FloatTensor of shape (N-1, num_bins, H, W) — one voxel per frame | |
| interval, matching the raw voxel format the LongE2V pipeline consumes. | |
| """ | |
| if len(frames_gray) < 2: | |
| raise ValueError("Need at least 2 frames to simulate events.") | |
| h, w = frames_gray[0].shape[:2] | |
| sensor_size = (h, w) | |
| device = torch.device(device) | |
| voxels = [] | |
| n_frames = len(frames_gray) | |
| for i in range(n_frames - 1): | |
| t_start = float(i) | |
| t_end = float(i + 1) | |
| xs, ys, ts, ps = simulate_events_between_frames( | |
| frames_gray[i], frames_gray[i + 1], t_start, t_end, threshold_c | |
| ) | |
| if len(xs) == 0: | |
| voxels.append(torch.zeros((num_bins, h, w), dtype=torch.float32, device=device)) | |
| continue | |
| xs_t = torch.from_numpy(xs.astype(np.int64)).to(device) | |
| ys_t = torch.from_numpy(ys.astype(np.int64)).to(device) | |
| ts_t = torch.from_numpy(ts.astype(np.float32)).to(device) | |
| ps_t = torch.from_numpy(ps.astype(np.float32)).to(device) | |
| voxel = events_to_voxel_torch(xs_t, ys_t, ts_t, ps_t, num_bins, sensor_size, device) | |
| voxels.append(voxel) | |
| return torch.stack(voxels, dim=0) # (N-1, num_bins, H, W) | |