longe2v-event-video / event_simulator.py
multimodalart's picture
multimodalart HF Staff
Upload event_simulator.py with huggingface_hub
6c1f693 verified
Raw History Blame Contribute Delete
6.4 kB
"""Lightweight event simulator: standard video / image sequence -> event voxel grids.
Provides an accessible input path for the LongE2V demo. Instead of requiring
pre-computed event-camera voxel `.npz` files (which most users won't have), this
module synthesizes an event stream from an ordinary video clip using the classic
frame-difference / log-intensity contrast-threshold model (the core model used by
ESIM / v2e), then voxelizes each consecutive-frame interval into a 3-bin voxel
grid using the SAME `events_to_voxel_torch` accumulation LongE2V uses to build its
own training/eval voxels (see cdfan0627/LongE2V `prepare_voxels.py`).
For each pair of consecutive frames we:
1. convert to grayscale log-intensity,
2. threshold the per-pixel log-intensity difference at contrast C to emit ON
(+1) / OFF (-1) events, one event per crossed contrast level,
3. spread the events' timestamps uniformly across the inter-frame interval,
4. accumulate them into a `num_bins`-bin voxel with bilinear temporal weighting.
The resulting per-interval voxel grids `(num_bins, H, W)` are exactly the format
the LongE2V pipeline consumes (the demo then resizes + normalizes them just like
it does for uploaded `.npz` voxels).
"""
from __future__ import annotations
import numpy as np
import torch
def _events_to_image_torch(xs, ys, ps, sensor_size, device):
"""Accumulate signed event weights into a (H, W) image (LongE2V-compatible)."""
img = torch.zeros(list(sensor_size), device=device, dtype=ps.dtype)
xs = xs.long().to(device)
ys = ys.long().to(device)
ps = ps.to(device)
img.index_put_((ys, xs), ps, accumulate=True)
return img
def events_to_voxel_torch(xs, ys, ts, ps, num_bins, sensor_size, device=None):
"""Build a num_bins-bin voxel grid from events.
Ported 1:1 from cdfan0627/LongE2V `prepare_voxels.py::events_to_voxel_torch`
so simulated events are voxelized identically to the model's real data.
"""
if device is None:
device = xs.device
if len(xs) == 0:
return torch.zeros((num_bins, *sensor_size), device=device, dtype=torch.float32)
dt = ts[-1] - ts[0]
if abs(float(dt)) < 1e-9:
t_norm = torch.linspace(0, num_bins - 1, steps=len(ts), device=device)
else:
t_norm = (ts - ts[0]) / dt * (num_bins - 1)
zeros = torch.zeros(t_norm.size(), device=device)
bins = []
for bin_idx in range(num_bins):
bilinear_weights = torch.max(zeros, 1.0 - torch.abs(t_norm - bin_idx))
weights = ps * bilinear_weights
bins.append(_events_to_image_torch(xs, ys, weights, sensor_size, device))
return torch.stack(bins)
def _to_log_intensity(frame_gray: np.ndarray, threshold_c: float) -> np.ndarray:
"""Map an 8-bit grayscale frame to log intensity, quantized into contrast levels."""
# log(intensity) with a small offset for numerical stability (v2e-style)
log = np.log(frame_gray.astype(np.float32) / 255.0 + 1e-3)
# Quantize into integer numbers of contrast steps so event counts are exact.
return log / threshold_c
def simulate_events_between_frames(
prev_gray: np.ndarray,
curr_gray: np.ndarray,
t_start: float,
t_end: float,
threshold_c: float,
):
"""Simulate ON/OFF events between two consecutive grayscale frames.
Uses the standard contrast-threshold model: a pixel emits one event per
crossed contrast level of log-intensity change (sign = polarity). Event
timestamps are spread uniformly across [t_start, t_end).
Returns numpy arrays (xs, ys, ts, ps) with ps in {+1, -1}.
"""
prev_q = _to_log_intensity(prev_gray, threshold_c)
curr_q = _to_log_intensity(curr_gray, threshold_c)
# Number of contrast levels crossed per pixel (signed).
diff_levels = np.floor(curr_q).astype(np.int32) - np.floor(prev_q).astype(np.int32)
ys, xs = np.nonzero(diff_levels)
if len(xs) == 0:
empty = np.array([], dtype=np.float32)
return empty, empty, empty, empty
counts = diff_levels[ys, xs] # signed number of events at each pixel
abs_counts = np.abs(counts)
polarity = np.sign(counts).astype(np.float32)
# Expand each pixel into |count| events.
xs_e = np.repeat(xs, abs_counts).astype(np.float32)
ys_e = np.repeat(ys, abs_counts).astype(np.float32)
ps_e = np.repeat(polarity, abs_counts).astype(np.float32)
n = len(xs_e)
# Spread timestamps uniformly across the interval (order within a pixel is arbitrary).
ts_e = (t_start + (t_end - t_start) * (np.arange(n, dtype=np.float32) + 0.5) / n).astype(np.float32)
return xs_e, ys_e, ts_e, ps_e
def video_frames_to_voxels(
frames_gray: list[np.ndarray],
num_bins: int = 3,
threshold_c: float = 0.2,
device: str | torch.device = "cpu",
):
"""Convert a list of consecutive grayscale frames into per-interval voxel grids.
Args:
frames_gray: list of HxW uint8 grayscale numpy arrays (>= 2 frames).
num_bins: temporal voxel bins (LongE2V uses 3).
threshold_c: contrast threshold for event generation (lower => more events).
device: torch device for accumulation.
Returns:
torch.FloatTensor of shape (N-1, num_bins, H, W) — one voxel per frame
interval, matching the raw voxel format the LongE2V pipeline consumes.
"""
if len(frames_gray) < 2:
raise ValueError("Need at least 2 frames to simulate events.")
h, w = frames_gray[0].shape[:2]
sensor_size = (h, w)
device = torch.device(device)
voxels = []
n_frames = len(frames_gray)
for i in range(n_frames - 1):
t_start = float(i)
t_end = float(i + 1)
xs, ys, ts, ps = simulate_events_between_frames(
frames_gray[i], frames_gray[i + 1], t_start, t_end, threshold_c
)
if len(xs) == 0:
voxels.append(torch.zeros((num_bins, h, w), dtype=torch.float32, device=device))
continue
xs_t = torch.from_numpy(xs.astype(np.int64)).to(device)
ys_t = torch.from_numpy(ys.astype(np.int64)).to(device)
ts_t = torch.from_numpy(ts.astype(np.float32)).to(device)
ps_t = torch.from_numpy(ps.astype(np.float32)).to(device)
voxel = events_to_voxel_torch(xs_t, ys_t, ts_t, ps_t, num_bins, sensor_size, device)
voxels.append(voxel)
return torch.stack(voxels, dim=0) # (N-1, num_bins, H, W)