Download prep/sample_frames.py from myned-ai/facepoints: direct link, hf CLI and curl.
- Browser
- Download file 5.29 kB
-
https://huggingface.co/myned-ai/facepoints/resolve/main/prep/sample_frames.py
- Command line
-
hf download hf://myned-ai/facepoints/prep/sample_frames.py
-
curl -L -o sample_frames.py https://huggingface.co/myned-ai/facepoints/resolve/main/prep/sample_frames.py
5.29 kB
| """The training sample list: per Ava-256 capture, the most diverse frames and random cameras.""" | |
| # Usage: python -m facepoints.prep.sample_frames <out.json> [--expressions 50] | |
| # [--cameras 20] [--seed 0] [--captures captures.txt] | |
| # Expression diversity is measured on Ava-256's 3D keypoints in the head frame (ids present in at | |
| # least 90 % of the capture's frames; an absent id takes its per-capture median; xyz concatenated) | |
| # by farthest-point sampling from the capture's first frame. The cameras are a seeded random subset | |
| # of the capture's 80 per selected frame (50 x 20 = 1,000 images per capture by default). The | |
| # keypoint and head-pose zips are read whole per capture (about 50 MB). Output: {capture: {frames: | |
| # [...], cams: {frame: [camera ids]}, n_frames, n_ids_used}}, or {skipped: reason}; resumable per | |
| # capture through <out>.part/. | |
| import argparse | |
| import collections | |
| import concurrent.futures as cf | |
| import io | |
| import json | |
| import os | |
| import random | |
| import urllib.request | |
| import zipfile | |
| import numpy as np | |
| BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/4TB" | |
| DEFAULT_CAPTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "captures.txt") | |
| MIN_SHARE = 0.9 # a keypoint id must be present in this share of the frames to be used | |
| def fetch_zip(url): | |
| """Return a whole remote zip as a ZipFile.""" | |
| with urllib.request.urlopen(url, timeout=300) as r: | |
| return zipfile.ZipFile(io.BytesIO(r.read())) | |
| def one_capture(cap, part, n_expr, n_cams, seed): | |
| """Return (capture, its sample dict), reading or writing <part>/<capture>.json.""" | |
| pf = os.path.join(part, cap + ".json") | |
| if os.path.exists(pf): | |
| with open(pf) as fh: | |
| return cap, json.load(fh) | |
| D = f"{BASE}/{cap}/decoder/" | |
| try: | |
| zk = fetch_zip(D + "keypoints_3d/keypoints_3d.zip") | |
| zp = fetch_zip(D + "head_pose/head_pose.zip") | |
| cal = json.load(urllib.request.urlopen(D + "camera_calibration.json", timeout=60))["KRT"] | |
| except Exception as e: # noqa: BLE001 - a capture that cannot be read is skipped, not fatal | |
| return cap, {"skipped": str(e)[:80]} | |
| cams = [e["cameraId"] for e in cal] | |
| frames = sorted( | |
| int(n.split("/")[-1].split(".")[0]) for n in zk.namelist() if n.endswith(".npy") | |
| ) | |
| poses = {int(n.split("/")[-1].split(".")[0]) for n in zp.namelist() if n.endswith(".txt")} | |
| frames = [f for f in frames if f in poses] | |
| K = {} | |
| for f in frames: | |
| k = np.load(io.BytesIO(zk.read(f"{f:06d}.npy"))) | |
| M = np.loadtxt(io.BytesIO(zp.read(f"{f:06d}.txt"))) | |
| P = (k[:, 1:4] - M[:3, 3]) @ M[:3, :3] # head frame | |
| K[f] = dict(zip(k[:, 0].astype(int).tolist(), P.tolist(), strict=True)) | |
| cnt = collections.Counter(i for v in K.values() for i in v) | |
| ids = sorted(i for i, n in cnt.items() if n >= MIN_SHARE * len(frames)) | |
| X = np.full((len(frames), len(ids), 3), np.nan) | |
| for r, f in enumerate(frames): | |
| for j, i in enumerate(ids): | |
| if i in K[f]: | |
| X[r, j] = K[f][i] | |
| X = np.where(np.isnan(X), np.nanmedian(X, axis=0)[None], X).reshape(len(frames), -1) | |
| chosen = [0] | |
| d = np.linalg.norm(X - X[0], axis=1) | |
| while len(chosen) < min(n_expr, len(frames)) and d.max() > 0: | |
| j = int(np.argmax(d)) | |
| chosen.append(j) | |
| d = np.minimum(d, np.linalg.norm(X - X[j], axis=1)) | |
| rng = random.Random(f"{seed}:{cap}") | |
| sel = sorted(frames[j] for j in chosen) | |
| res = { | |
| "frames": sel, | |
| "cams": {str(f): sorted(rng.sample(cams, min(n_cams, len(cams)))) for f in sel}, | |
| "n_frames": len(frames), | |
| "n_ids_used": len(ids), | |
| } | |
| with open(pf, "w") as fh: | |
| json.dump(res, fh) | |
| return cap, res | |
| def main(): | |
| """Sample every capture of the list and write the sample dict.""" | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("out", help="output json") | |
| ap.add_argument("--expressions", type=int, default=50, help="frames per capture") | |
| ap.add_argument("--cameras", type=int, default=20, help="cameras per frame") | |
| ap.add_argument("--seed", type=int, default=0) | |
| ap.add_argument("--captures", default=DEFAULT_CAPTURES, help="file of Ava-256 capture ids") | |
| a = ap.parse_args() | |
| with open(a.captures) as fh: | |
| caps = fh.read().split() | |
| part = a.out + ".part" | |
| os.makedirs(part, exist_ok=True) | |
| R = {} | |
| with cf.ThreadPoolExecutor(6) as ex: | |
| jobs = (ex.submit(one_capture, c, part, a.expressions, a.cameras, a.seed) for c in caps) | |
| for i, fu in enumerate(list(jobs)): | |
| cap, r = fu.result() | |
| R[cap] = r | |
| if i % 16 == 0: | |
| print( | |
| f"{i + 1}/{len(caps)} {cap[-6:]} {r.get('skipped', '') or len(r['frames'])}", | |
| flush=True, | |
| ) | |
| with open(a.out, "w") as fh: | |
| json.dump(R, fh) | |
| ok = [c for c, r in R.items() if "skipped" not in r] | |
| n_img = sum(len(v) for c in ok for v in R[c]["cams"].values()) | |
| n_fr = sum(len(R[c]["frames"]) for c in ok) | |
| skipped = [c[-6:] for c in R if "skipped" in R[c]] | |
| print( | |
| f"samples: {len(ok)} captures, {n_fr} frames, {n_img} images; skipped {skipped} -> {a.out}" | |
| ) | |
| if __name__ == "__main__": | |
| main() | |