"""The training sample list: per Ava-256 capture, the most diverse frames and random cameras.""" # Usage: python -m facepoints.prep.sample_frames [--expressions 50] # [--cameras 20] [--seed 0] [--captures captures.txt] # Expression diversity is measured on Ava-256's 3D keypoints in the head frame (ids present in at # least 90 % of the capture's frames; an absent id takes its per-capture median; xyz concatenated) # by farthest-point sampling from the capture's first frame. The cameras are a seeded random subset # of the capture's 80 per selected frame (50 x 20 = 1,000 images per capture by default). The # keypoint and head-pose zips are read whole per capture (about 50 MB). Output: {capture: {frames: # [...], cams: {frame: [camera ids]}, n_frames, n_ids_used}}, or {skipped: reason}; resumable per # capture through .part/. import argparse import collections import concurrent.futures as cf import io import json import os import random import urllib.request import zipfile import numpy as np BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/4TB" DEFAULT_CAPTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "captures.txt") MIN_SHARE = 0.9 # a keypoint id must be present in this share of the frames to be used def fetch_zip(url): """Return a whole remote zip as a ZipFile.""" with urllib.request.urlopen(url, timeout=300) as r: return zipfile.ZipFile(io.BytesIO(r.read())) def one_capture(cap, part, n_expr, n_cams, seed): """Return (capture, its sample dict), reading or writing /.json.""" pf = os.path.join(part, cap + ".json") if os.path.exists(pf): with open(pf) as fh: return cap, json.load(fh) D = f"{BASE}/{cap}/decoder/" try: zk = fetch_zip(D + "keypoints_3d/keypoints_3d.zip") zp = fetch_zip(D + "head_pose/head_pose.zip") cal = json.load(urllib.request.urlopen(D + "camera_calibration.json", timeout=60))["KRT"] except Exception as e: # noqa: BLE001 - a capture that cannot be read is skipped, not fatal return cap, {"skipped": str(e)[:80]} cams = [e["cameraId"] for e in cal] frames = sorted( int(n.split("/")[-1].split(".")[0]) for n in zk.namelist() if n.endswith(".npy") ) poses = {int(n.split("/")[-1].split(".")[0]) for n in zp.namelist() if n.endswith(".txt")} frames = [f for f in frames if f in poses] K = {} for f in frames: k = np.load(io.BytesIO(zk.read(f"{f:06d}.npy"))) M = np.loadtxt(io.BytesIO(zp.read(f"{f:06d}.txt"))) P = (k[:, 1:4] - M[:3, 3]) @ M[:3, :3] # head frame K[f] = dict(zip(k[:, 0].astype(int).tolist(), P.tolist(), strict=True)) cnt = collections.Counter(i for v in K.values() for i in v) ids = sorted(i for i, n in cnt.items() if n >= MIN_SHARE * len(frames)) X = np.full((len(frames), len(ids), 3), np.nan) for r, f in enumerate(frames): for j, i in enumerate(ids): if i in K[f]: X[r, j] = K[f][i] X = np.where(np.isnan(X), np.nanmedian(X, axis=0)[None], X).reshape(len(frames), -1) chosen = [0] d = np.linalg.norm(X - X[0], axis=1) while len(chosen) < min(n_expr, len(frames)) and d.max() > 0: j = int(np.argmax(d)) chosen.append(j) d = np.minimum(d, np.linalg.norm(X - X[j], axis=1)) rng = random.Random(f"{seed}:{cap}") sel = sorted(frames[j] for j in chosen) res = { "frames": sel, "cams": {str(f): sorted(rng.sample(cams, min(n_cams, len(cams)))) for f in sel}, "n_frames": len(frames), "n_ids_used": len(ids), } with open(pf, "w") as fh: json.dump(res, fh) return cap, res def main(): """Sample every capture of the list and write the sample dict.""" ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("out", help="output json") ap.add_argument("--expressions", type=int, default=50, help="frames per capture") ap.add_argument("--cameras", type=int, default=20, help="cameras per frame") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--captures", default=DEFAULT_CAPTURES, help="file of Ava-256 capture ids") a = ap.parse_args() with open(a.captures) as fh: caps = fh.read().split() part = a.out + ".part" os.makedirs(part, exist_ok=True) R = {} with cf.ThreadPoolExecutor(6) as ex: jobs = (ex.submit(one_capture, c, part, a.expressions, a.cameras, a.seed) for c in caps) for i, fu in enumerate(list(jobs)): cap, r = fu.result() R[cap] = r if i % 16 == 0: print( f"{i + 1}/{len(caps)} {cap[-6:]} {r.get('skipped', '') or len(r['frames'])}", flush=True, ) with open(a.out, "w") as fh: json.dump(R, fh) ok = [c for c, r in R.items() if "skipped" not in r] n_img = sum(len(v) for c in ok for v in R[c]["cams"].values()) n_fr = sum(len(R[c]["frames"]) for c in ok) skipped = [c[-6:] for c in R if "skipped" in R[c]] print( f"samples: {len(ok)} captures, {n_fr} frames, {n_img} images; skipped {skipped} -> {a.out}" ) if __name__ == "__main__": main()