facepoints / prep /sample_frames.py
antonios-makro's picture
Facepoints 1.0.0: code, model, prep data and card
88607d8 verified
Raw History Blame Contribute Delete
5.29 kB
"""The training sample list: per Ava-256 capture, the most diverse frames and random cameras."""
# Usage: python -m facepoints.prep.sample_frames <out.json> [--expressions 50]
# [--cameras 20] [--seed 0] [--captures captures.txt]
# Expression diversity is measured on Ava-256's 3D keypoints in the head frame (ids present in at
# least 90 % of the capture's frames; an absent id takes its per-capture median; xyz concatenated)
# by farthest-point sampling from the capture's first frame. The cameras are a seeded random subset
# of the capture's 80 per selected frame (50 x 20 = 1,000 images per capture by default). The
# keypoint and head-pose zips are read whole per capture (about 50 MB). Output: {capture: {frames:
# [...], cams: {frame: [camera ids]}, n_frames, n_ids_used}}, or {skipped: reason}; resumable per
# capture through <out>.part/.
import argparse
import collections
import concurrent.futures as cf
import io
import json
import os
import random
import urllib.request
import zipfile
import numpy as np
BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/4TB"
DEFAULT_CAPTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "captures.txt")
MIN_SHARE = 0.9 # a keypoint id must be present in this share of the frames to be used
def fetch_zip(url):
"""Return a whole remote zip as a ZipFile."""
with urllib.request.urlopen(url, timeout=300) as r:
return zipfile.ZipFile(io.BytesIO(r.read()))
def one_capture(cap, part, n_expr, n_cams, seed):
"""Return (capture, its sample dict), reading or writing <part>/<capture>.json."""
pf = os.path.join(part, cap + ".json")
if os.path.exists(pf):
with open(pf) as fh:
return cap, json.load(fh)
D = f"{BASE}/{cap}/decoder/"
try:
zk = fetch_zip(D + "keypoints_3d/keypoints_3d.zip")
zp = fetch_zip(D + "head_pose/head_pose.zip")
cal = json.load(urllib.request.urlopen(D + "camera_calibration.json", timeout=60))["KRT"]
except Exception as e: # noqa: BLE001 - a capture that cannot be read is skipped, not fatal
return cap, {"skipped": str(e)[:80]}
cams = [e["cameraId"] for e in cal]
frames = sorted(
int(n.split("/")[-1].split(".")[0]) for n in zk.namelist() if n.endswith(".npy")
)
poses = {int(n.split("/")[-1].split(".")[0]) for n in zp.namelist() if n.endswith(".txt")}
frames = [f for f in frames if f in poses]
K = {}
for f in frames:
k = np.load(io.BytesIO(zk.read(f"{f:06d}.npy")))
M = np.loadtxt(io.BytesIO(zp.read(f"{f:06d}.txt")))
P = (k[:, 1:4] - M[:3, 3]) @ M[:3, :3] # head frame
K[f] = dict(zip(k[:, 0].astype(int).tolist(), P.tolist(), strict=True))
cnt = collections.Counter(i for v in K.values() for i in v)
ids = sorted(i for i, n in cnt.items() if n >= MIN_SHARE * len(frames))
X = np.full((len(frames), len(ids), 3), np.nan)
for r, f in enumerate(frames):
for j, i in enumerate(ids):
if i in K[f]:
X[r, j] = K[f][i]
X = np.where(np.isnan(X), np.nanmedian(X, axis=0)[None], X).reshape(len(frames), -1)
chosen = [0]
d = np.linalg.norm(X - X[0], axis=1)
while len(chosen) < min(n_expr, len(frames)) and d.max() > 0:
j = int(np.argmax(d))
chosen.append(j)
d = np.minimum(d, np.linalg.norm(X - X[j], axis=1))
rng = random.Random(f"{seed}:{cap}")
sel = sorted(frames[j] for j in chosen)
res = {
"frames": sel,
"cams": {str(f): sorted(rng.sample(cams, min(n_cams, len(cams)))) for f in sel},
"n_frames": len(frames),
"n_ids_used": len(ids),
}
with open(pf, "w") as fh:
json.dump(res, fh)
return cap, res
def main():
"""Sample every capture of the list and write the sample dict."""
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("out", help="output json")
ap.add_argument("--expressions", type=int, default=50, help="frames per capture")
ap.add_argument("--cameras", type=int, default=20, help="cameras per frame")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--captures", default=DEFAULT_CAPTURES, help="file of Ava-256 capture ids")
a = ap.parse_args()
with open(a.captures) as fh:
caps = fh.read().split()
part = a.out + ".part"
os.makedirs(part, exist_ok=True)
R = {}
with cf.ThreadPoolExecutor(6) as ex:
jobs = (ex.submit(one_capture, c, part, a.expressions, a.cameras, a.seed) for c in caps)
for i, fu in enumerate(list(jobs)):
cap, r = fu.result()
R[cap] = r
if i % 16 == 0:
print(
f"{i + 1}/{len(caps)} {cap[-6:]} {r.get('skipped', '') or len(r['frames'])}",
flush=True,
)
with open(a.out, "w") as fh:
json.dump(R, fh)
ok = [c for c, r in R.items() if "skipped" not in r]
n_img = sum(len(v) for c in ok for v in R[c]["cams"].values())
n_fr = sum(len(R[c]["frames"]) for c in ok)
skipped = [c[-6:] for c in R if "skipped" in R[c]]
print(
f"samples: {len(ok)} captures, {n_fr} frames, {n_img} images; skipped {skipped} -> {a.out}"
)
if __name__ == "__main__":
main()