"""Labels for one Ava-256 frame: point set 3D positions, per-camera projections, visibility.""" # Mesh points (classes lmk, skin, lid) are vertices of Meta's registration mesh (points_ava.npz # `src`), taken to the capture world through the head pose; teeth points are Meta's 3D keypoints # (incisal-edge ids with at least MIN_INL inliers, NaN when absent); iris rows stay NaN here. # Projection uses the capture's calibration with lens distortion, scaled to the photo tier. # Visibility comes from the registration mesh's own depth buffer, without the membrane triangles # that bridge the eyes and the mouth. Per-frame flags: the signed lid gap per eye and the # inner-lip gap, in mm. The Ava template OBJ is read from $AVA_TOPOLOGY, or downloaded once from # the ava-256 repository into ~/.cache/facepoints/. import io import os import urllib.request import cv2 import numpy as np MIN_INL = 4 # keypoint inliers for a teeth point to count Z_TOL = 3.0 # mm a point may sit behind the depth buffer and still be visible DEFAULT_POINTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "points_ava.npz") TOPOLOGY_URL = ( "https://raw.githubusercontent.com/facebookresearch/ava-256/main/assets/face_topology.obj" ) TOPOLOGY_CACHE = os.path.join( os.path.expanduser("~"), ".cache", "facepoints", "face_topology.obj" ) def topology_path(path=None): """Return the Ava template OBJ path: the argument, $AVA_TOPOLOGY, or a cached download.""" path = path or os.environ.get("AVA_TOPOLOGY") or TOPOLOGY_CACHE if not os.path.exists(path): os.makedirs(os.path.dirname(path), exist_ok=True) urllib.request.urlretrieve(TOPOLOGY_URL, path) return path def read_ply(buf): """Return the (N, 3) float64 vertices of a binary PLY from its first three float properties.""" head, rest = buf.split(b"end_header\n", 1) lines = head.decode("ascii", "ignore").splitlines() n = next(int(ln.split()[2]) for ln in lines if ln.startswith("element vertex")) sz = {"float": 4, "float32": 4, "double": 8, "uchar": 1, "int": 4} stride = sum(sz[ln.split()[1]] for ln in lines if ln.startswith("property")) arr = np.frombuffer(rest[: n * stride], dtype=np.uint8).reshape(n, stride) return arr[:, :12].copy().view(np.float32).reshape(n, 3).astype(np.float64) def topology(path): """Return (vertices, faces, per-vertex UV, longest UV edge per face) of the Ava template OBJ.""" V, VT, F, FT = [], [], [], [] with open(path) as fh: for ln in fh: if ln.startswith("v "): V.append([float(x) for x in ln.split()[1:4]]) elif ln.startswith("vt "): VT.append([float(x) for x in ln.split()[1:3]]) elif ln.startswith("f "): F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]]) FT.append([int(t.split("/")[1]) - 1 for t in ln.split()[1:4]]) V, VT, F, FT = np.array(V), np.array(VT), np.array(F), np.array(FT) el = np.linalg.norm(VT[FT] - VT[np.roll(FT, 1, axis=1)], axis=-1).max(1) uv = np.full((len(V), 2), np.nan) for f_, ft in zip(F, FT, strict=True): uv[f_] = VT[ft] return V, F, uv, el def membranes(V, F, el, centres, radius): """Return a face mask of the triangles bridging an opening (near a centre, long UV edge).""" # the UV-edge test is relative to the local median because a global rule also catches UV seams out = np.zeros(len(F), bool) for c in centres: near = np.all(np.linalg.norm(V[F] - c, axis=-1) < radius, axis=1) out |= near & (el > 3 * np.median(el[near])) return out def lip_contours(V, F, mouth_bridge, uv, eye_c): """Return (upper ids, lower ids) of the inner-lip ring along the mouth (upper = nearer eyes).""" low = np.unique(F[mouth_bridge]) cen = uv[low].mean(0) ax = uv[low][np.argmax(uv[low][:, 0])] - uv[low][np.argmin(uv[low][:, 0])] ax /= np.linalg.norm(ax) nrm = np.array([-ax[1], ax[0]]) s_, al = (uv[low] - cen) @ nrm, (uv[low] - cen) @ ax a, b = low[s_ > 0][np.argsort(al[s_ > 0])], low[s_ <= 0][np.argsort(al[s_ <= 0])] a_first = np.linalg.norm(V[a].mean(0) - eye_c) < np.linalg.norm(V[b].mean(0) - eye_c) return (a, b) if a_first else (b, a) class PointSet: """The label point set on the Ava topology and its evaluation on a registration + keypoints.""" def __init__(self, points=DEFAULT_POINTS, topology_obj=None): z = np.load(points) self.cls, self.src = z["cls"].astype(str), z["src"].astype(int) self.region = z["region"].astype(str) self.n = len(self.cls) self.mesh = np.isin(self.cls, ["lmk", "skin", "lid"]) self.teeth = np.isin(self.cls, ["teeth_u", "teeth_l"]) self.V0, self.F, uv0, el = topology(topology_path(topology_obj)) lid = self.cls == "lid" eyes = [self.V0[self.src[lid & (self.region == f"{e}_upper")]].mean(0) for e in "LR"] mouth = self.V0[self.src[(self.cls == "skin") & (self.region == "perioral")]].mean(0) eye_b = membranes(self.V0, self.F, el, eyes, 22.0) mouth_b = membranes(self.V0, self.F, el, [mouth], 45.0) self.Fz = self.F[~(eye_b | mouth_b)] # the triangles the depth buffer uses self.lips = lip_contours(self.V0, self.F, mouth_b, uv0, np.mean(eyes, axis=0)) self.lid_sets = { e: ( self.src[lid & (self.region == f"{e}_upper")], self.src[lid & (self.region == f"{e}_lower")], ) for e in "LR" } def points3d(self, X, kp): """Return (n, 3) positions in capture world: mesh points from X, teeth from kp, else NaN.""" P = np.full((self.n, 3), np.nan) P[self.mesh] = X[self.src[self.mesh]] ok = kp[:, 5] >= MIN_INL ids = kp[:, 0].astype(int) for i in np.flatnonzero(self.teeth): m = ok & (ids == self.src[i]) if m.any(): P[i] = kp[m][0, 1:4] return P def flags(self, X, up): """Return {lid_L, lid_R: signed lid gap along `up` (mm), lip_gap: inner-lip gap (mm)}.""" out = {} for e, (u, lo) in self.lid_sets.items(): g = [] for i in u[1:-1]: d = X[lo] - X[i] j = np.argmin(np.linalg.norm(d, axis=1)) g.append(-(d[j] @ up[e])) out[f"lid_{e}"] = float(np.mean(g)) u, lo = self.lips out["lip_gap"] = float( np.mean([np.linalg.norm(X[lo] - X[i], axis=1).min() for i in u[1:-1]]) ) return out def camera(e, scale): """Return (K, RT, distortion, rvec) of an Ava KRT entry for a photo at `scale` x native size.""" K = np.array(e["K"], np.float64).T.copy() K[:2] *= scale RT = np.array(e["T"], np.float64)[:4, :3].T dist = np.array(list(e["distortion"]) + [0.0] * 5, np.float64)[:5] return K, RT, dist, cv2.Rodrigues(RT[:3, :3])[0] def project(P, cam): """Return ((n, 2) pixels, (n,) depth) of points P; NaN rows pass through.""" K, RT, dist, rvec = cam uv = np.full((len(P), 2), np.nan) z = np.full(len(P), np.nan) ok = np.isfinite(P).all(1) if ok.any(): u, _ = cv2.projectPoints(P[ok], rvec, RT[:3, 3], K, dist) uv[ok] = u[:, 0] z[ok] = (P[ok] @ RT[:3, :3].T + RT[:3, 3])[:, 2] return uv, z def zbuffer(X, Fz, cam, hw): """Return the painter's depth buffer (H, W) of the mesh X over the triangles Fz.""" uv, z = project(X, cam) H, W = hw zb = np.full((H, W), np.inf, np.float32) tz = z[Fz].mean(1) for t in np.argsort(-tz): pts = np.round(uv[Fz[t]]).astype(np.int32) if (pts < 0).any() or (pts[:, 0] >= W).any() or (pts[:, 1] >= H).any(): continue cv2.fillConvexPoly(zb, pts, float(tz[t])) return zb def visible(uv, z, zb): """Return a mask of the points not hidden by the depth buffer by more than Z_TOL mm.""" H, W = zb.shape ok = np.isfinite(uv).all(1) v = np.zeros(len(uv), bool) x = np.clip(np.round(uv[ok, 0]).astype(int), 0, W - 1) y = np.clip(np.round(uv[ok, 1]).astype(int), 0, H - 1) v[ok] = zb[y, x] >= z[ok] - Z_TOL return v def frame_labels(ps, X, kp, cal, cam_ids, hw, scale): """Return ({cam: (uv (n, 2), visible (n,), depth (n,))}, P (n, 3)) for one frame.""" P = ps.points3d(X, kp) out = {} for c in cam_ids: cam = camera(cal[c], scale) uv, z = project(P, cam) zb = zbuffer(X, ps.Fz, cam, hw) out[c] = (uv, visible(uv, z, zb), z) return out, P def world(regz, posz, kpz, f): """Return (registration in capture world, head pose, keypoints) of frame f from the zips.""" X = read_ply(regz.read(f"{f:06d}.ply")) M = np.loadtxt(io.BytesIO(posz.read(f"{f:06d}.txt"))) kp = np.load(io.BytesIO(kpz.read(f"{f:06d}.npy"))) return X @ M[:3, :3].T + M[:3, 3], M, kp