Download prep/labels.py from myned-ai/facepoints: direct link, hf CLI and curl.
- Browser
- Download file 8.93 kB
-
https://huggingface.co/myned-ai/facepoints/resolve/main/prep/labels.py
- Command line
-
hf download hf://myned-ai/facepoints/prep/labels.py
-
curl -L -o labels.py https://huggingface.co/myned-ai/facepoints/resolve/main/prep/labels.py
8.93 kB
| """Labels for one Ava-256 frame: point set 3D positions, per-camera projections, visibility.""" | |
| # Mesh points (classes lmk, skin, lid) are vertices of Meta's registration mesh (points_ava.npz | |
| # `src`), taken to the capture world through the head pose; teeth points are Meta's 3D keypoints | |
| # (incisal-edge ids with at least MIN_INL inliers, NaN when absent); iris rows stay NaN here. | |
| # Projection uses the capture's calibration with lens distortion, scaled to the photo tier. | |
| # Visibility comes from the registration mesh's own depth buffer, without the membrane triangles | |
| # that bridge the eyes and the mouth. Per-frame flags: the signed lid gap per eye and the | |
| # inner-lip gap, in mm. The Ava template OBJ is read from $AVA_TOPOLOGY, or downloaded once from | |
| # the ava-256 repository into ~/.cache/facepoints/. | |
| import io | |
| import os | |
| import urllib.request | |
| import cv2 | |
| import numpy as np | |
| MIN_INL = 4 # keypoint inliers for a teeth point to count | |
| Z_TOL = 3.0 # mm a point may sit behind the depth buffer and still be visible | |
| DEFAULT_POINTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "points_ava.npz") | |
| TOPOLOGY_URL = ( | |
| "https://raw.githubusercontent.com/facebookresearch/ava-256/main/assets/face_topology.obj" | |
| ) | |
| TOPOLOGY_CACHE = os.path.join( | |
| os.path.expanduser("~"), ".cache", "facepoints", "face_topology.obj" | |
| ) | |
| def topology_path(path=None): | |
| """Return the Ava template OBJ path: the argument, $AVA_TOPOLOGY, or a cached download.""" | |
| path = path or os.environ.get("AVA_TOPOLOGY") or TOPOLOGY_CACHE | |
| if not os.path.exists(path): | |
| os.makedirs(os.path.dirname(path), exist_ok=True) | |
| urllib.request.urlretrieve(TOPOLOGY_URL, path) | |
| return path | |
| def read_ply(buf): | |
| """Return the (N, 3) float64 vertices of a binary PLY from its first three float properties.""" | |
| head, rest = buf.split(b"end_header\n", 1) | |
| lines = head.decode("ascii", "ignore").splitlines() | |
| n = next(int(ln.split()[2]) for ln in lines if ln.startswith("element vertex")) | |
| sz = {"float": 4, "float32": 4, "double": 8, "uchar": 1, "int": 4} | |
| stride = sum(sz[ln.split()[1]] for ln in lines if ln.startswith("property")) | |
| arr = np.frombuffer(rest[: n * stride], dtype=np.uint8).reshape(n, stride) | |
| return arr[:, :12].copy().view(np.float32).reshape(n, 3).astype(np.float64) | |
| def topology(path): | |
| """Return (vertices, faces, per-vertex UV, longest UV edge per face) of the Ava template OBJ.""" | |
| V, VT, F, FT = [], [], [], [] | |
| with open(path) as fh: | |
| for ln in fh: | |
| if ln.startswith("v "): | |
| V.append([float(x) for x in ln.split()[1:4]]) | |
| elif ln.startswith("vt "): | |
| VT.append([float(x) for x in ln.split()[1:3]]) | |
| elif ln.startswith("f "): | |
| F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]]) | |
| FT.append([int(t.split("/")[1]) - 1 for t in ln.split()[1:4]]) | |
| V, VT, F, FT = np.array(V), np.array(VT), np.array(F), np.array(FT) | |
| el = np.linalg.norm(VT[FT] - VT[np.roll(FT, 1, axis=1)], axis=-1).max(1) | |
| uv = np.full((len(V), 2), np.nan) | |
| for f_, ft in zip(F, FT, strict=True): | |
| uv[f_] = VT[ft] | |
| return V, F, uv, el | |
| def membranes(V, F, el, centres, radius): | |
| """Return a face mask of the triangles bridging an opening (near a centre, long UV edge).""" | |
| # the UV-edge test is relative to the local median because a global rule also catches UV seams | |
| out = np.zeros(len(F), bool) | |
| for c in centres: | |
| near = np.all(np.linalg.norm(V[F] - c, axis=-1) < radius, axis=1) | |
| out |= near & (el > 3 * np.median(el[near])) | |
| return out | |
| def lip_contours(V, F, mouth_bridge, uv, eye_c): | |
| """Return (upper ids, lower ids) of the inner-lip ring along the mouth (upper = nearer eyes).""" | |
| low = np.unique(F[mouth_bridge]) | |
| cen = uv[low].mean(0) | |
| ax = uv[low][np.argmax(uv[low][:, 0])] - uv[low][np.argmin(uv[low][:, 0])] | |
| ax /= np.linalg.norm(ax) | |
| nrm = np.array([-ax[1], ax[0]]) | |
| s_, al = (uv[low] - cen) @ nrm, (uv[low] - cen) @ ax | |
| a, b = low[s_ > 0][np.argsort(al[s_ > 0])], low[s_ <= 0][np.argsort(al[s_ <= 0])] | |
| a_first = np.linalg.norm(V[a].mean(0) - eye_c) < np.linalg.norm(V[b].mean(0) - eye_c) | |
| return (a, b) if a_first else (b, a) | |
| class PointSet: | |
| """The label point set on the Ava topology and its evaluation on a registration + keypoints.""" | |
| def __init__(self, points=DEFAULT_POINTS, topology_obj=None): | |
| z = np.load(points) | |
| self.cls, self.src = z["cls"].astype(str), z["src"].astype(int) | |
| self.region = z["region"].astype(str) | |
| self.n = len(self.cls) | |
| self.mesh = np.isin(self.cls, ["lmk", "skin", "lid"]) | |
| self.teeth = np.isin(self.cls, ["teeth_u", "teeth_l"]) | |
| self.V0, self.F, uv0, el = topology(topology_path(topology_obj)) | |
| lid = self.cls == "lid" | |
| eyes = [self.V0[self.src[lid & (self.region == f"{e}_upper")]].mean(0) for e in "LR"] | |
| mouth = self.V0[self.src[(self.cls == "skin") & (self.region == "perioral")]].mean(0) | |
| eye_b = membranes(self.V0, self.F, el, eyes, 22.0) | |
| mouth_b = membranes(self.V0, self.F, el, [mouth], 45.0) | |
| self.Fz = self.F[~(eye_b | mouth_b)] # the triangles the depth buffer uses | |
| self.lips = lip_contours(self.V0, self.F, mouth_b, uv0, np.mean(eyes, axis=0)) | |
| self.lid_sets = { | |
| e: ( | |
| self.src[lid & (self.region == f"{e}_upper")], | |
| self.src[lid & (self.region == f"{e}_lower")], | |
| ) | |
| for e in "LR" | |
| } | |
| def points3d(self, X, kp): | |
| """Return (n, 3) positions in capture world: mesh points from X, teeth from kp, else NaN.""" | |
| P = np.full((self.n, 3), np.nan) | |
| P[self.mesh] = X[self.src[self.mesh]] | |
| ok = kp[:, 5] >= MIN_INL | |
| ids = kp[:, 0].astype(int) | |
| for i in np.flatnonzero(self.teeth): | |
| m = ok & (ids == self.src[i]) | |
| if m.any(): | |
| P[i] = kp[m][0, 1:4] | |
| return P | |
| def flags(self, X, up): | |
| """Return {lid_L, lid_R: signed lid gap along `up` (mm), lip_gap: inner-lip gap (mm)}.""" | |
| out = {} | |
| for e, (u, lo) in self.lid_sets.items(): | |
| g = [] | |
| for i in u[1:-1]: | |
| d = X[lo] - X[i] | |
| j = np.argmin(np.linalg.norm(d, axis=1)) | |
| g.append(-(d[j] @ up[e])) | |
| out[f"lid_{e}"] = float(np.mean(g)) | |
| u, lo = self.lips | |
| out["lip_gap"] = float( | |
| np.mean([np.linalg.norm(X[lo] - X[i], axis=1).min() for i in u[1:-1]]) | |
| ) | |
| return out | |
| def camera(e, scale): | |
| """Return (K, RT, distortion, rvec) of an Ava KRT entry for a photo at `scale` x native size.""" | |
| K = np.array(e["K"], np.float64).T.copy() | |
| K[:2] *= scale | |
| RT = np.array(e["T"], np.float64)[:4, :3].T | |
| dist = np.array(list(e["distortion"]) + [0.0] * 5, np.float64)[:5] | |
| return K, RT, dist, cv2.Rodrigues(RT[:3, :3])[0] | |
| def project(P, cam): | |
| """Return ((n, 2) pixels, (n,) depth) of points P; NaN rows pass through.""" | |
| K, RT, dist, rvec = cam | |
| uv = np.full((len(P), 2), np.nan) | |
| z = np.full(len(P), np.nan) | |
| ok = np.isfinite(P).all(1) | |
| if ok.any(): | |
| u, _ = cv2.projectPoints(P[ok], rvec, RT[:3, 3], K, dist) | |
| uv[ok] = u[:, 0] | |
| z[ok] = (P[ok] @ RT[:3, :3].T + RT[:3, 3])[:, 2] | |
| return uv, z | |
| def zbuffer(X, Fz, cam, hw): | |
| """Return the painter's depth buffer (H, W) of the mesh X over the triangles Fz.""" | |
| uv, z = project(X, cam) | |
| H, W = hw | |
| zb = np.full((H, W), np.inf, np.float32) | |
| tz = z[Fz].mean(1) | |
| for t in np.argsort(-tz): | |
| pts = np.round(uv[Fz[t]]).astype(np.int32) | |
| if (pts < 0).any() or (pts[:, 0] >= W).any() or (pts[:, 1] >= H).any(): | |
| continue | |
| cv2.fillConvexPoly(zb, pts, float(tz[t])) | |
| return zb | |
| def visible(uv, z, zb): | |
| """Return a mask of the points not hidden by the depth buffer by more than Z_TOL mm.""" | |
| H, W = zb.shape | |
| ok = np.isfinite(uv).all(1) | |
| v = np.zeros(len(uv), bool) | |
| x = np.clip(np.round(uv[ok, 0]).astype(int), 0, W - 1) | |
| y = np.clip(np.round(uv[ok, 1]).astype(int), 0, H - 1) | |
| v[ok] = zb[y, x] >= z[ok] - Z_TOL | |
| return v | |
| def frame_labels(ps, X, kp, cal, cam_ids, hw, scale): | |
| """Return ({cam: (uv (n, 2), visible (n,), depth (n,))}, P (n, 3)) for one frame.""" | |
| P = ps.points3d(X, kp) | |
| out = {} | |
| for c in cam_ids: | |
| cam = camera(cal[c], scale) | |
| uv, z = project(P, cam) | |
| zb = zbuffer(X, ps.Fz, cam, hw) | |
| out[c] = (uv, visible(uv, z, zb), z) | |
| return out, P | |
| def world(regz, posz, kpz, f): | |
| """Return (registration in capture world, head pose, keypoints) of frame f from the zips.""" | |
| X = read_ply(regz.read(f"{f:06d}.ply")) | |
| M = np.loadtxt(io.BytesIO(posz.read(f"{f:06d}.txt"))) | |
| kp = np.load(io.BytesIO(kpz.read(f"{f:06d}.npy"))) | |
| return X @ M[:3, :3].T + M[:3, 3], M, kp | |