facepoints / prep /labels.py
antonios-makro's picture
Facepoints 1.0.0: code, model, prep data and card
88607d8 verified
Raw History Blame Contribute Delete
8.93 kB
"""Labels for one Ava-256 frame: point set 3D positions, per-camera projections, visibility."""
# Mesh points (classes lmk, skin, lid) are vertices of Meta's registration mesh (points_ava.npz
# `src`), taken to the capture world through the head pose; teeth points are Meta's 3D keypoints
# (incisal-edge ids with at least MIN_INL inliers, NaN when absent); iris rows stay NaN here.
# Projection uses the capture's calibration with lens distortion, scaled to the photo tier.
# Visibility comes from the registration mesh's own depth buffer, without the membrane triangles
# that bridge the eyes and the mouth. Per-frame flags: the signed lid gap per eye and the
# inner-lip gap, in mm. The Ava template OBJ is read from $AVA_TOPOLOGY, or downloaded once from
# the ava-256 repository into ~/.cache/facepoints/.
import io
import os
import urllib.request
import cv2
import numpy as np
MIN_INL = 4 # keypoint inliers for a teeth point to count
Z_TOL = 3.0 # mm a point may sit behind the depth buffer and still be visible
DEFAULT_POINTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "points_ava.npz")
TOPOLOGY_URL = (
"https://raw.githubusercontent.com/facebookresearch/ava-256/main/assets/face_topology.obj"
)
TOPOLOGY_CACHE = os.path.join(
os.path.expanduser("~"), ".cache", "facepoints", "face_topology.obj"
)
def topology_path(path=None):
"""Return the Ava template OBJ path: the argument, $AVA_TOPOLOGY, or a cached download."""
path = path or os.environ.get("AVA_TOPOLOGY") or TOPOLOGY_CACHE
if not os.path.exists(path):
os.makedirs(os.path.dirname(path), exist_ok=True)
urllib.request.urlretrieve(TOPOLOGY_URL, path)
return path
def read_ply(buf):
"""Return the (N, 3) float64 vertices of a binary PLY from its first three float properties."""
head, rest = buf.split(b"end_header\n", 1)
lines = head.decode("ascii", "ignore").splitlines()
n = next(int(ln.split()[2]) for ln in lines if ln.startswith("element vertex"))
sz = {"float": 4, "float32": 4, "double": 8, "uchar": 1, "int": 4}
stride = sum(sz[ln.split()[1]] for ln in lines if ln.startswith("property"))
arr = np.frombuffer(rest[: n * stride], dtype=np.uint8).reshape(n, stride)
return arr[:, :12].copy().view(np.float32).reshape(n, 3).astype(np.float64)
def topology(path):
"""Return (vertices, faces, per-vertex UV, longest UV edge per face) of the Ava template OBJ."""
V, VT, F, FT = [], [], [], []
with open(path) as fh:
for ln in fh:
if ln.startswith("v "):
V.append([float(x) for x in ln.split()[1:4]])
elif ln.startswith("vt "):
VT.append([float(x) for x in ln.split()[1:3]])
elif ln.startswith("f "):
F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]])
FT.append([int(t.split("/")[1]) - 1 for t in ln.split()[1:4]])
V, VT, F, FT = np.array(V), np.array(VT), np.array(F), np.array(FT)
el = np.linalg.norm(VT[FT] - VT[np.roll(FT, 1, axis=1)], axis=-1).max(1)
uv = np.full((len(V), 2), np.nan)
for f_, ft in zip(F, FT, strict=True):
uv[f_] = VT[ft]
return V, F, uv, el
def membranes(V, F, el, centres, radius):
"""Return a face mask of the triangles bridging an opening (near a centre, long UV edge)."""
# the UV-edge test is relative to the local median because a global rule also catches UV seams
out = np.zeros(len(F), bool)
for c in centres:
near = np.all(np.linalg.norm(V[F] - c, axis=-1) < radius, axis=1)
out |= near & (el > 3 * np.median(el[near]))
return out
def lip_contours(V, F, mouth_bridge, uv, eye_c):
"""Return (upper ids, lower ids) of the inner-lip ring along the mouth (upper = nearer eyes)."""
low = np.unique(F[mouth_bridge])
cen = uv[low].mean(0)
ax = uv[low][np.argmax(uv[low][:, 0])] - uv[low][np.argmin(uv[low][:, 0])]
ax /= np.linalg.norm(ax)
nrm = np.array([-ax[1], ax[0]])
s_, al = (uv[low] - cen) @ nrm, (uv[low] - cen) @ ax
a, b = low[s_ > 0][np.argsort(al[s_ > 0])], low[s_ <= 0][np.argsort(al[s_ <= 0])]
a_first = np.linalg.norm(V[a].mean(0) - eye_c) < np.linalg.norm(V[b].mean(0) - eye_c)
return (a, b) if a_first else (b, a)
class PointSet:
"""The label point set on the Ava topology and its evaluation on a registration + keypoints."""
def __init__(self, points=DEFAULT_POINTS, topology_obj=None):
z = np.load(points)
self.cls, self.src = z["cls"].astype(str), z["src"].astype(int)
self.region = z["region"].astype(str)
self.n = len(self.cls)
self.mesh = np.isin(self.cls, ["lmk", "skin", "lid"])
self.teeth = np.isin(self.cls, ["teeth_u", "teeth_l"])
self.V0, self.F, uv0, el = topology(topology_path(topology_obj))
lid = self.cls == "lid"
eyes = [self.V0[self.src[lid & (self.region == f"{e}_upper")]].mean(0) for e in "LR"]
mouth = self.V0[self.src[(self.cls == "skin") & (self.region == "perioral")]].mean(0)
eye_b = membranes(self.V0, self.F, el, eyes, 22.0)
mouth_b = membranes(self.V0, self.F, el, [mouth], 45.0)
self.Fz = self.F[~(eye_b | mouth_b)] # the triangles the depth buffer uses
self.lips = lip_contours(self.V0, self.F, mouth_b, uv0, np.mean(eyes, axis=0))
self.lid_sets = {
e: (
self.src[lid & (self.region == f"{e}_upper")],
self.src[lid & (self.region == f"{e}_lower")],
)
for e in "LR"
}
def points3d(self, X, kp):
"""Return (n, 3) positions in capture world: mesh points from X, teeth from kp, else NaN."""
P = np.full((self.n, 3), np.nan)
P[self.mesh] = X[self.src[self.mesh]]
ok = kp[:, 5] >= MIN_INL
ids = kp[:, 0].astype(int)
for i in np.flatnonzero(self.teeth):
m = ok & (ids == self.src[i])
if m.any():
P[i] = kp[m][0, 1:4]
return P
def flags(self, X, up):
"""Return {lid_L, lid_R: signed lid gap along `up` (mm), lip_gap: inner-lip gap (mm)}."""
out = {}
for e, (u, lo) in self.lid_sets.items():
g = []
for i in u[1:-1]:
d = X[lo] - X[i]
j = np.argmin(np.linalg.norm(d, axis=1))
g.append(-(d[j] @ up[e]))
out[f"lid_{e}"] = float(np.mean(g))
u, lo = self.lips
out["lip_gap"] = float(
np.mean([np.linalg.norm(X[lo] - X[i], axis=1).min() for i in u[1:-1]])
)
return out
def camera(e, scale):
"""Return (K, RT, distortion, rvec) of an Ava KRT entry for a photo at `scale` x native size."""
K = np.array(e["K"], np.float64).T.copy()
K[:2] *= scale
RT = np.array(e["T"], np.float64)[:4, :3].T
dist = np.array(list(e["distortion"]) + [0.0] * 5, np.float64)[:5]
return K, RT, dist, cv2.Rodrigues(RT[:3, :3])[0]
def project(P, cam):
"""Return ((n, 2) pixels, (n,) depth) of points P; NaN rows pass through."""
K, RT, dist, rvec = cam
uv = np.full((len(P), 2), np.nan)
z = np.full(len(P), np.nan)
ok = np.isfinite(P).all(1)
if ok.any():
u, _ = cv2.projectPoints(P[ok], rvec, RT[:3, 3], K, dist)
uv[ok] = u[:, 0]
z[ok] = (P[ok] @ RT[:3, :3].T + RT[:3, 3])[:, 2]
return uv, z
def zbuffer(X, Fz, cam, hw):
"""Return the painter's depth buffer (H, W) of the mesh X over the triangles Fz."""
uv, z = project(X, cam)
H, W = hw
zb = np.full((H, W), np.inf, np.float32)
tz = z[Fz].mean(1)
for t in np.argsort(-tz):
pts = np.round(uv[Fz[t]]).astype(np.int32)
if (pts < 0).any() or (pts[:, 0] >= W).any() or (pts[:, 1] >= H).any():
continue
cv2.fillConvexPoly(zb, pts, float(tz[t]))
return zb
def visible(uv, z, zb):
"""Return a mask of the points not hidden by the depth buffer by more than Z_TOL mm."""
H, W = zb.shape
ok = np.isfinite(uv).all(1)
v = np.zeros(len(uv), bool)
x = np.clip(np.round(uv[ok, 0]).astype(int), 0, W - 1)
y = np.clip(np.round(uv[ok, 1]).astype(int), 0, H - 1)
v[ok] = zb[y, x] >= z[ok] - Z_TOL
return v
def frame_labels(ps, X, kp, cal, cam_ids, hw, scale):
"""Return ({cam: (uv (n, 2), visible (n,), depth (n,))}, P (n, 3)) for one frame."""
P = ps.points3d(X, kp)
out = {}
for c in cam_ids:
cam = camera(cal[c], scale)
uv, z = project(P, cam)
zb = zbuffer(X, ps.Fz, cam, hw)
out[c] = (uv, visible(uv, z, zb), z)
return out, P
def world(regz, posz, kpz, f):
"""Return (registration in capture world, head pose, keypoints) of frame f from the zips."""
X = read_ply(regz.read(f"{f:06d}.ply"))
M = np.loadtxt(io.BytesIO(posz.read(f"{f:06d}.txt")))
kp = np.load(io.BytesIO(kpz.read(f"{f:06d}.npy")))
return X @ M[:3, :3].T + M[:3, 3], M, kp