Facepoints 1.0.0: code, model, prep data and card
Browse files- .gitattributes +4 -0
- LICENSE +202 -0
- LICENSE-WEIGHTS +10 -0
- README.md +134 -0
- __init__.py +4 -0
- detect.py +80 -0
- images/bench_2d.png +3 -0
- images/examples_scan.jpg +3 -0
- images/examples_wflw.jpg +3 -0
- images/intro.jpg +3 -0
- images/scan_mm.png +0 -0
- model.py +186 -0
- model/config.json +698 -0
- model/model.safetensors +3 -0
- points.py +24 -0
- points_gnm.json +0 -0
- prep/__init__.py +1 -0
- prep/captures.txt +256 -0
- prep/closed_frames.py +162 -0
- prep/detect_wild.py +42 -0
- prep/extract.py +163 -0
- prep/extract_masks.py +100 -0
- prep/http_zip.py +59 -0
- prep/iris_labels.py +123 -0
- prep/labels.py +211 -0
- prep/merge_data.py +65 -0
- prep/points_ava.npz +3 -0
- prep/sample_frames.py +123 -0
- prep/samples.json +0 -0
- prep/samples_closed.json +0 -0
- prep/split.json +273 -0
- prep/wild620_map.npz +3 -0
- prep/wild620_map.py +163 -0
- prep/wild_prep.py +154 -0
- pyproject.toml +45 -0
- run.py +58 -0
- tests/test_detect.py +54 -0
- train/README.md +85 -0
- train/__init__.py +1 -0
- train/__main__.py +5 -0
- train/config.py +110 -0
- train/data/__init__.py +6 -0
- train/data/augment.py +216 -0
- train/data/ava.py +115 -0
- train/data/wild.py +164 -0
- train/evaluate.py +156 -0
- train/export.py +73 -0
- train/losses.py +164 -0
- train/trainer.py +454 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
images/bench_2d.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
images/examples_scan.jpg filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
images/examples_wflw.jpg filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
images/intro.jpg filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
LICENSE-WEIGHTS
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Copyright 2026 Myned AI (opensource@myned.ai)
|
| 2 |
+
|
| 3 |
+
The model weights (model.safetensors with config.json) and the point definitions (points_gnm.json,
|
| 4 |
+
prep/points_ava.npz, prep/wild620_map.npz) are released by Myned AI under the
|
| 5 |
+
Creative Commons Attribution-NonCommercial 4.0 International licence (CC BY-NC 4.0):
|
| 6 |
+
https://creativecommons.org/licenses/by-nc/4.0/legalcode
|
| 7 |
+
|
| 8 |
+
They were trained on Ava-256 (Meta, CC BY-NC 4.0) and on the 300W and WFLW training photos, whose terms
|
| 9 |
+
do not allow commercial use; those terms bind the weights. The source code in this repository is under
|
| 10 |
+
the Apache License 2.0 (see LICENSE).
|
README.md
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-4.0
|
| 3 |
+
library_name: pytorch
|
| 4 |
+
pipeline_tag: keypoint-detection
|
| 5 |
+
tags:
|
| 6 |
+
- face-landmarks
|
| 7 |
+
- gnm
|
| 8 |
+
- 3d-face
|
| 9 |
+
- dense-landmarks
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Facepoints: 620 face points on the GNM Head mesh
|
| 13 |
+
|
| 14 |
+

|
| 15 |
+
|
| 16 |
+
Finds **620 points on a face in a photo**. Every point is a fixed location on Google's open
|
| 17 |
+
[GNM Head](https://github.com/google/GNM) mesh, so the output fits that model to a person directly —
|
| 18 |
+
for building or animating a 3D avatar. The same forward pass also gives the standard 68-point (iBUG)
|
| 19 |
+
and 98-point (WFLW) landmark sets and an error estimate for every point. To our knowledge (2026-10-23)
|
| 20 |
+
it is the first publicly released detector trained on points defined on the GNM mesh.
|
| 21 |
+
|
| 22 |
+
| | |
|
| 23 |
+
|---|---|
|
| 24 |
+
| Input | one face, cropped from a photo or video frame; a face detector supplies the box |
|
| 25 |
+
| Output | 620 points, each (x, y) plus its own error estimate, in pixels; the 68- and 98-point sets |
|
| 26 |
+
| Points | 416 landmark and 150 skin points, 36 eyelid-edge, 16 front teeth (7 upper, 9 lower), 2 iris centres (`points_gnm.json` maps each to the GNM Head mesh, v3.0) |
|
| 27 |
+
| Model | HRNet-W48, 256-px input, 102.7 M parameters (+1.2 M for the 68/98 heads) |
|
| 28 |
+
| Weights | `model/model.safetensors` + `model/config.json`; SHA-256 of the weights `b37075caa3eaf0d00f23f32925b16ec3a8e2e45c42aa9c00956f94f2574e6526` |
|
| 29 |
+
|
| 30 |
+
For face tracking and avatar research. Not for identifying or judging people, and not for commercial
|
| 31 |
+
use (the weights are CC BY-NC 4.0; see *Licence*).
|
| 32 |
+
|
| 33 |
+
## Use
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
git clone https://huggingface.co/myned-ai/facepoints && cd facepoints # git-lfs for the weights
|
| 37 |
+
pip install -e . # torch, timm, safetensors, opencv-python-headless, numpy
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
```python
|
| 41 |
+
import cv2
|
| 42 |
+
from facepoints import Detector, square_box, gnm_positions
|
| 43 |
+
|
| 44 |
+
det = Detector("./model") # the folder with model.safetensors + config.json
|
| 45 |
+
img = cv2.imread("face.jpg") # BGR
|
| 46 |
+
x, y, w, h = my_face_detector(img) # any face box, e.g. OpenCV's YuNet
|
| 47 |
+
out = det.detect(img, square_box(x, y, w, h)) # the box, then two passes on the model's own extent
|
| 48 |
+
|
| 49 |
+
out["points"], out["std"] # (620, 2) image px, and one predicted std per point (px)
|
| 50 |
+
out["300w"], out["wflw"] # (68, 2) and (98, 2)
|
| 51 |
+
gnm_positions(mesh_xyz) # (620, 3): the same points on any posed GNM Head mesh (weights x vertices)
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
Fitting the GNM Head to a face = solving for the mesh whose projected `gnm_positions` match
|
| 55 |
+
`out["points"]`, weighted by `1 / std`. One face at a time; the output moves with the box, which is
|
| 56 |
+
why `detect` re-frames on its own points. A box with no face in it yields points with a very large
|
| 57 |
+
`std` — threshold it. A complete example with OpenCV's face finder:
|
| 58 |
+
`python run.py photo.jpg out.jpg` (the model from `./model`, or `--model DIR`); tests: `pytest tests`.
|
| 59 |
+
|
| 60 |
+
## Results
|
| 61 |
+
|
| 62 |
+
**Against registered 3D meshes** of 9 studio people none of the models trained on, 14 cameras, their
|
| 63 |
+
widest mouth shapes. The score is how well the points follow the face: median distance to the paired
|
| 64 |
+
truth vertex, in mm, after removing each point's constant offset at rest (protocol in the report).
|
| 65 |
+
|
| 66 |
+

|
| 67 |
+
|
| 68 |
+
| | lips | cheeks | chin | rest of face |
|
| 69 |
+
|---|---|---|---|---|
|
| 70 |
+
| **ours** | **1.78** | **2.00** | 2.12 | 0.81 |
|
| 71 |
+
| Pixel3DMM | 1.81 | 2.19 | **1.67** | **0.78** |
|
| 72 |
+
| 3DDFA-V3 | 2.40 | 2.61 | 2.84 | 1.41 |
|
| 73 |
+
|
| 74 |
+
Level with Pixel3DMM except on the chin, ahead of 3DDFA-V3 on every region.
|
| 75 |
+
|
| 76 |
+

|
| 77 |
+
|
| 78 |
+
**On the 2D benchmarks** (the 68/98-point outputs; full test sets; NME = mean point error as % of the
|
| 79 |
+
eye-corner distance):
|
| 80 |
+
|
| 81 |
+

|
| 82 |
+
|
| 83 |
+
| | 300W | 300VW | WFLW |
|
| 84 |
+
|---|---|---|---|
|
| 85 |
+
| **ours, 68/98-point heads** | 3.33 | 3.52 | 4.66 |
|
| 86 |
+
| **ours, 620 points through a linear map** | 3.31 | 3.56 | 4.59 |
|
| 87 |
+
| STAR (official checkpoints) | 3.06 | 3.45 | 4.47 |
|
| 88 |
+
| SPIGA | 3.02 | 3.37 | 4.06 |
|
| 89 |
+
| RwSA (ICCV 2025) | 2.96 | 3.38 | 4.23 |
|
| 90 |
+
| FAN | 3.69 | 4.11 | 7.27 |
|
| 91 |
+
| MediaPipe (478 points, mapped) | 3.87 | 4.00 | 7.72 |
|
| 92 |
+
|
| 93 |
+
Behind the three models built for these benchmarks by 0.2–0.6, level with STAR on video, ahead of FAN
|
| 94 |
+
and MediaPipe everywhere. Profile faces are the weak spot.
|
| 95 |
+
|
| 96 |
+

|
| 97 |
+
|
| 98 |
+
**The error estimate** (`std`) comes from a Student-t likelihood (ν = 10). Error rises with it, but
|
| 99 |
+
the intervals are too narrow: multiply `std` by 1.34 to calibrate on average. Measured on studio
|
| 100 |
+
faces only.
|
| 101 |
+
|
| 102 |
+
## Limitations
|
| 103 |
+
|
| 104 |
+
- Hidden points (teeth behind closed lips, the far side of a turned face) are estimated, not seen;
|
| 105 |
+
sunglasses give guessed eyelids and irises; closed eyes come out about 1 mm open.
|
| 106 |
+
- The 620 points have no hand-labelled truth on wild photos; their wild accuracy is known only through
|
| 107 |
+
the 68/98 outputs.
|
| 108 |
+
- On 24 held-out studio people, error correlated with darker skin (Spearman −0.44, p = 0.03),
|
| 109 |
+
confounded with exposure; Ava-256 is biased to lighter-skinned people under 35.
|
| 110 |
+
|
| 111 |
+
## Training
|
| 112 |
+
|
| 113 |
+
The 620 points: Ava-256 (Meta), 230 people, 16 cameras; labels are projections of each person's
|
| 114 |
+
registered mesh, whose vertices were mapped onto the GNM template. The 68/98 heads: the 300W and WFLW
|
| 115 |
+
training photos; on those photos the 620 points carry pseudo-labels. HRNet-W48, three stages, one
|
| 116 |
+
run. To replicate the model, follow [train/README.md](train/README.md): data preparation, the three
|
| 117 |
+
commands and the export. The scripts and the shipped sample lists, split and pseudo-label map:
|
| 118 |
+
[prep/](prep/); every key: [train/config.py](train/config.py).
|
| 119 |
+
Protocol, data and analysis in full: technical report (in preparation).
|
| 120 |
+
|
| 121 |
+
## Licence
|
| 122 |
+
|
| 123 |
+
Copyright 2026 Myned AI (opensource@myned.ai).
|
| 124 |
+
|
| 125 |
+
| | licence | source |
|
| 126 |
+
|---|---|---|
|
| 127 |
+
| code | Apache-2.0 | `LICENSE` |
|
| 128 |
+
| `model/model.safetensors`, `points_gnm.json`, `prep/*.npz` | CC BY-NC 4.0 | `LICENSE-WEIGHTS` |
|
| 129 |
+
| Ava-256 · Multiface (images) | CC BY-NC 4.0 | LICENSE files |
|
| 130 |
+
| 300W · WFLW (WIDER FACE images) · 300VW | research only · CC BY-NC-ND · none stated | dataset pages |
|
| 131 |
+
| ImageNet start (timm `hrnet_w48.ms_in1k`) | weights MIT; ImageNet terms non-commercial | model card |
|
| 132 |
+
| GNM Head | Apache-2.0, attribution required | LICENSE file |
|
| 133 |
+
|
| 134 |
+
Figures show Multiface people (consented) and WFLW test faces as the benchmark publishes them.
|
__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Facepoints: 620 points on the GNM Head mesh from one face in a photo."""
|
| 2 |
+
|
| 3 |
+
from facepoints.detect import Detector, box_of_points, square_box # noqa: F401
|
| 4 |
+
from facepoints.points import gnm_positions, load_points # noqa: F401
|
detect.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference: the released model on one face -> 620 GNM points with std, and the 68 / 98 sets."""
|
| 2 |
+
|
| 3 |
+
# The deployment contract: a square box (x0, y0, side) in image px -> warpAffine to res x res
|
| 4 |
+
# (INTER_AREA, black border) -> ImageNet-normalised RGB -> Net -> (x, y, log scale) per point in
|
| 5 |
+
# crop px -> image px (x side / res). std = exp(log scale) x the likelihood's std factor
|
| 6 |
+
# (config.json): one standard deviation per point, in image px, valid for either coordinate.
|
| 7 |
+
# detect() runs `passes` forwards: the given box first, then the box of the predicted mesh points
|
| 8 |
+
# x SELF_BOX, because the face finder's box is only a start and the output moves with the box.
|
| 9 |
+
import cv2
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
from facepoints.model import load_release, to_tensor
|
| 14 |
+
|
| 15 |
+
SELF_BOX = 1.25 # side of the self-box = the mesh points' extent x this
|
| 16 |
+
MESH_CLASSES = ("lmk", "skin", "lid") # the points on the skin surface frame the self-box
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def square_box(x, y, w, h, factor=1.1):
|
| 20 |
+
"""Return the square (x0, y0, side) around a face box (x, y, w, h), side = factor x max."""
|
| 21 |
+
side = factor * max(w, h)
|
| 22 |
+
return (x + w / 2 - side / 2, y + h / 2 - side / 2, side)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def box_of_points(uv, factor=SELF_BOX):
|
| 26 |
+
"""Return the square (x0, y0, side) around points uv (n, 2): side = their extent x factor."""
|
| 27 |
+
lo, hi = uv.min(0), uv.max(0)
|
| 28 |
+
side = float((hi - lo).max() * factor)
|
| 29 |
+
c = (lo + hi) / 2
|
| 30 |
+
return (float(c[0] - side / 2), float(c[1] - side / 2), side)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class Detector:
|
| 34 |
+
"""The released model, loaded once; detect(image, box) returns the points of one face."""
|
| 35 |
+
|
| 36 |
+
def __init__(self, model_dir, device=None):
|
| 37 |
+
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
| 38 |
+
self.net, self.side, self.side_rf, self.config = load_release(model_dir, self.device)
|
| 39 |
+
self.cls = np.array(self.config["classes"])
|
| 40 |
+
self.res = int(self.config["input_px"])
|
| 41 |
+
self.std_factor = float(self.config["std_factor"])
|
| 42 |
+
self.log_scale_range = tuple(self.config["log_scale_range"])
|
| 43 |
+
self.mesh = np.isin(self.cls, MESH_CLASSES)
|
| 44 |
+
|
| 45 |
+
def crop(self, image_bgr, box):
|
| 46 |
+
"""Return the model's input tensor (1, 3, res, res) for the box."""
|
| 47 |
+
x0, y0, side = box
|
| 48 |
+
s = self.res / side
|
| 49 |
+
M = np.array([[s, 0, -x0 * s], [0, s, -y0 * s]])
|
| 50 |
+
crop = cv2.warpAffine(
|
| 51 |
+
image_bgr, M, (self.res, self.res), flags=cv2.INTER_AREA, borderValue=(0, 0, 0)
|
| 52 |
+
)
|
| 53 |
+
return to_tensor(crop)[None].to(self.device)
|
| 54 |
+
|
| 55 |
+
def forward(self, image_bgr, box):
|
| 56 |
+
"""Return one pass: {"points": (620, 2), "std": (620,), "<set>": (K, 2) ...} in image px."""
|
| 57 |
+
x0, y0, side = box
|
| 58 |
+
s = side / self.res
|
| 59 |
+
kind = self.device.split(":")[0]
|
| 60 |
+
with torch.no_grad(), torch.autocast(kind, dtype=torch.float16, enabled=kind == "cuda"):
|
| 61 |
+
points, shared, maps = self.net.run(self.crop(image_bgr, box))
|
| 62 |
+
o = points.float()[0].cpu().numpy()
|
| 63 |
+
out = {
|
| 64 |
+
"points": o[:, :2] * s + [x0, y0],
|
| 65 |
+
"std": np.exp(np.clip(o[:, 2], *self.log_scale_range)) * s * self.std_factor,
|
| 66 |
+
}
|
| 67 |
+
for name, head in self.side.items():
|
| 68 |
+
so = head(shared).view(1, -1, 3)
|
| 69 |
+
if name in self.side_rf:
|
| 70 |
+
so = self.side_rf[name]([m.float() for m in maps], so.float())
|
| 71 |
+
out[name] = so.float()[0, :, :2].cpu().numpy() * s + [x0, y0]
|
| 72 |
+
return out
|
| 73 |
+
|
| 74 |
+
def detect(self, image_bgr, box, passes=3):
|
| 75 |
+
"""Return the last of `passes` forwards, each re-framed on the previous points, plus box."""
|
| 76 |
+
out = self.forward(image_bgr, box)
|
| 77 |
+
for _ in range(passes - 1):
|
| 78 |
+
out = self.forward(image_bgr, box_of_points(out["points"][self.mesh]))
|
| 79 |
+
out["box"] = box
|
| 80 |
+
return out
|
images/bench_2d.png
ADDED
|
Git LFS Details
|
images/examples_scan.jpg
ADDED
|
Git LFS Details
|
images/examples_wflw.jpg
ADDED
|
Git LFS Details
|
images/intro.jpg
ADDED
|
Git LFS Details
|
images/scan_mm.png
ADDED
|
model.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The network (timm backbone, 1x1 neck, flattened MLP head, side heads) and its two loaders."""
|
| 2 |
+
|
| 3 |
+
# Every head predicts (x, y, log scale) per point. The third channel is a log scale, clamped to
|
| 4 |
+
# LOG_SCALE_RANGE; std_factor() turns the scale into the per-coordinate standard deviation of the
|
| 5 |
+
# likelihood the model was trained with (losses.gnll).
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import timm
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
from safetensors.torch import load_file
|
| 15 |
+
from torch import nn
|
| 16 |
+
|
| 17 |
+
LOG_SCALE_RANGE = (-3.0, 6.0)
|
| 18 |
+
WEIGHTS, CONFIG = "model.safetensors", "config.json" # the two release files
|
| 19 |
+
FORMAT = "facepoints-1" # config.json "format"
|
| 20 |
+
MEAN = np.array([0.485, 0.456, 0.406], np.float32) # ImageNet input normalisation, RGB
|
| 21 |
+
STD = np.array([0.229, 0.224, 0.225], np.float32)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def to_tensor(im_bgr):
|
| 25 |
+
"""Return a uint8 BGR image as a float (3, H, W) tensor: RGB, ImageNet-normalised."""
|
| 26 |
+
x = (im_bgr[:, :, ::-1].astype(np.float32) / 255.0 - MEAN) / STD
|
| 27 |
+
return torch.from_numpy(np.ascontiguousarray(x.transpose(2, 0, 1)))
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def std_factor(likelihood, nu=10.0):
|
| 31 |
+
"""Return the factor from the predicted scale to the per-coordinate standard deviation."""
|
| 32 |
+
if likelihood == "laplace":
|
| 33 |
+
return math.sqrt(2.0)
|
| 34 |
+
if likelihood == "studentt":
|
| 35 |
+
return math.sqrt(nu / (nu - 2.0))
|
| 36 |
+
return 1.0
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class Refine(nn.Module):
|
| 40 |
+
"""Local refinement of a point set: features around each coarse point -> a correction."""
|
| 41 |
+
|
| 42 |
+
def __init__(self, n_points, channels, res, ch=32, emb=32, hidden=256):
|
| 43 |
+
super().__init__()
|
| 44 |
+
self.res = res
|
| 45 |
+
self.proj = nn.ModuleList([nn.Conv2d(c, ch, 1) for c in channels]) # one per feature map
|
| 46 |
+
|
| 47 |
+
def grid(n, step):
|
| 48 |
+
a = torch.arange(n, dtype=torch.float32) - (n - 1) / 2
|
| 49 |
+
return torch.stack(torch.meshgrid(a, a, indexing="xy"), -1).view(-1, 2) * step
|
| 50 |
+
|
| 51 |
+
self.register_buffer("off4", grid(5, 3.0)) # sample offsets in crop px, stride-4 map
|
| 52 |
+
self.register_buffer("off8", grid(3, 6.0)) # and stride-8 map
|
| 53 |
+
self.emb = nn.Embedding(n_points, emb)
|
| 54 |
+
n_in = ch * (len(self.off4) + len(self.off8)) + emb + 1
|
| 55 |
+
self.mlp = nn.Sequential(
|
| 56 |
+
nn.Linear(n_in, hidden),
|
| 57 |
+
nn.GELU(),
|
| 58 |
+
nn.Linear(hidden, hidden),
|
| 59 |
+
nn.GELU(),
|
| 60 |
+
nn.Linear(hidden, 3),
|
| 61 |
+
)
|
| 62 |
+
nn.init.zeros_(self.mlp[-1].weight) # refined == coarse at initialisation
|
| 63 |
+
nn.init.zeros_(self.mlp[-1].bias)
|
| 64 |
+
|
| 65 |
+
def sample(self, feature_map, points, offsets):
|
| 66 |
+
"""Return bilinear samples (B, P, Q * C) of one map at each point plus Q offsets."""
|
| 67 |
+
B, P = points.shape[:2]
|
| 68 |
+
g = (points[:, :, None] + offsets[None, None]) * (2.0 / self.res) - 1.0 # (B, P, Q, 2)
|
| 69 |
+
s = F.grid_sample(
|
| 70 |
+
feature_map, g, mode="bilinear", padding_mode="zeros", align_corners=False
|
| 71 |
+
)
|
| 72 |
+
return s.permute(0, 2, 3, 1).reshape(B, P, -1)
|
| 73 |
+
|
| 74 |
+
def forward(self, maps, coarse):
|
| 75 |
+
"""Return the refined points (B, P, 3); gradients do not reach the coarse points."""
|
| 76 |
+
c = coarse.detach().float()
|
| 77 |
+
f4, f8 = (p(f).float() for p, f in zip(self.proj, maps, strict=True))
|
| 78 |
+
x = torch.cat(
|
| 79 |
+
[
|
| 80 |
+
self.sample(f4, c[..., :2], self.off4),
|
| 81 |
+
self.sample(f8, c[..., :2], self.off8),
|
| 82 |
+
self.emb.weight[None].expand(len(c), -1, -1),
|
| 83 |
+
c[..., 2:3],
|
| 84 |
+
],
|
| 85 |
+
-1,
|
| 86 |
+
)
|
| 87 |
+
return c + self.mlp(x)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class Net(nn.Module):
|
| 91 |
+
"""Backbone -> 1x1 neck on the stride-`stride` grid -> flatten -> Linear 1024 -> Linear 3P."""
|
| 92 |
+
|
| 93 |
+
def __init__(
|
| 94 |
+
self, backbone, res, n_points, pretrained=False, neck=32, stride=8, side_maps=False
|
| 95 |
+
):
|
| 96 |
+
super().__init__()
|
| 97 |
+
self.n_points = n_points
|
| 98 |
+
probe = timm.create_model(backbone, pretrained=False, features_only=True)
|
| 99 |
+
red = probe.feature_info.reduction() # hrnet_w48: strides 2, 4, 8, 16, 32
|
| 100 |
+
del probe
|
| 101 |
+
# side_maps: also return the stride-4 map (the side heads' refinement reads it)
|
| 102 |
+
idx = (red.index(4), red.index(stride)) if side_maps else (red.index(stride),)
|
| 103 |
+
self.b = timm.create_model(
|
| 104 |
+
backbone, pretrained=pretrained, features_only=True, out_indices=idx
|
| 105 |
+
)
|
| 106 |
+
D = self.b.feature_info.channels()[-1]
|
| 107 |
+
with torch.no_grad():
|
| 108 |
+
g = self.b(torch.zeros(1, 3, res, res))[-1].shape[2:]
|
| 109 |
+
self.neck = nn.Sequential(nn.Conv2d(D, neck, 1), nn.GELU())
|
| 110 |
+
self.h = nn.Sequential(
|
| 111 |
+
nn.Flatten(),
|
| 112 |
+
nn.Linear(neck * g[0] * g[1], 1024),
|
| 113 |
+
nn.GELU(),
|
| 114 |
+
nn.Linear(1024, n_points * 3),
|
| 115 |
+
)
|
| 116 |
+
print(f"MODEL feature grid {tuple(g)} x {D} -> neck {neck}", flush=True)
|
| 117 |
+
|
| 118 |
+
@property
|
| 119 |
+
def shared_features(self):
|
| 120 |
+
"""The width of the shared layer the point head and the side heads read."""
|
| 121 |
+
return self.h[-1].in_features
|
| 122 |
+
|
| 123 |
+
def shared(self, x):
|
| 124 |
+
"""Return (shared layer (B, 1024), backbone feature maps) for a batch of crops."""
|
| 125 |
+
maps = self.b(x)
|
| 126 |
+
return self.h[:-1](self.neck(maps[-1])), maps
|
| 127 |
+
|
| 128 |
+
def run(self, x, n=None):
|
| 129 |
+
"""Return (points (n, P, 3), shared (B, 1024), maps); the point head reads the first n."""
|
| 130 |
+
z, maps = self.shared(x)
|
| 131 |
+
n = len(x) if n is None else n
|
| 132 |
+
return self.h[-1](z[:n]).view(-1, self.n_points, 3), z, maps
|
| 133 |
+
|
| 134 |
+
def forward(self, x):
|
| 135 |
+
"""Return the points (B, P, 3) in crop px with their log scale."""
|
| 136 |
+
return self.h[-1](self.shared(x)[0]).view(-1, self.n_points, 3)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def side_head(net, n_points):
|
| 140 |
+
"""Return a new side head: one Linear on the shared layer -> n_points x (x, y, log scale)."""
|
| 141 |
+
return nn.Linear(net.shared_features, n_points * 3)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def side_refine(net, n_points, res):
|
| 145 |
+
"""Return a new refinement stage for a side head of n_points points."""
|
| 146 |
+
return Refine(n_points, net.b.feature_info.channels(), res)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def load_net(checkpoint):
|
| 150 |
+
"""Return a training checkpoint's Net (the ckpt.pt dict) with its weights, in eval mode."""
|
| 151 |
+
a = checkpoint["args"]
|
| 152 |
+
net = Net(a["backbone"], a["res"], len(checkpoint["cls"]), False, a["neck"], a.get("stride", 8),
|
| 153 |
+
bool(a.get("side_rf", 0))) # fmt: skip
|
| 154 |
+
net.load_state_dict(checkpoint["model"])
|
| 155 |
+
return net.eval()
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def load_release(path, device="cpu"):
|
| 159 |
+
"""Return (net, side heads, refinement stages, config) from a release folder, in eval mode."""
|
| 160 |
+
# The release = model.safetensors + config.json, written by train/export.py. Tensors are stored
|
| 161 |
+
# flat under the prefixes net. / side.<set>. / side_rf.<set>.; config.json holds the
|
| 162 |
+
# architecture, the point classes in output order, the likelihood and its std factor, the side
|
| 163 |
+
# heads and the training keys as run.
|
| 164 |
+
d = path if os.path.isdir(path) else os.path.dirname(os.path.abspath(path))
|
| 165 |
+
with open(os.path.join(d, CONFIG)) as fh:
|
| 166 |
+
config = json.load(fh)
|
| 167 |
+
if config.get("format") != FORMAT:
|
| 168 |
+
raise ValueError(f"{d}: unknown config format {config.get('format')!r}")
|
| 169 |
+
t = load_file(os.path.join(d, WEIGHTS), device=str(device))
|
| 170 |
+
|
| 171 |
+
def sub(prefix):
|
| 172 |
+
return {k[len(prefix) :]: v for k, v in t.items() if k.startswith(prefix)}
|
| 173 |
+
|
| 174 |
+
net = Net(config["backbone"], config["input_px"], config["n_points"], False, config["neck"],
|
| 175 |
+
config["stride"], config["side_refine"]) # fmt: skip
|
| 176 |
+
net.load_state_dict(sub("net."))
|
| 177 |
+
side, side_rf = {}, {}
|
| 178 |
+
for name, K in config["side_heads"].items():
|
| 179 |
+
side[name] = side_head(net, K)
|
| 180 |
+
side[name].load_state_dict(sub(f"side.{name}."))
|
| 181 |
+
side[name].to(device).eval()
|
| 182 |
+
if config["side_refine"]:
|
| 183 |
+
side_rf[name] = side_refine(net, K, config["input_px"])
|
| 184 |
+
side_rf[name].load_state_dict(sub(f"side_rf.{name}."))
|
| 185 |
+
side_rf[name].to(device).eval()
|
| 186 |
+
return net.to(device).eval(), side, side_rf, config
|
model/config.json
ADDED
|
@@ -0,0 +1,698 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "facepoints-1",
|
| 3 |
+
"backbone": "hrnet_w48",
|
| 4 |
+
"input_px": 256,
|
| 5 |
+
"neck": 32,
|
| 6 |
+
"stride": 8,
|
| 7 |
+
"n_points": 620,
|
| 8 |
+
"classes": [
|
| 9 |
+
"lmk",
|
| 10 |
+
"lmk",
|
| 11 |
+
"lmk",
|
| 12 |
+
"lmk",
|
| 13 |
+
"lmk",
|
| 14 |
+
"lmk",
|
| 15 |
+
"lmk",
|
| 16 |
+
"lmk",
|
| 17 |
+
"lmk",
|
| 18 |
+
"lmk",
|
| 19 |
+
"lmk",
|
| 20 |
+
"lmk",
|
| 21 |
+
"lmk",
|
| 22 |
+
"lmk",
|
| 23 |
+
"lmk",
|
| 24 |
+
"lmk",
|
| 25 |
+
"lmk",
|
| 26 |
+
"lmk",
|
| 27 |
+
"lmk",
|
| 28 |
+
"lmk",
|
| 29 |
+
"lmk",
|
| 30 |
+
"lmk",
|
| 31 |
+
"lmk",
|
| 32 |
+
"lmk",
|
| 33 |
+
"lmk",
|
| 34 |
+
"lmk",
|
| 35 |
+
"lmk",
|
| 36 |
+
"lmk",
|
| 37 |
+
"lmk",
|
| 38 |
+
"lmk",
|
| 39 |
+
"lmk",
|
| 40 |
+
"lmk",
|
| 41 |
+
"lmk",
|
| 42 |
+
"lmk",
|
| 43 |
+
"lmk",
|
| 44 |
+
"lmk",
|
| 45 |
+
"lmk",
|
| 46 |
+
"lmk",
|
| 47 |
+
"lmk",
|
| 48 |
+
"lmk",
|
| 49 |
+
"lmk",
|
| 50 |
+
"lmk",
|
| 51 |
+
"lmk",
|
| 52 |
+
"lmk",
|
| 53 |
+
"lmk",
|
| 54 |
+
"lmk",
|
| 55 |
+
"lmk",
|
| 56 |
+
"lmk",
|
| 57 |
+
"lmk",
|
| 58 |
+
"lmk",
|
| 59 |
+
"lmk",
|
| 60 |
+
"lmk",
|
| 61 |
+
"lmk",
|
| 62 |
+
"lmk",
|
| 63 |
+
"lmk",
|
| 64 |
+
"lmk",
|
| 65 |
+
"lmk",
|
| 66 |
+
"lmk",
|
| 67 |
+
"lmk",
|
| 68 |
+
"lmk",
|
| 69 |
+
"lmk",
|
| 70 |
+
"lmk",
|
| 71 |
+
"lmk",
|
| 72 |
+
"lmk",
|
| 73 |
+
"lmk",
|
| 74 |
+
"lmk",
|
| 75 |
+
"lmk",
|
| 76 |
+
"lmk",
|
| 77 |
+
"lmk",
|
| 78 |
+
"lmk",
|
| 79 |
+
"lmk",
|
| 80 |
+
"lmk",
|
| 81 |
+
"lmk",
|
| 82 |
+
"lmk",
|
| 83 |
+
"lmk",
|
| 84 |
+
"lmk",
|
| 85 |
+
"lmk",
|
| 86 |
+
"lmk",
|
| 87 |
+
"lmk",
|
| 88 |
+
"lmk",
|
| 89 |
+
"lmk",
|
| 90 |
+
"lmk",
|
| 91 |
+
"lmk",
|
| 92 |
+
"lmk",
|
| 93 |
+
"lmk",
|
| 94 |
+
"lmk",
|
| 95 |
+
"lmk",
|
| 96 |
+
"lmk",
|
| 97 |
+
"lmk",
|
| 98 |
+
"lmk",
|
| 99 |
+
"lmk",
|
| 100 |
+
"lmk",
|
| 101 |
+
"lmk",
|
| 102 |
+
"lmk",
|
| 103 |
+
"lmk",
|
| 104 |
+
"lmk",
|
| 105 |
+
"lmk",
|
| 106 |
+
"lmk",
|
| 107 |
+
"lmk",
|
| 108 |
+
"lmk",
|
| 109 |
+
"lmk",
|
| 110 |
+
"lmk",
|
| 111 |
+
"lmk",
|
| 112 |
+
"lmk",
|
| 113 |
+
"lmk",
|
| 114 |
+
"lmk",
|
| 115 |
+
"lmk",
|
| 116 |
+
"lmk",
|
| 117 |
+
"lmk",
|
| 118 |
+
"lmk",
|
| 119 |
+
"lmk",
|
| 120 |
+
"lmk",
|
| 121 |
+
"lmk",
|
| 122 |
+
"lmk",
|
| 123 |
+
"lmk",
|
| 124 |
+
"lmk",
|
| 125 |
+
"lmk",
|
| 126 |
+
"lmk",
|
| 127 |
+
"lmk",
|
| 128 |
+
"lmk",
|
| 129 |
+
"lmk",
|
| 130 |
+
"lmk",
|
| 131 |
+
"lmk",
|
| 132 |
+
"lmk",
|
| 133 |
+
"lmk",
|
| 134 |
+
"lmk",
|
| 135 |
+
"lmk",
|
| 136 |
+
"lmk",
|
| 137 |
+
"lmk",
|
| 138 |
+
"lmk",
|
| 139 |
+
"lmk",
|
| 140 |
+
"lmk",
|
| 141 |
+
"lmk",
|
| 142 |
+
"lmk",
|
| 143 |
+
"lmk",
|
| 144 |
+
"lmk",
|
| 145 |
+
"lmk",
|
| 146 |
+
"lmk",
|
| 147 |
+
"lmk",
|
| 148 |
+
"lmk",
|
| 149 |
+
"lmk",
|
| 150 |
+
"lmk",
|
| 151 |
+
"lmk",
|
| 152 |
+
"lmk",
|
| 153 |
+
"lmk",
|
| 154 |
+
"lmk",
|
| 155 |
+
"lmk",
|
| 156 |
+
"lmk",
|
| 157 |
+
"lmk",
|
| 158 |
+
"lmk",
|
| 159 |
+
"lmk",
|
| 160 |
+
"lmk",
|
| 161 |
+
"lmk",
|
| 162 |
+
"lmk",
|
| 163 |
+
"lmk",
|
| 164 |
+
"lmk",
|
| 165 |
+
"lmk",
|
| 166 |
+
"lmk",
|
| 167 |
+
"lmk",
|
| 168 |
+
"lmk",
|
| 169 |
+
"lmk",
|
| 170 |
+
"lmk",
|
| 171 |
+
"lmk",
|
| 172 |
+
"lmk",
|
| 173 |
+
"lmk",
|
| 174 |
+
"lmk",
|
| 175 |
+
"lmk",
|
| 176 |
+
"lmk",
|
| 177 |
+
"lmk",
|
| 178 |
+
"lmk",
|
| 179 |
+
"lmk",
|
| 180 |
+
"lmk",
|
| 181 |
+
"lmk",
|
| 182 |
+
"lmk",
|
| 183 |
+
"lmk",
|
| 184 |
+
"lmk",
|
| 185 |
+
"lmk",
|
| 186 |
+
"lmk",
|
| 187 |
+
"lmk",
|
| 188 |
+
"lmk",
|
| 189 |
+
"lmk",
|
| 190 |
+
"lmk",
|
| 191 |
+
"lmk",
|
| 192 |
+
"lmk",
|
| 193 |
+
"lmk",
|
| 194 |
+
"lmk",
|
| 195 |
+
"lmk",
|
| 196 |
+
"lmk",
|
| 197 |
+
"lmk",
|
| 198 |
+
"lmk",
|
| 199 |
+
"lmk",
|
| 200 |
+
"lmk",
|
| 201 |
+
"lmk",
|
| 202 |
+
"lmk",
|
| 203 |
+
"lmk",
|
| 204 |
+
"lmk",
|
| 205 |
+
"lmk",
|
| 206 |
+
"lmk",
|
| 207 |
+
"lmk",
|
| 208 |
+
"lmk",
|
| 209 |
+
"lmk",
|
| 210 |
+
"lmk",
|
| 211 |
+
"lmk",
|
| 212 |
+
"lmk",
|
| 213 |
+
"lmk",
|
| 214 |
+
"lmk",
|
| 215 |
+
"lmk",
|
| 216 |
+
"lmk",
|
| 217 |
+
"lmk",
|
| 218 |
+
"lmk",
|
| 219 |
+
"lmk",
|
| 220 |
+
"lmk",
|
| 221 |
+
"lmk",
|
| 222 |
+
"lmk",
|
| 223 |
+
"lmk",
|
| 224 |
+
"lmk",
|
| 225 |
+
"lmk",
|
| 226 |
+
"lmk",
|
| 227 |
+
"lmk",
|
| 228 |
+
"lmk",
|
| 229 |
+
"lmk",
|
| 230 |
+
"lmk",
|
| 231 |
+
"lmk",
|
| 232 |
+
"lmk",
|
| 233 |
+
"lmk",
|
| 234 |
+
"lmk",
|
| 235 |
+
"lmk",
|
| 236 |
+
"lmk",
|
| 237 |
+
"lmk",
|
| 238 |
+
"lmk",
|
| 239 |
+
"lmk",
|
| 240 |
+
"lmk",
|
| 241 |
+
"lmk",
|
| 242 |
+
"lmk",
|
| 243 |
+
"lmk",
|
| 244 |
+
"lmk",
|
| 245 |
+
"lmk",
|
| 246 |
+
"lmk",
|
| 247 |
+
"lmk",
|
| 248 |
+
"lmk",
|
| 249 |
+
"lmk",
|
| 250 |
+
"lmk",
|
| 251 |
+
"lmk",
|
| 252 |
+
"lmk",
|
| 253 |
+
"lmk",
|
| 254 |
+
"lmk",
|
| 255 |
+
"lmk",
|
| 256 |
+
"lmk",
|
| 257 |
+
"lmk",
|
| 258 |
+
"lmk",
|
| 259 |
+
"lmk",
|
| 260 |
+
"lmk",
|
| 261 |
+
"lmk",
|
| 262 |
+
"lmk",
|
| 263 |
+
"lmk",
|
| 264 |
+
"lmk",
|
| 265 |
+
"lmk",
|
| 266 |
+
"lmk",
|
| 267 |
+
"lmk",
|
| 268 |
+
"lmk",
|
| 269 |
+
"lmk",
|
| 270 |
+
"lmk",
|
| 271 |
+
"lmk",
|
| 272 |
+
"lmk",
|
| 273 |
+
"lmk",
|
| 274 |
+
"lmk",
|
| 275 |
+
"lmk",
|
| 276 |
+
"lmk",
|
| 277 |
+
"lmk",
|
| 278 |
+
"lmk",
|
| 279 |
+
"lmk",
|
| 280 |
+
"lmk",
|
| 281 |
+
"lmk",
|
| 282 |
+
"lmk",
|
| 283 |
+
"lmk",
|
| 284 |
+
"lmk",
|
| 285 |
+
"lmk",
|
| 286 |
+
"lmk",
|
| 287 |
+
"lmk",
|
| 288 |
+
"lmk",
|
| 289 |
+
"lmk",
|
| 290 |
+
"lmk",
|
| 291 |
+
"lmk",
|
| 292 |
+
"lmk",
|
| 293 |
+
"lmk",
|
| 294 |
+
"lmk",
|
| 295 |
+
"lmk",
|
| 296 |
+
"lmk",
|
| 297 |
+
"lmk",
|
| 298 |
+
"lmk",
|
| 299 |
+
"lmk",
|
| 300 |
+
"lmk",
|
| 301 |
+
"lmk",
|
| 302 |
+
"lmk",
|
| 303 |
+
"lmk",
|
| 304 |
+
"lmk",
|
| 305 |
+
"lmk",
|
| 306 |
+
"lmk",
|
| 307 |
+
"lmk",
|
| 308 |
+
"lmk",
|
| 309 |
+
"lmk",
|
| 310 |
+
"lmk",
|
| 311 |
+
"lmk",
|
| 312 |
+
"lmk",
|
| 313 |
+
"lmk",
|
| 314 |
+
"lmk",
|
| 315 |
+
"lmk",
|
| 316 |
+
"lmk",
|
| 317 |
+
"lmk",
|
| 318 |
+
"lmk",
|
| 319 |
+
"lmk",
|
| 320 |
+
"lmk",
|
| 321 |
+
"lmk",
|
| 322 |
+
"lmk",
|
| 323 |
+
"lmk",
|
| 324 |
+
"lmk",
|
| 325 |
+
"lmk",
|
| 326 |
+
"lmk",
|
| 327 |
+
"lmk",
|
| 328 |
+
"lmk",
|
| 329 |
+
"lmk",
|
| 330 |
+
"lmk",
|
| 331 |
+
"lmk",
|
| 332 |
+
"lmk",
|
| 333 |
+
"lmk",
|
| 334 |
+
"lmk",
|
| 335 |
+
"lmk",
|
| 336 |
+
"lmk",
|
| 337 |
+
"lmk",
|
| 338 |
+
"lmk",
|
| 339 |
+
"lmk",
|
| 340 |
+
"lmk",
|
| 341 |
+
"lmk",
|
| 342 |
+
"lmk",
|
| 343 |
+
"lmk",
|
| 344 |
+
"lmk",
|
| 345 |
+
"lmk",
|
| 346 |
+
"lmk",
|
| 347 |
+
"lmk",
|
| 348 |
+
"lmk",
|
| 349 |
+
"lmk",
|
| 350 |
+
"lmk",
|
| 351 |
+
"lmk",
|
| 352 |
+
"lmk",
|
| 353 |
+
"lmk",
|
| 354 |
+
"lmk",
|
| 355 |
+
"lmk",
|
| 356 |
+
"lmk",
|
| 357 |
+
"lmk",
|
| 358 |
+
"lmk",
|
| 359 |
+
"lmk",
|
| 360 |
+
"lmk",
|
| 361 |
+
"lmk",
|
| 362 |
+
"lmk",
|
| 363 |
+
"lmk",
|
| 364 |
+
"lmk",
|
| 365 |
+
"lmk",
|
| 366 |
+
"lmk",
|
| 367 |
+
"lmk",
|
| 368 |
+
"lmk",
|
| 369 |
+
"lmk",
|
| 370 |
+
"lmk",
|
| 371 |
+
"lmk",
|
| 372 |
+
"lmk",
|
| 373 |
+
"lmk",
|
| 374 |
+
"lmk",
|
| 375 |
+
"lmk",
|
| 376 |
+
"lmk",
|
| 377 |
+
"lmk",
|
| 378 |
+
"lmk",
|
| 379 |
+
"lmk",
|
| 380 |
+
"lmk",
|
| 381 |
+
"lmk",
|
| 382 |
+
"lmk",
|
| 383 |
+
"lmk",
|
| 384 |
+
"lmk",
|
| 385 |
+
"lmk",
|
| 386 |
+
"lmk",
|
| 387 |
+
"lmk",
|
| 388 |
+
"lmk",
|
| 389 |
+
"lmk",
|
| 390 |
+
"lmk",
|
| 391 |
+
"lmk",
|
| 392 |
+
"lmk",
|
| 393 |
+
"lmk",
|
| 394 |
+
"lmk",
|
| 395 |
+
"lmk",
|
| 396 |
+
"lmk",
|
| 397 |
+
"lmk",
|
| 398 |
+
"lmk",
|
| 399 |
+
"lmk",
|
| 400 |
+
"lmk",
|
| 401 |
+
"lmk",
|
| 402 |
+
"lmk",
|
| 403 |
+
"lmk",
|
| 404 |
+
"lmk",
|
| 405 |
+
"lmk",
|
| 406 |
+
"lmk",
|
| 407 |
+
"lmk",
|
| 408 |
+
"lmk",
|
| 409 |
+
"lmk",
|
| 410 |
+
"lmk",
|
| 411 |
+
"lmk",
|
| 412 |
+
"lmk",
|
| 413 |
+
"lmk",
|
| 414 |
+
"lmk",
|
| 415 |
+
"lmk",
|
| 416 |
+
"lmk",
|
| 417 |
+
"lmk",
|
| 418 |
+
"lmk",
|
| 419 |
+
"lmk",
|
| 420 |
+
"lmk",
|
| 421 |
+
"lmk",
|
| 422 |
+
"lmk",
|
| 423 |
+
"lmk",
|
| 424 |
+
"lmk",
|
| 425 |
+
"skin",
|
| 426 |
+
"skin",
|
| 427 |
+
"skin",
|
| 428 |
+
"skin",
|
| 429 |
+
"skin",
|
| 430 |
+
"skin",
|
| 431 |
+
"skin",
|
| 432 |
+
"skin",
|
| 433 |
+
"skin",
|
| 434 |
+
"skin",
|
| 435 |
+
"skin",
|
| 436 |
+
"skin",
|
| 437 |
+
"skin",
|
| 438 |
+
"skin",
|
| 439 |
+
"skin",
|
| 440 |
+
"skin",
|
| 441 |
+
"skin",
|
| 442 |
+
"skin",
|
| 443 |
+
"skin",
|
| 444 |
+
"skin",
|
| 445 |
+
"skin",
|
| 446 |
+
"skin",
|
| 447 |
+
"skin",
|
| 448 |
+
"skin",
|
| 449 |
+
"skin",
|
| 450 |
+
"skin",
|
| 451 |
+
"skin",
|
| 452 |
+
"skin",
|
| 453 |
+
"skin",
|
| 454 |
+
"skin",
|
| 455 |
+
"skin",
|
| 456 |
+
"skin",
|
| 457 |
+
"skin",
|
| 458 |
+
"skin",
|
| 459 |
+
"skin",
|
| 460 |
+
"skin",
|
| 461 |
+
"skin",
|
| 462 |
+
"skin",
|
| 463 |
+
"skin",
|
| 464 |
+
"skin",
|
| 465 |
+
"skin",
|
| 466 |
+
"skin",
|
| 467 |
+
"skin",
|
| 468 |
+
"skin",
|
| 469 |
+
"skin",
|
| 470 |
+
"skin",
|
| 471 |
+
"skin",
|
| 472 |
+
"skin",
|
| 473 |
+
"skin",
|
| 474 |
+
"skin",
|
| 475 |
+
"skin",
|
| 476 |
+
"skin",
|
| 477 |
+
"skin",
|
| 478 |
+
"skin",
|
| 479 |
+
"skin",
|
| 480 |
+
"skin",
|
| 481 |
+
"skin",
|
| 482 |
+
"skin",
|
| 483 |
+
"skin",
|
| 484 |
+
"skin",
|
| 485 |
+
"skin",
|
| 486 |
+
"skin",
|
| 487 |
+
"skin",
|
| 488 |
+
"skin",
|
| 489 |
+
"skin",
|
| 490 |
+
"skin",
|
| 491 |
+
"skin",
|
| 492 |
+
"skin",
|
| 493 |
+
"skin",
|
| 494 |
+
"skin",
|
| 495 |
+
"skin",
|
| 496 |
+
"skin",
|
| 497 |
+
"skin",
|
| 498 |
+
"skin",
|
| 499 |
+
"skin",
|
| 500 |
+
"skin",
|
| 501 |
+
"skin",
|
| 502 |
+
"skin",
|
| 503 |
+
"skin",
|
| 504 |
+
"skin",
|
| 505 |
+
"skin",
|
| 506 |
+
"skin",
|
| 507 |
+
"skin",
|
| 508 |
+
"skin",
|
| 509 |
+
"skin",
|
| 510 |
+
"skin",
|
| 511 |
+
"skin",
|
| 512 |
+
"skin",
|
| 513 |
+
"skin",
|
| 514 |
+
"skin",
|
| 515 |
+
"skin",
|
| 516 |
+
"skin",
|
| 517 |
+
"skin",
|
| 518 |
+
"skin",
|
| 519 |
+
"skin",
|
| 520 |
+
"skin",
|
| 521 |
+
"skin",
|
| 522 |
+
"skin",
|
| 523 |
+
"skin",
|
| 524 |
+
"skin",
|
| 525 |
+
"skin",
|
| 526 |
+
"skin",
|
| 527 |
+
"skin",
|
| 528 |
+
"skin",
|
| 529 |
+
"skin",
|
| 530 |
+
"skin",
|
| 531 |
+
"skin",
|
| 532 |
+
"skin",
|
| 533 |
+
"skin",
|
| 534 |
+
"skin",
|
| 535 |
+
"skin",
|
| 536 |
+
"skin",
|
| 537 |
+
"skin",
|
| 538 |
+
"skin",
|
| 539 |
+
"skin",
|
| 540 |
+
"skin",
|
| 541 |
+
"skin",
|
| 542 |
+
"skin",
|
| 543 |
+
"skin",
|
| 544 |
+
"skin",
|
| 545 |
+
"skin",
|
| 546 |
+
"skin",
|
| 547 |
+
"skin",
|
| 548 |
+
"skin",
|
| 549 |
+
"skin",
|
| 550 |
+
"skin",
|
| 551 |
+
"skin",
|
| 552 |
+
"skin",
|
| 553 |
+
"skin",
|
| 554 |
+
"skin",
|
| 555 |
+
"skin",
|
| 556 |
+
"skin",
|
| 557 |
+
"skin",
|
| 558 |
+
"skin",
|
| 559 |
+
"skin",
|
| 560 |
+
"skin",
|
| 561 |
+
"skin",
|
| 562 |
+
"skin",
|
| 563 |
+
"skin",
|
| 564 |
+
"skin",
|
| 565 |
+
"skin",
|
| 566 |
+
"skin",
|
| 567 |
+
"skin",
|
| 568 |
+
"skin",
|
| 569 |
+
"skin",
|
| 570 |
+
"skin",
|
| 571 |
+
"skin",
|
| 572 |
+
"skin",
|
| 573 |
+
"skin",
|
| 574 |
+
"skin",
|
| 575 |
+
"lid",
|
| 576 |
+
"lid",
|
| 577 |
+
"lid",
|
| 578 |
+
"lid",
|
| 579 |
+
"lid",
|
| 580 |
+
"lid",
|
| 581 |
+
"lid",
|
| 582 |
+
"lid",
|
| 583 |
+
"lid",
|
| 584 |
+
"lid",
|
| 585 |
+
"lid",
|
| 586 |
+
"lid",
|
| 587 |
+
"lid",
|
| 588 |
+
"lid",
|
| 589 |
+
"lid",
|
| 590 |
+
"lid",
|
| 591 |
+
"lid",
|
| 592 |
+
"lid",
|
| 593 |
+
"lid",
|
| 594 |
+
"lid",
|
| 595 |
+
"lid",
|
| 596 |
+
"lid",
|
| 597 |
+
"lid",
|
| 598 |
+
"lid",
|
| 599 |
+
"lid",
|
| 600 |
+
"lid",
|
| 601 |
+
"lid",
|
| 602 |
+
"lid",
|
| 603 |
+
"lid",
|
| 604 |
+
"lid",
|
| 605 |
+
"lid",
|
| 606 |
+
"lid",
|
| 607 |
+
"lid",
|
| 608 |
+
"lid",
|
| 609 |
+
"lid",
|
| 610 |
+
"lid",
|
| 611 |
+
"teeth_u",
|
| 612 |
+
"teeth_u",
|
| 613 |
+
"teeth_u",
|
| 614 |
+
"teeth_u",
|
| 615 |
+
"teeth_u",
|
| 616 |
+
"teeth_u",
|
| 617 |
+
"teeth_u",
|
| 618 |
+
"teeth_l",
|
| 619 |
+
"teeth_l",
|
| 620 |
+
"teeth_l",
|
| 621 |
+
"teeth_l",
|
| 622 |
+
"teeth_l",
|
| 623 |
+
"teeth_l",
|
| 624 |
+
"teeth_l",
|
| 625 |
+
"teeth_l",
|
| 626 |
+
"teeth_l",
|
| 627 |
+
"iris",
|
| 628 |
+
"iris"
|
| 629 |
+
],
|
| 630 |
+
"likelihood": "studentt",
|
| 631 |
+
"nu": 10.0,
|
| 632 |
+
"std_factor": 1.118033988749895,
|
| 633 |
+
"log_scale_range": [
|
| 634 |
+
-3.0,
|
| 635 |
+
6.0
|
| 636 |
+
],
|
| 637 |
+
"side_heads": {
|
| 638 |
+
"wflw": 98,
|
| 639 |
+
"300w": 68
|
| 640 |
+
},
|
| 641 |
+
"side_refine": true,
|
| 642 |
+
"iterations": 12000,
|
| 643 |
+
"training": {
|
| 644 |
+
"backbone": "hrnet_w48",
|
| 645 |
+
"res": 256,
|
| 646 |
+
"batch": 48,
|
| 647 |
+
"iters": 12000,
|
| 648 |
+
"lr": 3e-05,
|
| 649 |
+
"lr_bb": 1e-05,
|
| 650 |
+
"nll": "studentt",
|
| 651 |
+
"flat": 0.5,
|
| 652 |
+
"wd": 0.05,
|
| 653 |
+
"warmup": 200,
|
| 654 |
+
"workers": 6,
|
| 655 |
+
"val_every": 2000,
|
| 656 |
+
"val_n": 4096,
|
| 657 |
+
"pretrained": 0,
|
| 658 |
+
"neck": 32,
|
| 659 |
+
"stride": 8,
|
| 660 |
+
"w_lid": 1.0,
|
| 661 |
+
"w_teeth": 1.0,
|
| 662 |
+
"w_occ": 1.0,
|
| 663 |
+
"seed": 0,
|
| 664 |
+
"amp": 1,
|
| 665 |
+
"min_vis": 0.2,
|
| 666 |
+
"lowres_min": 48,
|
| 667 |
+
"lowres_p": 0.5,
|
| 668 |
+
"lowres_jpeg": 0.5,
|
| 669 |
+
"lowres_q": 30,
|
| 670 |
+
"aff_smin": 0.5,
|
| 671 |
+
"aff_smax": 1.5,
|
| 672 |
+
"aff_rot": 80.0,
|
| 673 |
+
"aff_rot_p": 0.6,
|
| 674 |
+
"log_every": 200,
|
| 675 |
+
"wild_sets": "wflw,300w",
|
| 676 |
+
"wild_bs": 24,
|
| 677 |
+
"wild_w": 1.0,
|
| 678 |
+
"bg_p": 0.5,
|
| 679 |
+
"relight_p": 0.3,
|
| 680 |
+
"occ2_p": 0.2,
|
| 681 |
+
"glasses_p": 0.1,
|
| 682 |
+
"wild_mix": "0.5,0.5",
|
| 683 |
+
"lr_rf": 0.0003,
|
| 684 |
+
"side_rf": 1,
|
| 685 |
+
"aff_shift": 0.1,
|
| 686 |
+
"occ_p": 1.0,
|
| 687 |
+
"grey_p": 0.05,
|
| 688 |
+
"flip": 1,
|
| 689 |
+
"fresh_aug": 1,
|
| 690 |
+
"studio_fresh": 1,
|
| 691 |
+
"mild_p": 0.5,
|
| 692 |
+
"w620": 1.0,
|
| 693 |
+
"w620_jaw": 1.0,
|
| 694 |
+
"w620_dense": 0.3,
|
| 695 |
+
"w620_uc": 1.0,
|
| 696 |
+
"nu": 10.0
|
| 697 |
+
}
|
| 698 |
+
}
|
model/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b37075caa3eaf0d00f23f32925b16ec3a8e2e45c42aa9c00956f94f2574e6526
|
| 3 |
+
size 415939712
|
points.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The 620 output points as locations on the GNM Head mesh (points_gnm.json)."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
DEFAULT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "points_gnm.json")
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def load_points(path=DEFAULT):
|
| 12 |
+
"""Return the point rows in output order: dicts with index, class, region, vertices, weights."""
|
| 13 |
+
with open(path, encoding="utf-8") as fh:
|
| 14 |
+
return json.load(fh)["points"]
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def gnm_positions(vertices_xyz, points=None):
|
| 18 |
+
"""Return (620, 3) positions of the points on a posed GNM Head mesh from its (V, 3) vertices."""
|
| 19 |
+
points = load_points() if points is None else points
|
| 20 |
+
V = np.asarray(vertices_xyz, np.float64)
|
| 21 |
+
out = np.zeros((len(points), 3), np.float64)
|
| 22 |
+
for p in points:
|
| 23 |
+
out[p["index"]] = np.asarray(p["weights"], np.float64) @ V[p["vertices"]]
|
| 24 |
+
return out
|
points_gnm.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
prep/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Data preparation: Ava-256 crops and labels, in-the-wild faces, 620-point pseudo-label maps."""
|
prep/captures.txt
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
20230405--1635--AAN112
|
| 2 |
+
20230831--0814--ADL311
|
| 3 |
+
20230324--0820--AEY864
|
| 4 |
+
20230810--1355--AJR151
|
| 5 |
+
20230810--1630--ANX726
|
| 6 |
+
20230629--1159--APP152
|
| 7 |
+
20230726--1657--AYE877
|
| 8 |
+
20230308--1352--BDF920
|
| 9 |
+
20220412--1326--BGR645
|
| 10 |
+
20220711--1300--BHC106
|
| 11 |
+
20230316--1103--BHK376
|
| 12 |
+
20220809--1034--BJM420
|
| 13 |
+
20230328--0800--BLY735
|
| 14 |
+
20220815--1307--BMP511
|
| 15 |
+
20220627--0812--BPM833
|
| 16 |
+
20220819--0803--BUS250
|
| 17 |
+
20230914--1105--BXQ083
|
| 18 |
+
20210818--1332--CDR970
|
| 19 |
+
20220831--0751--CMS162
|
| 20 |
+
20230224--1359--CMZ386
|
| 21 |
+
20230309--0820--CPA784
|
| 22 |
+
20230901--1429--CPP930
|
| 23 |
+
20230728--0757--CRV122
|
| 24 |
+
20230720--1025--CTP175
|
| 25 |
+
20230706--1016--CVR434
|
| 26 |
+
20230306--0812--CYQ521
|
| 27 |
+
20230615--1702--DAT749
|
| 28 |
+
20230404--1217--DED409
|
| 29 |
+
20230908--1645--DHA971
|
| 30 |
+
20220614--1135--DNM410
|
| 31 |
+
20210819--0903--DOT682
|
| 32 |
+
20220808--0809--DPE040
|
| 33 |
+
20230207--1619--DTW301
|
| 34 |
+
20230511--0810--DUA525
|
| 35 |
+
20230817--1136--DZS003
|
| 36 |
+
20230303--0802--EFQ599
|
| 37 |
+
20211006--0836--EID363
|
| 38 |
+
20230801--1420--EJG940
|
| 39 |
+
20230410--1641--ENU192
|
| 40 |
+
20230816--1219--EPP420
|
| 41 |
+
20230608--1209--EPY965
|
| 42 |
+
20230915--0810--EVY126
|
| 43 |
+
20230728--1002--EXA831
|
| 44 |
+
20230310--1106--FCT871
|
| 45 |
+
20230728--1203--FGK870
|
| 46 |
+
20230601--1428--FJC651
|
| 47 |
+
20230602--1453--FMB793
|
| 48 |
+
20230728--1401--FPK790
|
| 49 |
+
20230307--0805--FQC133
|
| 50 |
+
20220315--1053--FWC239
|
| 51 |
+
20210810--1306--FXN596
|
| 52 |
+
20220815--1041--FZC097
|
| 53 |
+
20230914--1035--GBH400
|
| 54 |
+
20230317--1054--GBQ558
|
| 55 |
+
20230313--0819--GCZ208
|
| 56 |
+
20230404--0804--GJI283
|
| 57 |
+
20230814--0822--GJL377
|
| 58 |
+
20230814--0822--GTA798
|
| 59 |
+
20220824--0804--GTU803
|
| 60 |
+
20230410--1017--GVM433
|
| 61 |
+
20230316--1356--GXR876
|
| 62 |
+
20220815--0815--GZL442
|
| 63 |
+
20230726--0805--GZU008
|
| 64 |
+
20220811--1704--HAZ933
|
| 65 |
+
20230906--0803--HBM931
|
| 66 |
+
20230705--1657--HBT289
|
| 67 |
+
20230503--1006--HDM935
|
| 68 |
+
20220315--0818--HDZ165
|
| 69 |
+
20220726--1314--HNU630
|
| 70 |
+
20220214--0846--HOS732
|
| 71 |
+
20230323--1044--HRO745
|
| 72 |
+
20230630--1651--HTL154
|
| 73 |
+
20230228--1358--HUJ848
|
| 74 |
+
20230607--1409--HVO722
|
| 75 |
+
20230815--0816--HWN858
|
| 76 |
+
20230322--1706--IBQ026
|
| 77 |
+
20220808--1036--IBX730
|
| 78 |
+
20220802--0838--IDD857
|
| 79 |
+
20230406--1655--IEM175
|
| 80 |
+
20230328--1214--IEY203
|
| 81 |
+
20210902--0817--IFG774
|
| 82 |
+
20230525--1403--IHO061
|
| 83 |
+
20230724--0814--IIQ590
|
| 84 |
+
20230628--1405--IJB027
|
| 85 |
+
20230308--1639--IKB041
|
| 86 |
+
20230920--1404--INQ807
|
| 87 |
+
20230809--1645--ITH925
|
| 88 |
+
20230306--1402--IXG477
|
| 89 |
+
20220712--1040--JEH262
|
| 90 |
+
20230919--1056--JEN167
|
| 91 |
+
20230612--1211--JFS610
|
| 92 |
+
20230605--1354--JJA915
|
| 93 |
+
20230905--1354--JOX535
|
| 94 |
+
20220705--1055--JPG686
|
| 95 |
+
20230515--0817--JUP244
|
| 96 |
+
20230503--1400--JUP632
|
| 97 |
+
20220114--0820--JZE802
|
| 98 |
+
20220830--1711--JZN879
|
| 99 |
+
20230802--1003--KAP399
|
| 100 |
+
20210827--0906--KDA058
|
| 101 |
+
20230213--0817--KGV285
|
| 102 |
+
20230707--0817--KIX259
|
| 103 |
+
20211001--0855--KJJ701
|
| 104 |
+
20230124--1415--KKF424
|
| 105 |
+
20220314--1116--KMD200
|
| 106 |
+
20230516--0802--KOH994
|
| 107 |
+
20220829--0836--KSA222
|
| 108 |
+
20230531--1207--KTI779
|
| 109 |
+
20230307--1648--KUG325
|
| 110 |
+
20220812--1031--KVG225
|
| 111 |
+
20230925--1644--KWL586
|
| 112 |
+
20230725--1218--KXI621
|
| 113 |
+
20230222--1639--KYL945
|
| 114 |
+
20210901--0833--LAS440
|
| 115 |
+
20230412--1218--LCJ763
|
| 116 |
+
20231003--1715--LGV914
|
| 117 |
+
20230330--0823--LMT312
|
| 118 |
+
20230810--0819--LNL587
|
| 119 |
+
20220901--1305--LNP057
|
| 120 |
+
20230802--1205--LPB114
|
| 121 |
+
20230814--1700--LVD531
|
| 122 |
+
20230302--1654--LVY864
|
| 123 |
+
20230809--0814--LWN501
|
| 124 |
+
20230609--0959--LXC339
|
| 125 |
+
20210929--0827--MCR809
|
| 126 |
+
20230206--1354--MFS340
|
| 127 |
+
20230925--0843--MGR740
|
| 128 |
+
20230720--1415--MJX355
|
| 129 |
+
20220815--1656--MOP211
|
| 130 |
+
20230707--1200--MQL211
|
| 131 |
+
20230719--1655--MVB233
|
| 132 |
+
20230323--1656--MZL038
|
| 133 |
+
20230804--1159--MZW030
|
| 134 |
+
20220714--1031--NAB660
|
| 135 |
+
20230908--1549--NCC597
|
| 136 |
+
20220805--1649--NCU289
|
| 137 |
+
20230810--1201--NDZ630
|
| 138 |
+
20230601--1051--NEB896
|
| 139 |
+
20230109--1327--NFS933
|
| 140 |
+
20220311--1115--NFU480
|
| 141 |
+
20230912--1313--NJI020
|
| 142 |
+
20220304--1320--NNQ107
|
| 143 |
+
20230502--1030--NOG724
|
| 144 |
+
20210817--0900--NRE683
|
| 145 |
+
20230918--1040--NTA876
|
| 146 |
+
20230405--1213--NWM763
|
| 147 |
+
20220819--1313--NYY338
|
| 148 |
+
20230829--1404--OKX508
|
| 149 |
+
20220215--0801--ONK705
|
| 150 |
+
20230317--1648--ONK818
|
| 151 |
+
20230502--0816--ORZ494
|
| 152 |
+
20230725--1407--OWI326
|
| 153 |
+
20230810--0812--OXS375
|
| 154 |
+
20220407--0816--OYQ923
|
| 155 |
+
20220510--1051--OZZ515
|
| 156 |
+
20210928--0843--PAK800
|
| 157 |
+
20230413--1031--PDG961
|
| 158 |
+
20230817--0805--PDX936
|
| 159 |
+
20230314--1632--PFN770
|
| 160 |
+
20220406--1314--PGO261
|
| 161 |
+
20230501--1015--PHE887
|
| 162 |
+
20220401--0841--PHT202
|
| 163 |
+
20230925--1356--PKH444
|
| 164 |
+
20230717--0836--PSV686
|
| 165 |
+
20230815--1401--PWG860
|
| 166 |
+
20230607--0809--PZC599
|
| 167 |
+
20230629--1004--QBN395
|
| 168 |
+
20230106--1047--QCW581
|
| 169 |
+
20230808--1656--QFA788
|
| 170 |
+
20230322--0820--QHY185
|
| 171 |
+
20230906--1059--QLL112
|
| 172 |
+
20230717--1225--QNG975
|
| 173 |
+
20230915--1648--QVU001
|
| 174 |
+
20230811--1202--QWY097
|
| 175 |
+
20230517--1211--QZP994
|
| 176 |
+
20230518--1009--RFZ285
|
| 177 |
+
20220307--1342--RGA575
|
| 178 |
+
20230313--1653--RHL466
|
| 179 |
+
20230919--1626--RHU956
|
| 180 |
+
20230309--1343--RIM072
|
| 181 |
+
20230530--1417--RKG241
|
| 182 |
+
20230726--1016--RLB874
|
| 183 |
+
20230327--1036--RSC756
|
| 184 |
+
20230727--1211--RSJ093
|
| 185 |
+
20230630--1424--RSY180
|
| 186 |
+
20230502--1418--RWA092
|
| 187 |
+
20230403--1022--RWS274
|
| 188 |
+
20220426--1126--SCG995
|
| 189 |
+
20230302--0808--SEI058
|
| 190 |
+
20230627--1701--SIQ409
|
| 191 |
+
20210929--1334--SKB942
|
| 192 |
+
20220818--1653--SSF476
|
| 193 |
+
20230728--1206--SVG441
|
| 194 |
+
20220303--1247--TAY503
|
| 195 |
+
20230807--0839--TCE049
|
| 196 |
+
20230315--0816--TEA904
|
| 197 |
+
20220202--0844--THN461
|
| 198 |
+
20230526--1657--TLD140
|
| 199 |
+
20220727--1059--TMA321
|
| 200 |
+
20230602--0800--TOU998
|
| 201 |
+
20220610--0901--TRP396
|
| 202 |
+
20230227--1659--TRT983
|
| 203 |
+
20230807--1011--TWH159
|
| 204 |
+
20230410--0805--TWP078
|
| 205 |
+
20230227--1052--TXE513
|
| 206 |
+
20230816--0807--TYX812
|
| 207 |
+
20230330--1421--UCD539
|
| 208 |
+
20220711--1048--UGU718
|
| 209 |
+
20210922--0839--UHV563
|
| 210 |
+
20230515--1236--UII082
|
| 211 |
+
20230918--1402--UIQ957
|
| 212 |
+
20230315--1638--UIS049
|
| 213 |
+
20230601--1208--UJD603
|
| 214 |
+
20230627--1045--UMJ550
|
| 215 |
+
20230407--1229--UMQ592
|
| 216 |
+
20230224--1102--UPC002
|
| 217 |
+
20230516--1240--UQD073
|
| 218 |
+
20230804--1714--URP626
|
| 219 |
+
20220809--1321--UTC375
|
| 220 |
+
20230831--1037--UTI042
|
| 221 |
+
20230302--1343--UWK768
|
| 222 |
+
20230718--1021--UYF211
|
| 223 |
+
20221213--1425--VBY735
|
| 224 |
+
20220818--1304--VCN655
|
| 225 |
+
20230301--1646--VNS883
|
| 226 |
+
20230424--1024--VOG927
|
| 227 |
+
20230131--1229--WBX393
|
| 228 |
+
20220510--0837--WFU321
|
| 229 |
+
20230721--1207--WOZ406
|
| 230 |
+
20230914--1702--WOZ485
|
| 231 |
+
20230710--0814--WWY595
|
| 232 |
+
20230615--0808--WXT125
|
| 233 |
+
20230508--1015--XGD109
|
| 234 |
+
20230508--0828--XIK500
|
| 235 |
+
20220218--0839--XIT043
|
| 236 |
+
20210831--0815--XJT672
|
| 237 |
+
20211001--1304--XMY093
|
| 238 |
+
20220310--1336--XRZ540
|
| 239 |
+
20230811--1636--XUD838
|
| 240 |
+
20230608--1009--YIK625
|
| 241 |
+
20230808--1157--YJF815
|
| 242 |
+
20230808--0759--YLC760
|
| 243 |
+
20230324--1647--YWZ007
|
| 244 |
+
20230718--1434--YYA788
|
| 245 |
+
20230512--1009--ZEL435
|
| 246 |
+
20230915--1145--ZEM686
|
| 247 |
+
20230320--0810--ZJN985
|
| 248 |
+
20230303--1634--ZKN852
|
| 249 |
+
20230323--1348--ZQK191
|
| 250 |
+
20220310--1128--ZSC414
|
| 251 |
+
20220216--1324--ZSE894
|
| 252 |
+
20230607--1652--ZUE409
|
| 253 |
+
20220831--1047--ZUS575
|
| 254 |
+
20230703--0826--ZWG600
|
| 255 |
+
20230421--1716--ZWR419
|
| 256 |
+
20230324--1118--ZZS850
|
prep/closed_frames.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Extra closed-eye frames: lid gaps of every registered frame, then a sample list of new ones."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.closed_frames scan <scan dir> <samples.json>
|
| 4 |
+
# [--workers 4] [--captures ids.txt]
|
| 5 |
+
# python -m facepoints.prep.closed_frames pick <scan dir> <samples.json> <out.json>
|
| 6 |
+
# [--closed 15] [--half 5] [--cams 20] [--seed 1]
|
| 7 |
+
# scan: streams each capture's registration_vertices.zip (about 440 MB, about 5,000 frames) once and
|
| 8 |
+
# writes <scan dir>/<capture>.npz: frame, lid_L, lid_R (signed lid gap in mm along each eye's
|
| 9 |
+
# neutral up direction), lip_gap (mm). The neutral frame is the sample list's first frame.
|
| 10 |
+
# pick: per capture, closed events (runs of consecutive registered frames whose smaller lid gap is
|
| 11 |
+
# under 1 mm; a wink counts) not within 3 frames of an already-sampled frame -> the deepest frame
|
| 12 |
+
# of each event, --closed events spread evenly over the timeline; plus --half frames with a gap of
|
| 13 |
+
# 1-3 mm the same way. Cameras: a seeded random subset of the capture's sampled cameras. The
|
| 14 |
+
# output is extract.py's sample format with "neutral" set, to extract into a second root and
|
| 15 |
+
# merge with merge_data.py. Training holds few closed eyes otherwise (a median of 7 frames in
|
| 16 |
+
# 50 per capture).
|
| 17 |
+
import argparse
|
| 18 |
+
import concurrent.futures as cf
|
| 19 |
+
import io
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import urllib.request
|
| 23 |
+
import zipfile
|
| 24 |
+
|
| 25 |
+
import numpy as np
|
| 26 |
+
|
| 27 |
+
from facepoints.prep import labels as L
|
| 28 |
+
|
| 29 |
+
BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/16TB"
|
| 30 |
+
CLOSED_MM, HALF_MM = 1.0, 3.0
|
| 31 |
+
NEAR = 7 # a window of 7 registered frames = within 3 of a sampled one
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def scan_one(cap, neutral, out, points):
|
| 35 |
+
"""Write one capture's lid and lip gaps per registered frame; return (capture, summary)."""
|
| 36 |
+
p = os.path.join(out, f"{cap}.npz")
|
| 37 |
+
if os.path.exists(p):
|
| 38 |
+
return cap, "skip"
|
| 39 |
+
url = f"{BASE}/{cap}/decoder/kinematic_tracking/registration_vertices.zip"
|
| 40 |
+
with urllib.request.urlopen(url, timeout=600) as r:
|
| 41 |
+
z = zipfile.ZipFile(io.BytesIO(r.read()))
|
| 42 |
+
ps = L.PointSet(points)
|
| 43 |
+
X0 = L.read_ply(z.read(f"{neutral:06d}.ply"))
|
| 44 |
+
up = {}
|
| 45 |
+
for e, (u, lo) in ps.lid_sets.items():
|
| 46 |
+
v = X0[u].mean(0) - X0[lo].mean(0)
|
| 47 |
+
up[e] = v / np.linalg.norm(v)
|
| 48 |
+
fr, gl, gr, lg = [], [], [], []
|
| 49 |
+
for n in sorted(z.namelist()):
|
| 50 |
+
if not n.endswith(".ply"):
|
| 51 |
+
continue
|
| 52 |
+
try:
|
| 53 |
+
f = ps.flags(L.read_ply(z.read(n)), up)
|
| 54 |
+
except Exception: # noqa: BLE001 - one bad entry must not kill the capture
|
| 55 |
+
continue
|
| 56 |
+
fr.append(int(n[:-4]))
|
| 57 |
+
gl.append(f["lid_L"])
|
| 58 |
+
gr.append(f["lid_R"])
|
| 59 |
+
lg.append(f["lip_gap"])
|
| 60 |
+
gaps = dict(lid_L=np.array(gl, np.float32), lid_R=np.array(gr, np.float32))
|
| 61 |
+
np.savez_compressed(p, frame=np.array(fr), lip_gap=np.array(lg, np.float32), **gaps)
|
| 62 |
+
return cap, f"{len(fr)} frames, closed {(np.minimum(gl, gr) < CLOSED_MM).sum()}"
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def scan(out, samples, workers, capfile, points):
|
| 66 |
+
"""Scan every capture of the sample list with a process pool."""
|
| 67 |
+
os.makedirs(out, exist_ok=True)
|
| 68 |
+
with open(samples) as fh:
|
| 69 |
+
S = json.load(fh)
|
| 70 |
+
if capfile:
|
| 71 |
+
with open(capfile) as fh:
|
| 72 |
+
caps = fh.read().split()
|
| 73 |
+
else:
|
| 74 |
+
caps = sorted(S)
|
| 75 |
+
caps = [c for c in caps if "skipped" not in S[c]]
|
| 76 |
+
L.topology_path() # download once before the pool starts
|
| 77 |
+
with cf.ProcessPoolExecutor(workers) as ex:
|
| 78 |
+
futs = {ex.submit(scan_one, c, S[c]["frames"][0], out, points): c for c in caps}
|
| 79 |
+
for k, fu in enumerate(cf.as_completed(futs)):
|
| 80 |
+
try:
|
| 81 |
+
c, msg = fu.result()
|
| 82 |
+
print(f"{k + 1}/{len(caps)} {c[-6:]} {msg}", flush=True)
|
| 83 |
+
except Exception as e: # noqa: BLE001 - a failed capture is redone on the next run
|
| 84 |
+
print(f"ERROR {futs[fu][-6:]} {type(e).__name__}: {e}", flush=True)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def events(mask):
|
| 88 |
+
"""Return the runs of consecutive True entries of mask as index arrays."""
|
| 89 |
+
idx = np.flatnonzero(mask)
|
| 90 |
+
if not len(idx):
|
| 91 |
+
return []
|
| 92 |
+
cut = np.flatnonzero(np.diff(idx) > 1) + 1
|
| 93 |
+
return np.split(idx, cut)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def pick(scan_dir, samples, outp, n_closed, n_half, n_cams, seed):
|
| 97 |
+
"""Write a sample list of new closed and half-closed frames per capture."""
|
| 98 |
+
with open(samples) as fh:
|
| 99 |
+
S = json.load(fh)
|
| 100 |
+
rng = np.random.default_rng(seed)
|
| 101 |
+
res, tot = {}, {"closed": 0, "half": 0, "captures": 0}
|
| 102 |
+
for cap in sorted(S):
|
| 103 |
+
p = os.path.join(scan_dir, f"{cap}.npz")
|
| 104 |
+
if "skipped" in S[cap] or not os.path.exists(p):
|
| 105 |
+
continue
|
| 106 |
+
z = np.load(p)
|
| 107 |
+
fr = z["frame"]
|
| 108 |
+
g = np.minimum(z["lid_L"], z["lid_R"])
|
| 109 |
+
old = np.isin(fr, S[cap]["frames"])
|
| 110 |
+
near = np.convolve(old.astype(int), np.ones(NEAR, int), "same") > 0
|
| 111 |
+
chosen = []
|
| 112 |
+
for lo, hi, n, key in (
|
| 113 |
+
(-99.0, CLOSED_MM, n_closed, "closed"),
|
| 114 |
+
(CLOSED_MM, HALF_MM, n_half, "half"),
|
| 115 |
+
):
|
| 116 |
+
ev = [e for e in events((g >= lo) & (g < hi)) if not near[e].any()]
|
| 117 |
+
deep = [e[np.argmin(g[e])] for e in ev]
|
| 118 |
+
if len(deep) > n:
|
| 119 |
+
deep = [deep[int(j)] for j in np.linspace(0, len(deep) - 1, n).round()]
|
| 120 |
+
chosen += [int(fr[i]) for i in deep]
|
| 121 |
+
tot[key] += len(deep)
|
| 122 |
+
if not chosen:
|
| 123 |
+
continue
|
| 124 |
+
cams_all = sorted({c for v in S[cap]["cams"].values() for c in v})
|
| 125 |
+
cams = {
|
| 126 |
+
str(f): sorted(rng.choice(cams_all, min(n_cams, len(cams_all)), replace=False).tolist())
|
| 127 |
+
for f in chosen
|
| 128 |
+
}
|
| 129 |
+
res[cap] = {"neutral": S[cap]["frames"][0], "frames": chosen, "cams": cams}
|
| 130 |
+
tot["captures"] += 1
|
| 131 |
+
with open(outp, "w") as fh:
|
| 132 |
+
json.dump(res, fh)
|
| 133 |
+
print(f"PICK {tot} -> {outp}", flush=True)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def main():
|
| 137 |
+
"""Run the scan or pick subcommand."""
|
| 138 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 139 |
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
| 140 |
+
s = sub.add_parser("scan")
|
| 141 |
+
s.add_argument("scan_dir")
|
| 142 |
+
s.add_argument("samples", help="the sample list (its first frame per capture is the neutral)")
|
| 143 |
+
s.add_argument("--workers", type=int, default=4)
|
| 144 |
+
s.add_argument("--captures", help="file of capture ids")
|
| 145 |
+
s.add_argument("--points", default=L.DEFAULT_POINTS)
|
| 146 |
+
k = sub.add_parser("pick")
|
| 147 |
+
k.add_argument("scan_dir")
|
| 148 |
+
k.add_argument("samples", help="the sample list already extracted")
|
| 149 |
+
k.add_argument("out", help="the new sample list to extract")
|
| 150 |
+
k.add_argument("--closed", type=int, default=15, help="closed events per capture")
|
| 151 |
+
k.add_argument("--half", type=int, default=5, help="half-closed frames per capture")
|
| 152 |
+
k.add_argument("--cams", type=int, default=20, help="cameras per frame")
|
| 153 |
+
k.add_argument("--seed", type=int, default=1)
|
| 154 |
+
a = ap.parse_args()
|
| 155 |
+
if a.cmd == "scan":
|
| 156 |
+
scan(a.scan_dir, a.samples, a.workers, a.captures, a.points)
|
| 157 |
+
else:
|
| 158 |
+
pick(a.scan_dir, a.samples, a.out, a.closed, a.half, a.cams, a.seed)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
if __name__ == "__main__":
|
| 162 |
+
main()
|
prep/detect_wild.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run a released model on the wild training crops: the detections the pseudo-label maps use."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.detect_wild <model dir> <wild dir> <out dir>
|
| 4 |
+
# One forward pass per crop with the crop itself as the box (it is already the training frame).
|
| 5 |
+
# Writes <out>/<set>.npz: pts (N, 620, 2) detections and gt (N, K, 2) hand labels, both in crop px,
|
| 6 |
+
# val (N,) the held-out flag, eyes (2,) the outer eye corner ids.
|
| 7 |
+
import argparse
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
import cv2
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
from facepoints.detect import Detector
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def main():
|
| 17 |
+
"""Detect every face of every set under the wild dir."""
|
| 18 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 19 |
+
ap.add_argument("model", help="folder with model.safetensors and config.json")
|
| 20 |
+
ap.add_argument("wild", help="wild_prep.py output")
|
| 21 |
+
ap.add_argument("out", help="output folder")
|
| 22 |
+
ap.add_argument("--sets", default="wflw,300w")
|
| 23 |
+
a = ap.parse_args()
|
| 24 |
+
det = Detector(a.model)
|
| 25 |
+
os.makedirs(a.out, exist_ok=True)
|
| 26 |
+
for ds in a.sets.split(","):
|
| 27 |
+
z = np.load(os.path.join(a.wild, ds, "labels.npz"))
|
| 28 |
+
pts = []
|
| 29 |
+
for i in range(len(z["uv"])):
|
| 30 |
+
im = cv2.imread(os.path.join(a.wild, ds, f"{i:05d}.jpg"))
|
| 31 |
+
pts.append(det.forward(im, (0.0, 0.0, float(im.shape[1])))["points"])
|
| 32 |
+
if i % 1000 == 0:
|
| 33 |
+
print(f"{ds} {i}/{len(z['uv'])}", flush=True)
|
| 34 |
+
p = os.path.join(a.out, f"{ds}.npz")
|
| 35 |
+
np.savez_compressed(
|
| 36 |
+
p, pts=np.array(pts, np.float32), gt=z["uv"], val=z["val"], eyes=z["eyes"]
|
| 37 |
+
)
|
| 38 |
+
print(f"wrote {p}: {len(pts)} faces", flush=True)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
main()
|
prep/extract.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Training crops and labels from Ava-256: one square face crop per sampled frame and camera."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.extract <samples.json> <out dir> [--tier 16TB]
|
| 4 |
+
# [--crop 512] [--workers 16] [--captures ids.txt]
|
| 5 |
+
# Reads Ava-256's public S3 zips by HTTP range (http_zip.HttpFile): per capture the registration,
|
| 6 |
+
# head-pose and keypoint zips once, then each sampled frame's photo per camera. The crop is a square
|
| 7 |
+
# around the projected mesh points (margin 25 % of their extent) resized to --crop px; labels are in
|
| 8 |
+
# crop pixels. Writes <out>/<capture>/<frame>_<cam>.jpg and <out>/<capture>/labels.npz with, per
|
| 9 |
+
# image: uv (n, 2) crop px (NaN = no label), vis (n,) bool, z (n,) mm, box (4,) photo px, frame,
|
| 10 |
+
# cam, fx (photo px; mm at a label = px x z / fx) and the per-frame flags lid_L / lid_R / lip_gap
|
| 11 |
+
# (mm). Resumable per capture (labels.npz present = done). The sample list comes from
|
| 12 |
+
# sample_frames.py: {capture: {frames: [...], cams: {frame: [camera ids]}, neutral?: frame}}.
|
| 13 |
+
import argparse
|
| 14 |
+
import concurrent.futures as cf
|
| 15 |
+
import io
|
| 16 |
+
import json
|
| 17 |
+
import multiprocessing
|
| 18 |
+
import os
|
| 19 |
+
import traceback
|
| 20 |
+
import urllib.request
|
| 21 |
+
import zipfile
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
from PIL import Image
|
| 25 |
+
|
| 26 |
+
from facepoints.prep import labels as L
|
| 27 |
+
from facepoints.prep.http_zip import HttpFile
|
| 28 |
+
|
| 29 |
+
BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256"
|
| 30 |
+
TIERS = {"4TB": (0.25, (1024, 667)), "16TB": (0.5, (2048, 1334))} # photo scale, (W, H)
|
| 31 |
+
BOX_MARGIN = 1.25 # crop side = the mesh points' extent x this
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def one_capture(cap, sample, out, tier, crop, points, topology_obj):
|
| 35 |
+
"""Extract one capture's sampled frames; return (capture, image count or "skip")."""
|
| 36 |
+
d = os.path.join(out, cap)
|
| 37 |
+
done = os.path.join(d, "labels.npz")
|
| 38 |
+
if os.path.exists(done) or "skipped" in sample:
|
| 39 |
+
return cap, "skip"
|
| 40 |
+
os.makedirs(d, exist_ok=True)
|
| 41 |
+
scale, HW = TIERS[tier]
|
| 42 |
+
ps = L.PointSet(points, topology_obj)
|
| 43 |
+
D = f"{BASE}/{tier}/{cap}/decoder/"
|
| 44 |
+
with urllib.request.urlopen(D + "camera_calibration.json", timeout=60) as r:
|
| 45 |
+
cal = {e["cameraId"]: e for e in json.load(r)["KRT"]}
|
| 46 |
+
regz, posz, kpz = (
|
| 47 |
+
zipfile.ZipFile(HttpFile(D + p))
|
| 48 |
+
for p in (
|
| 49 |
+
"kinematic_tracking/registration_vertices.zip",
|
| 50 |
+
"head_pose/head_pose.zip",
|
| 51 |
+
"keypoints_3d/keypoints_3d.zip",
|
| 52 |
+
)
|
| 53 |
+
)
|
| 54 |
+
frames = sample["frames"]
|
| 55 |
+
# the lid "up" direction comes from a neutral frame: the first sampled frame, or an explicit one
|
| 56 |
+
X0, _, _ = L.world(regz, posz, kpz, sample.get("neutral", frames[0]))
|
| 57 |
+
up = {}
|
| 58 |
+
for e, (u, lo) in ps.lid_sets.items():
|
| 59 |
+
v = X0[u].mean(0) - X0[lo].mean(0)
|
| 60 |
+
up[e] = v / np.linalg.norm(v)
|
| 61 |
+
imz = {}
|
| 62 |
+
keys = ("uv", "vis", "z", "box", "frame", "cam", "fx", "lid_L", "lid_R", "lip_gap")
|
| 63 |
+
rec = {k: [] for k in keys}
|
| 64 |
+
n_img = 0
|
| 65 |
+
for f in frames:
|
| 66 |
+
X, M, kp = L.world(regz, posz, kpz, f)
|
| 67 |
+
fl = ps.flags(X, up)
|
| 68 |
+
cams = sample["cams"][str(f)]
|
| 69 |
+
lab, P = L.frame_labels(ps, X, kp, cal, cams, HW, scale)
|
| 70 |
+
for c in cams:
|
| 71 |
+
uv, vis, z = lab[c]
|
| 72 |
+
mesh = uv[ps.mesh]
|
| 73 |
+
mesh = mesh[np.isfinite(mesh).all(1)]
|
| 74 |
+
lo, hi = mesh.min(0), mesh.max(0)
|
| 75 |
+
side = float((hi - lo).max() * BOX_MARGIN)
|
| 76 |
+
if not (50.0 < side < 3.0 * HW[1]): # a blown-up projection from a broken registration
|
| 77 |
+
print(f"BAD {cap[-6:]} f{f} c{c} side {side:.0f} px", flush=True)
|
| 78 |
+
continue
|
| 79 |
+
cx, cy = (lo + hi) / 2
|
| 80 |
+
box = np.array([cx - side / 2, cy - side / 2, cx + side / 2, cy + side / 2])
|
| 81 |
+
try:
|
| 82 |
+
if c not in imz:
|
| 83 |
+
imz[c] = zipfile.ZipFile(HttpFile(D + f"image/cam{c}.zip"))
|
| 84 |
+
im = Image.open(io.BytesIO(imz[c].read(f"cam{c}/{f:06d}.avif"))).convert("RGB")
|
| 85 |
+
im = im.crop(tuple(int(round(v)) for v in box)).resize((crop, crop), Image.BILINEAR)
|
| 86 |
+
im.save(os.path.join(d, f"{f:06d}_{c}.jpg"), quality=95)
|
| 87 |
+
except KeyError:
|
| 88 |
+
continue # entry absent from the zip (a few per person)
|
| 89 |
+
except Exception as e: # noqa: BLE001 - one bad entry must not kill the pool
|
| 90 |
+
tb = traceback.extract_tb(e.__traceback__)[-1]
|
| 91 |
+
print(
|
| 92 |
+
f"BAD {cap[-6:]} f{f} c{c} {type(e).__name__} at {tb.name}:{tb.lineno}",
|
| 93 |
+
flush=True,
|
| 94 |
+
)
|
| 95 |
+
continue
|
| 96 |
+
s = crop / side
|
| 97 |
+
rec["uv"].append(((uv - box[:2]) * s).astype(np.float32))
|
| 98 |
+
rec["vis"].append(vis)
|
| 99 |
+
rec["z"].append(z.astype(np.float32))
|
| 100 |
+
rec["box"].append(box.astype(np.float32))
|
| 101 |
+
rec["frame"].append(f)
|
| 102 |
+
rec["cam"].append(int(c))
|
| 103 |
+
rec["fx"].append(L.camera(cal[c], scale)[0][0, 0])
|
| 104 |
+
for k in ("lid_L", "lid_R", "lip_gap"):
|
| 105 |
+
rec[k].append(fl[k])
|
| 106 |
+
n_img += 1
|
| 107 |
+
np.savez_compressed(
|
| 108 |
+
done,
|
| 109 |
+
uv=np.stack(rec["uv"]),
|
| 110 |
+
vis=np.stack(rec["vis"]),
|
| 111 |
+
z=np.stack(rec["z"]),
|
| 112 |
+
box=np.stack(rec["box"]),
|
| 113 |
+
frame=np.array(rec["frame"]),
|
| 114 |
+
cam=np.array(rec["cam"]),
|
| 115 |
+
fx=np.array(rec["fx"], np.float32),
|
| 116 |
+
lid_L=np.array(rec["lid_L"], np.float32),
|
| 117 |
+
lid_R=np.array(rec["lid_R"], np.float32),
|
| 118 |
+
lip_gap=np.array(rec["lip_gap"], np.float32),
|
| 119 |
+
cls=ps.cls,
|
| 120 |
+
)
|
| 121 |
+
return cap, n_img
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def main():
|
| 125 |
+
"""Extract every capture of the sample list with a process pool."""
|
| 126 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 127 |
+
ap.add_argument("samples", help="sample list from sample_frames.py")
|
| 128 |
+
ap.add_argument("out", help="output root: <out>/<capture>/")
|
| 129 |
+
ap.add_argument("--tier", default="16TB", choices=sorted(TIERS), help="Ava-256 photo tier")
|
| 130 |
+
ap.add_argument("--crop", type=int, default=512, help="crop side in px")
|
| 131 |
+
ap.add_argument("--workers", type=int, default=16, help="capture processes")
|
| 132 |
+
ap.add_argument("--captures", help="file of capture ids to extract (default: all in the list)")
|
| 133 |
+
ap.add_argument("--points", default=L.DEFAULT_POINTS, help="point set on the Ava topology")
|
| 134 |
+
ap.add_argument(
|
| 135 |
+
"--topology", help="Ava template OBJ (default: $AVA_TOPOLOGY or a cached download)"
|
| 136 |
+
)
|
| 137 |
+
a = ap.parse_args()
|
| 138 |
+
with open(a.samples) as fh:
|
| 139 |
+
S = json.load(fh)
|
| 140 |
+
caps = list(S)
|
| 141 |
+
if a.captures:
|
| 142 |
+
with open(a.captures) as fh:
|
| 143 |
+
only = set(fh.read().split())
|
| 144 |
+
caps = [c for c in caps if c in only]
|
| 145 |
+
L.topology_path(a.topology) # download once before the pool starts
|
| 146 |
+
# processes, not threads: the depth-buffer loop is Python-bound
|
| 147 |
+
ctx = multiprocessing.get_context("fork") if hasattr(os, "fork") else None
|
| 148 |
+
with cf.ProcessPoolExecutor(a.workers, mp_context=ctx) as ex:
|
| 149 |
+
futs = {
|
| 150 |
+
ex.submit(one_capture, c, S[c], a.out, a.tier, a.crop, a.points, a.topology): c
|
| 151 |
+
for c in caps
|
| 152 |
+
}
|
| 153 |
+
for i, fu in enumerate(cf.as_completed(futs)):
|
| 154 |
+
try:
|
| 155 |
+
r = fu.result()[1]
|
| 156 |
+
except Exception as e: # noqa: BLE001 - a failed capture is redone on the next run
|
| 157 |
+
r = f"ERROR {type(e).__name__}: {str(e)[:80]} | {str(e.__cause__)[-300:]}"
|
| 158 |
+
print(f"{i + 1}/{len(caps)} {futs[fu][-6:]} {r}", flush=True)
|
| 159 |
+
print("EXTRACT DONE", flush=True)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
main()
|
prep/extract_masks.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Person masks for extracted crops: Ava-256's segmentation, cut with each crop's own box."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.extract_masks <data root> [--workers 16] [--threads 8]
|
| 4 |
+
# [--captures ids.txt]
|
| 5 |
+
# For every row of <root>/<capture>/labels.npz: reads cam<c>/<frame>.png from Ava-256's public S3
|
| 6 |
+
# (16TB/<capture>/decoder/segmentation_parts/cam<c>.zip, by HTTP range; one zip open per camera),
|
| 7 |
+
# crops it with the row's box (photo px, the same box as the jpg), nearest-resizes it to 512 px and
|
| 8 |
+
# writes <root>/<capture>/<frame>_<cam>_m.png: 255 = person (every class but background and chair;
|
| 9 |
+
# hair included), 0 = not. Resumable per capture (masks.done holds "<written> <missing>"). The masks
|
| 10 |
+
# feed the background swap of data.wild.Wild only: offline augmentation, never a runtime input.
|
| 11 |
+
import argparse
|
| 12 |
+
import concurrent.futures as cf
|
| 13 |
+
import io
|
| 14 |
+
import multiprocessing
|
| 15 |
+
import os
|
| 16 |
+
import zipfile
|
| 17 |
+
from collections import defaultdict
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
from PIL import Image
|
| 21 |
+
|
| 22 |
+
from facepoints.prep.http_zip import HttpFile
|
| 23 |
+
|
| 24 |
+
BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/16TB"
|
| 25 |
+
CROP = 512
|
| 26 |
+
BACKGROUND, CHAIR = 0, 3 # segmentation classes that are not the person
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def one_capture(root, cap, threads):
|
| 30 |
+
"""Write the masks of one capture; return (capture, summary)."""
|
| 31 |
+
d = os.path.join(root, cap)
|
| 32 |
+
done = os.path.join(d, "masks.done")
|
| 33 |
+
if os.path.exists(done):
|
| 34 |
+
return cap, "skip"
|
| 35 |
+
z = np.load(os.path.join(d, "labels.npz"))
|
| 36 |
+
by_cam = defaultdict(list)
|
| 37 |
+
for f, c, b in zip(z["frame"], z["cam"], z["box"], strict=True):
|
| 38 |
+
by_cam[int(c)].append((int(f), b))
|
| 39 |
+
|
| 40 |
+
def cam(c, rows): # one thread per camera: each read is a small range request, latency-bound
|
| 41 |
+
n = miss = 0
|
| 42 |
+
try:
|
| 43 |
+
sz = zipfile.ZipFile(HttpFile(f"{BASE}/{cap}/decoder/segmentation_parts/cam{c}.zip"))
|
| 44 |
+
except Exception: # noqa: BLE001 - a camera without masks: its crops just never swap
|
| 45 |
+
return 0, len(rows)
|
| 46 |
+
for f, b in rows:
|
| 47 |
+
out = os.path.join(d, f"{f:06d}_{c}_m.png")
|
| 48 |
+
if os.path.exists(out):
|
| 49 |
+
n += 1
|
| 50 |
+
continue
|
| 51 |
+
try:
|
| 52 |
+
seg = Image.open(io.BytesIO(sz.read(f"cam{c}/{f:06d}.png")))
|
| 53 |
+
except KeyError:
|
| 54 |
+
miss += 1
|
| 55 |
+
continue
|
| 56 |
+
a = np.array(
|
| 57 |
+
seg.crop(tuple(int(round(v)) for v in b)).resize((CROP, CROP), Image.NEAREST)
|
| 58 |
+
)
|
| 59 |
+
person = (a != BACKGROUND) & (a != CHAIR)
|
| 60 |
+
Image.fromarray((person * 255).astype(np.uint8)).save(out, optimize=True)
|
| 61 |
+
n += 1
|
| 62 |
+
return n, miss
|
| 63 |
+
|
| 64 |
+
with cf.ThreadPoolExecutor(threads) as ex:
|
| 65 |
+
r = list(ex.map(lambda kv: cam(*kv), by_cam.items()))
|
| 66 |
+
n, miss = sum(a for a, _ in r), sum(b for _, b in r)
|
| 67 |
+
with open(done, "w") as fh:
|
| 68 |
+
fh.write(f"{n} {miss}\n")
|
| 69 |
+
return cap, f"{n} masks, {miss} missing"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def main():
|
| 73 |
+
"""Write the masks of every extracted capture with a process pool."""
|
| 74 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 75 |
+
ap.add_argument("root", help="data root written by extract.py")
|
| 76 |
+
ap.add_argument("--workers", type=int, default=16, help="capture processes")
|
| 77 |
+
ap.add_argument("--threads", type=int, default=8, help="camera threads per capture process")
|
| 78 |
+
ap.add_argument("--captures", help="file of capture ids (default: every extracted capture)")
|
| 79 |
+
a = ap.parse_args()
|
| 80 |
+
caps = sorted(
|
| 81 |
+
c for c in os.listdir(a.root) if os.path.exists(os.path.join(a.root, c, "labels.npz"))
|
| 82 |
+
)
|
| 83 |
+
if a.captures:
|
| 84 |
+
with open(a.captures) as fh:
|
| 85 |
+
only = set(fh.read().split())
|
| 86 |
+
caps = [c for c in caps if c in only]
|
| 87 |
+
ctx = multiprocessing.get_context("fork") if hasattr(os, "fork") else None
|
| 88 |
+
with cf.ProcessPoolExecutor(a.workers, mp_context=ctx) as ex:
|
| 89 |
+
futs = {ex.submit(one_capture, a.root, c, a.threads): c for c in caps}
|
| 90 |
+
for i, fu in enumerate(cf.as_completed(futs)):
|
| 91 |
+
try:
|
| 92 |
+
r = fu.result()[1]
|
| 93 |
+
except Exception as e: # noqa: BLE001 - a failed capture is redone on the next run
|
| 94 |
+
r = f"ERROR {type(e).__name__}: {str(e)[:120]}"
|
| 95 |
+
print(f"{i + 1}/{len(caps)} {futs[fu][-6:]} {r}", flush=True)
|
| 96 |
+
print("MASKS DONE", flush=True)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
if __name__ == "__main__":
|
| 100 |
+
main()
|
prep/http_zip.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Read single entries of a remote zip (public HTTP, e.g. Ava's S3) by Range requests."""
|
| 2 |
+
|
| 3 |
+
import io
|
| 4 |
+
import urllib.request
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class HttpFile(io.RawIOBase):
|
| 8 |
+
"""Seekable read-only file over HTTP Range requests, with a one-block read-ahead cache."""
|
| 9 |
+
|
| 10 |
+
def __init__(self, url, block=1 << 17):
|
| 11 |
+
self.url, self.pos, self.block = url, 0, block
|
| 12 |
+
with urllib.request.urlopen(urllib.request.Request(url, method="HEAD"), timeout=60) as r:
|
| 13 |
+
self.size = int(r.headers["Content-Length"])
|
| 14 |
+
self.cache = (0, b"")
|
| 15 |
+
self.nreq = self.nbytes = 0
|
| 16 |
+
|
| 17 |
+
def seekable(self):
|
| 18 |
+
return True
|
| 19 |
+
|
| 20 |
+
def readable(self):
|
| 21 |
+
return True
|
| 22 |
+
|
| 23 |
+
def tell(self):
|
| 24 |
+
return self.pos
|
| 25 |
+
|
| 26 |
+
def seek(self, off, whence=0):
|
| 27 |
+
self.pos = off if whence == 0 else self.pos + off if whence == 1 else self.size + off
|
| 28 |
+
return self.pos
|
| 29 |
+
|
| 30 |
+
def _fetch(self, a, b):
|
| 31 |
+
req = urllib.request.Request(self.url, headers={"Range": f"bytes={a}-{b - 1}"})
|
| 32 |
+
for attempt in range(4):
|
| 33 |
+
try:
|
| 34 |
+
with urllib.request.urlopen(req, timeout=120) as r:
|
| 35 |
+
data = r.read()
|
| 36 |
+
break
|
| 37 |
+
except OSError:
|
| 38 |
+
if attempt == 3:
|
| 39 |
+
raise
|
| 40 |
+
self.nreq += 1
|
| 41 |
+
self.nbytes += len(data)
|
| 42 |
+
return data
|
| 43 |
+
|
| 44 |
+
def read(self, n=-1):
|
| 45 |
+
n = self.size - self.pos if n is None or n < 0 else min(n, self.size - self.pos)
|
| 46 |
+
if n <= 0:
|
| 47 |
+
return b""
|
| 48 |
+
c0, cb = self.cache
|
| 49 |
+
if not (c0 <= self.pos and self.pos + n <= c0 + len(cb)):
|
| 50 |
+
cb = self._fetch(self.pos, min(self.size, self.pos + max(n, self.block)))
|
| 51 |
+
self.cache = c0, cb = self.pos, cb
|
| 52 |
+
out = cb[self.pos - c0 : self.pos - c0 + n]
|
| 53 |
+
self.pos += len(out)
|
| 54 |
+
return out
|
| 55 |
+
|
| 56 |
+
def readinto(self, buf):
|
| 57 |
+
d = self.read(len(buf))
|
| 58 |
+
buf[: len(d)] = d
|
| 59 |
+
return len(d)
|
prep/iris_labels.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Iris-centre labels (rows 618 and 619) from Ava-256's 3D iris-ring keypoints, written in place."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.iris_labels <data root> [--workers 8] [--captures file]
|
| 4 |
+
# Ava-256's keypoints 238-273 are two 18-point iris rings (left = 238-246 + 256-264, right =
|
| 5 |
+
# 247-255 + 265-273). The centre of a ring with at least MIN_RING keypoints is projected with the
|
| 6 |
+
# capture's calibration exactly as labels.py projects every point, then into each image's stored
|
| 7 |
+
# crop box. It counts as visible when that eye's lid gap is over 2 mm, at least 75 % of that eye's
|
| 8 |
+
# lid points are visible in the image, and the centre lies at least 1.5 crop px inside the image's
|
| 9 |
+
# lid-opening polygon (a lowered gaze hides it). Writes uv / z / vis of rows 618 (L) and 619 (R) in
|
| 10 |
+
# <capture>/labels.npz and drops iris.done, so a re-run is a no-op.
|
| 11 |
+
import argparse
|
| 12 |
+
import concurrent.futures as cf
|
| 13 |
+
import io
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
+
import urllib.request
|
| 17 |
+
import zipfile
|
| 18 |
+
|
| 19 |
+
import cv2
|
| 20 |
+
import numpy as np
|
| 21 |
+
|
| 22 |
+
from facepoints.prep import labels as L
|
| 23 |
+
from facepoints.prep.http_zip import HttpFile
|
| 24 |
+
|
| 25 |
+
BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/16TB"
|
| 26 |
+
RING = {
|
| 27 |
+
"L": list(range(238, 247)) + list(range(256, 265)),
|
| 28 |
+
"R": list(range(247, 256)) + list(range(265, 274)),
|
| 29 |
+
}
|
| 30 |
+
ROW = {"L": 618, "R": 619}
|
| 31 |
+
MIN_RING = 14 # ring keypoints needed for a centre
|
| 32 |
+
CROP, SCALE = 512, 0.5 # extract.py's crop side and the 16TB photo scale
|
| 33 |
+
LID_OPEN_MM, LID_VISIBLE, INSIDE_PX = 2.0, 0.75, 1.5
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def one_capture(root, cap, points):
|
| 37 |
+
"""Label one capture's iris rows; return (capture, summary)."""
|
| 38 |
+
d = os.path.join(root, cap)
|
| 39 |
+
mark = os.path.join(d, "iris.done")
|
| 40 |
+
if os.path.exists(mark) or not os.path.exists(os.path.join(d, "labels.npz")):
|
| 41 |
+
return cap, "skip"
|
| 42 |
+
z = dict(np.load(os.path.join(d, "labels.npz")))
|
| 43 |
+
cls = z["cls"].astype(str)
|
| 44 |
+
reg = np.load(points)["region"].astype(str)
|
| 45 |
+
assert (cls[[618, 619]] == "iris").all() and list(reg[[618, 619]]) == ["L", "R"]
|
| 46 |
+
lidpts = {e: np.flatnonzero((cls == "lid") & np.char.startswith(reg, e + "_")) for e in "LR"}
|
| 47 |
+
lidu = {
|
| 48 |
+
e: np.flatnonzero((cls == "lid") & (reg == f"{e}_upper")) for e in "LR"
|
| 49 |
+
} # corner-ordered
|
| 50 |
+
lidl = {e: np.flatnonzero((cls == "lid") & (reg == f"{e}_lower")) for e in "LR"}
|
| 51 |
+
D = f"{BASE}/{cap}/decoder/"
|
| 52 |
+
with urllib.request.urlopen(D + "camera_calibration.json", timeout=60) as r:
|
| 53 |
+
cal = {e["cameraId"]: e for e in json.load(r)["KRT"]}
|
| 54 |
+
kpz = zipfile.ZipFile(HttpFile(D + "keypoints_3d/keypoints_3d.zip"))
|
| 55 |
+
centre = {}
|
| 56 |
+
for f in np.unique(z["frame"]):
|
| 57 |
+
kp = np.load(io.BytesIO(kpz.read(f"{int(f):06d}.npy")))
|
| 58 |
+
ok = kp[:, 5] >= L.MIN_INL
|
| 59 |
+
ids = kp[:, 0].astype(int)
|
| 60 |
+
centre[int(f)] = {}
|
| 61 |
+
for e, ring in RING.items():
|
| 62 |
+
m = ok & np.isin(ids, ring)
|
| 63 |
+
centre[int(f)][e] = kp[m, 1:4].mean(0) if m.sum() >= MIN_RING else None
|
| 64 |
+
n_vis = 0
|
| 65 |
+
for i in range(len(z["frame"])):
|
| 66 |
+
f, c = int(z["frame"][i]), str(int(z["cam"][i]))
|
| 67 |
+
cam = L.camera(cal[c] if c in cal else cal[int(c)], SCALE)
|
| 68 |
+
side = float(z["box"][i][2] - z["box"][i][0])
|
| 69 |
+
for e in "LR":
|
| 70 |
+
j = ROW[e]
|
| 71 |
+
P = centre[f][e]
|
| 72 |
+
z["uv"][i, j], z["z"][i, j], z["vis"][i, j] = np.nan, np.nan, False
|
| 73 |
+
if P is None:
|
| 74 |
+
continue
|
| 75 |
+
uv, zz = L.project(P[None], cam)
|
| 76 |
+
z["uv"][i, j] = (uv[0] - z["box"][i][:2]) * (CROP / side)
|
| 77 |
+
z["z"][i, j] = zz[0]
|
| 78 |
+
v = z[f"lid_{e}"][i] > LID_OPEN_MM and z["vis"][i, lidpts[e]].mean() >= LID_VISIBLE
|
| 79 |
+
if v: # the centre must also lie inside this image's lid opening
|
| 80 |
+
up, lo = z["uv"][i, lidu[e]], z["uv"][i, lidl[e]]
|
| 81 |
+
if np.linalg.norm(lo[0] - up[0]) > np.linalg.norm(lo[-1] - up[0]):
|
| 82 |
+
lo = lo[::-1]
|
| 83 |
+
poly = np.concatenate([up, lo[::-1]]).astype(np.float32)
|
| 84 |
+
c2 = z["uv"][i, j].astype(np.float32)
|
| 85 |
+
inside = cv2.pointPolygonTest(poly, (float(c2[0]), float(c2[1])), True)
|
| 86 |
+
v = np.isfinite(poly).all() and inside >= INSIDE_PX
|
| 87 |
+
z["vis"][i, j] = bool(v)
|
| 88 |
+
n_vis += int(v)
|
| 89 |
+
np.savez_compressed(os.path.join(d, "labels.npz"), **z)
|
| 90 |
+
with open(mark, "w") as fh:
|
| 91 |
+
fh.write(f"{n_vis} visible iris labels of {2 * len(z['frame'])}\n")
|
| 92 |
+
return cap, f"{len(z['frame'])} images, {n_vis} visible iris labels"
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def main():
|
| 96 |
+
"""Label the iris rows of every extracted capture with a process pool."""
|
| 97 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 98 |
+
ap.add_argument("root", help="data root written by extract.py")
|
| 99 |
+
ap.add_argument("--workers", type=int, default=8)
|
| 100 |
+
ap.add_argument("--captures", help="file of capture ids (default: every extracted capture)")
|
| 101 |
+
ap.add_argument("--points", default=L.DEFAULT_POINTS, help="point set (for the lid regions)")
|
| 102 |
+
a = ap.parse_args()
|
| 103 |
+
caps = sorted(
|
| 104 |
+
c for c in os.listdir(a.root) if os.path.exists(os.path.join(a.root, c, "labels.npz"))
|
| 105 |
+
)
|
| 106 |
+
if a.captures:
|
| 107 |
+
with open(a.captures) as fh:
|
| 108 |
+
only = set(fh.read().split())
|
| 109 |
+
caps = [c for c in caps if c in only]
|
| 110 |
+
with cf.ProcessPoolExecutor(a.workers) as ex:
|
| 111 |
+
futs = {ex.submit(one_capture, a.root, c, a.points): c for c in caps}
|
| 112 |
+
for k, fu in enumerate(cf.as_completed(futs)):
|
| 113 |
+
try:
|
| 114 |
+
c, msg = fu.result()
|
| 115 |
+
if k % 25 == 0 or "skip" not in msg:
|
| 116 |
+
print(f"{k + 1}/{len(caps)} {c[-6:]} {msg}", flush=True)
|
| 117 |
+
except Exception as e: # noqa: BLE001 - a failed capture is redone on the next run
|
| 118 |
+
print(f"ERROR {futs[fu][-6:]} {type(e).__name__}: {e}", flush=True)
|
| 119 |
+
print("IRIS LABELS DONE", flush=True)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
if __name__ == "__main__":
|
| 123 |
+
main()
|
prep/labels.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Labels for one Ava-256 frame: point set 3D positions, per-camera projections, visibility."""
|
| 2 |
+
|
| 3 |
+
# Mesh points (classes lmk, skin, lid) are vertices of Meta's registration mesh (points_ava.npz
|
| 4 |
+
# `src`), taken to the capture world through the head pose; teeth points are Meta's 3D keypoints
|
| 5 |
+
# (incisal-edge ids with at least MIN_INL inliers, NaN when absent); iris rows stay NaN here.
|
| 6 |
+
# Projection uses the capture's calibration with lens distortion, scaled to the photo tier.
|
| 7 |
+
# Visibility comes from the registration mesh's own depth buffer, without the membrane triangles
|
| 8 |
+
# that bridge the eyes and the mouth. Per-frame flags: the signed lid gap per eye and the
|
| 9 |
+
# inner-lip gap, in mm. The Ava template OBJ is read from $AVA_TOPOLOGY, or downloaded once from
|
| 10 |
+
# the ava-256 repository into ~/.cache/facepoints/.
|
| 11 |
+
import io
|
| 12 |
+
import os
|
| 13 |
+
import urllib.request
|
| 14 |
+
|
| 15 |
+
import cv2
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
MIN_INL = 4 # keypoint inliers for a teeth point to count
|
| 19 |
+
Z_TOL = 3.0 # mm a point may sit behind the depth buffer and still be visible
|
| 20 |
+
DEFAULT_POINTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "points_ava.npz")
|
| 21 |
+
TOPOLOGY_URL = (
|
| 22 |
+
"https://raw.githubusercontent.com/facebookresearch/ava-256/main/assets/face_topology.obj"
|
| 23 |
+
)
|
| 24 |
+
TOPOLOGY_CACHE = os.path.join(
|
| 25 |
+
os.path.expanduser("~"), ".cache", "facepoints", "face_topology.obj"
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def topology_path(path=None):
|
| 30 |
+
"""Return the Ava template OBJ path: the argument, $AVA_TOPOLOGY, or a cached download."""
|
| 31 |
+
path = path or os.environ.get("AVA_TOPOLOGY") or TOPOLOGY_CACHE
|
| 32 |
+
if not os.path.exists(path):
|
| 33 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 34 |
+
urllib.request.urlretrieve(TOPOLOGY_URL, path)
|
| 35 |
+
return path
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def read_ply(buf):
|
| 39 |
+
"""Return the (N, 3) float64 vertices of a binary PLY from its first three float properties."""
|
| 40 |
+
head, rest = buf.split(b"end_header\n", 1)
|
| 41 |
+
lines = head.decode("ascii", "ignore").splitlines()
|
| 42 |
+
n = next(int(ln.split()[2]) for ln in lines if ln.startswith("element vertex"))
|
| 43 |
+
sz = {"float": 4, "float32": 4, "double": 8, "uchar": 1, "int": 4}
|
| 44 |
+
stride = sum(sz[ln.split()[1]] for ln in lines if ln.startswith("property"))
|
| 45 |
+
arr = np.frombuffer(rest[: n * stride], dtype=np.uint8).reshape(n, stride)
|
| 46 |
+
return arr[:, :12].copy().view(np.float32).reshape(n, 3).astype(np.float64)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def topology(path):
|
| 50 |
+
"""Return (vertices, faces, per-vertex UV, longest UV edge per face) of the Ava template OBJ."""
|
| 51 |
+
V, VT, F, FT = [], [], [], []
|
| 52 |
+
with open(path) as fh:
|
| 53 |
+
for ln in fh:
|
| 54 |
+
if ln.startswith("v "):
|
| 55 |
+
V.append([float(x) for x in ln.split()[1:4]])
|
| 56 |
+
elif ln.startswith("vt "):
|
| 57 |
+
VT.append([float(x) for x in ln.split()[1:3]])
|
| 58 |
+
elif ln.startswith("f "):
|
| 59 |
+
F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]])
|
| 60 |
+
FT.append([int(t.split("/")[1]) - 1 for t in ln.split()[1:4]])
|
| 61 |
+
V, VT, F, FT = np.array(V), np.array(VT), np.array(F), np.array(FT)
|
| 62 |
+
el = np.linalg.norm(VT[FT] - VT[np.roll(FT, 1, axis=1)], axis=-1).max(1)
|
| 63 |
+
uv = np.full((len(V), 2), np.nan)
|
| 64 |
+
for f_, ft in zip(F, FT, strict=True):
|
| 65 |
+
uv[f_] = VT[ft]
|
| 66 |
+
return V, F, uv, el
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def membranes(V, F, el, centres, radius):
|
| 70 |
+
"""Return a face mask of the triangles bridging an opening (near a centre, long UV edge)."""
|
| 71 |
+
# the UV-edge test is relative to the local median because a global rule also catches UV seams
|
| 72 |
+
out = np.zeros(len(F), bool)
|
| 73 |
+
for c in centres:
|
| 74 |
+
near = np.all(np.linalg.norm(V[F] - c, axis=-1) < radius, axis=1)
|
| 75 |
+
out |= near & (el > 3 * np.median(el[near]))
|
| 76 |
+
return out
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def lip_contours(V, F, mouth_bridge, uv, eye_c):
|
| 80 |
+
"""Return (upper ids, lower ids) of the inner-lip ring along the mouth (upper = nearer eyes)."""
|
| 81 |
+
low = np.unique(F[mouth_bridge])
|
| 82 |
+
cen = uv[low].mean(0)
|
| 83 |
+
ax = uv[low][np.argmax(uv[low][:, 0])] - uv[low][np.argmin(uv[low][:, 0])]
|
| 84 |
+
ax /= np.linalg.norm(ax)
|
| 85 |
+
nrm = np.array([-ax[1], ax[0]])
|
| 86 |
+
s_, al = (uv[low] - cen) @ nrm, (uv[low] - cen) @ ax
|
| 87 |
+
a, b = low[s_ > 0][np.argsort(al[s_ > 0])], low[s_ <= 0][np.argsort(al[s_ <= 0])]
|
| 88 |
+
a_first = np.linalg.norm(V[a].mean(0) - eye_c) < np.linalg.norm(V[b].mean(0) - eye_c)
|
| 89 |
+
return (a, b) if a_first else (b, a)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
class PointSet:
|
| 93 |
+
"""The label point set on the Ava topology and its evaluation on a registration + keypoints."""
|
| 94 |
+
|
| 95 |
+
def __init__(self, points=DEFAULT_POINTS, topology_obj=None):
|
| 96 |
+
z = np.load(points)
|
| 97 |
+
self.cls, self.src = z["cls"].astype(str), z["src"].astype(int)
|
| 98 |
+
self.region = z["region"].astype(str)
|
| 99 |
+
self.n = len(self.cls)
|
| 100 |
+
self.mesh = np.isin(self.cls, ["lmk", "skin", "lid"])
|
| 101 |
+
self.teeth = np.isin(self.cls, ["teeth_u", "teeth_l"])
|
| 102 |
+
self.V0, self.F, uv0, el = topology(topology_path(topology_obj))
|
| 103 |
+
lid = self.cls == "lid"
|
| 104 |
+
eyes = [self.V0[self.src[lid & (self.region == f"{e}_upper")]].mean(0) for e in "LR"]
|
| 105 |
+
mouth = self.V0[self.src[(self.cls == "skin") & (self.region == "perioral")]].mean(0)
|
| 106 |
+
eye_b = membranes(self.V0, self.F, el, eyes, 22.0)
|
| 107 |
+
mouth_b = membranes(self.V0, self.F, el, [mouth], 45.0)
|
| 108 |
+
self.Fz = self.F[~(eye_b | mouth_b)] # the triangles the depth buffer uses
|
| 109 |
+
self.lips = lip_contours(self.V0, self.F, mouth_b, uv0, np.mean(eyes, axis=0))
|
| 110 |
+
self.lid_sets = {
|
| 111 |
+
e: (
|
| 112 |
+
self.src[lid & (self.region == f"{e}_upper")],
|
| 113 |
+
self.src[lid & (self.region == f"{e}_lower")],
|
| 114 |
+
)
|
| 115 |
+
for e in "LR"
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
def points3d(self, X, kp):
|
| 119 |
+
"""Return (n, 3) positions in capture world: mesh points from X, teeth from kp, else NaN."""
|
| 120 |
+
P = np.full((self.n, 3), np.nan)
|
| 121 |
+
P[self.mesh] = X[self.src[self.mesh]]
|
| 122 |
+
ok = kp[:, 5] >= MIN_INL
|
| 123 |
+
ids = kp[:, 0].astype(int)
|
| 124 |
+
for i in np.flatnonzero(self.teeth):
|
| 125 |
+
m = ok & (ids == self.src[i])
|
| 126 |
+
if m.any():
|
| 127 |
+
P[i] = kp[m][0, 1:4]
|
| 128 |
+
return P
|
| 129 |
+
|
| 130 |
+
def flags(self, X, up):
|
| 131 |
+
"""Return {lid_L, lid_R: signed lid gap along `up` (mm), lip_gap: inner-lip gap (mm)}."""
|
| 132 |
+
out = {}
|
| 133 |
+
for e, (u, lo) in self.lid_sets.items():
|
| 134 |
+
g = []
|
| 135 |
+
for i in u[1:-1]:
|
| 136 |
+
d = X[lo] - X[i]
|
| 137 |
+
j = np.argmin(np.linalg.norm(d, axis=1))
|
| 138 |
+
g.append(-(d[j] @ up[e]))
|
| 139 |
+
out[f"lid_{e}"] = float(np.mean(g))
|
| 140 |
+
u, lo = self.lips
|
| 141 |
+
out["lip_gap"] = float(
|
| 142 |
+
np.mean([np.linalg.norm(X[lo] - X[i], axis=1).min() for i in u[1:-1]])
|
| 143 |
+
)
|
| 144 |
+
return out
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def camera(e, scale):
|
| 148 |
+
"""Return (K, RT, distortion, rvec) of an Ava KRT entry for a photo at `scale` x native size."""
|
| 149 |
+
K = np.array(e["K"], np.float64).T.copy()
|
| 150 |
+
K[:2] *= scale
|
| 151 |
+
RT = np.array(e["T"], np.float64)[:4, :3].T
|
| 152 |
+
dist = np.array(list(e["distortion"]) + [0.0] * 5, np.float64)[:5]
|
| 153 |
+
return K, RT, dist, cv2.Rodrigues(RT[:3, :3])[0]
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def project(P, cam):
|
| 157 |
+
"""Return ((n, 2) pixels, (n,) depth) of points P; NaN rows pass through."""
|
| 158 |
+
K, RT, dist, rvec = cam
|
| 159 |
+
uv = np.full((len(P), 2), np.nan)
|
| 160 |
+
z = np.full(len(P), np.nan)
|
| 161 |
+
ok = np.isfinite(P).all(1)
|
| 162 |
+
if ok.any():
|
| 163 |
+
u, _ = cv2.projectPoints(P[ok], rvec, RT[:3, 3], K, dist)
|
| 164 |
+
uv[ok] = u[:, 0]
|
| 165 |
+
z[ok] = (P[ok] @ RT[:3, :3].T + RT[:3, 3])[:, 2]
|
| 166 |
+
return uv, z
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def zbuffer(X, Fz, cam, hw):
|
| 170 |
+
"""Return the painter's depth buffer (H, W) of the mesh X over the triangles Fz."""
|
| 171 |
+
uv, z = project(X, cam)
|
| 172 |
+
H, W = hw
|
| 173 |
+
zb = np.full((H, W), np.inf, np.float32)
|
| 174 |
+
tz = z[Fz].mean(1)
|
| 175 |
+
for t in np.argsort(-tz):
|
| 176 |
+
pts = np.round(uv[Fz[t]]).astype(np.int32)
|
| 177 |
+
if (pts < 0).any() or (pts[:, 0] >= W).any() or (pts[:, 1] >= H).any():
|
| 178 |
+
continue
|
| 179 |
+
cv2.fillConvexPoly(zb, pts, float(tz[t]))
|
| 180 |
+
return zb
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def visible(uv, z, zb):
|
| 184 |
+
"""Return a mask of the points not hidden by the depth buffer by more than Z_TOL mm."""
|
| 185 |
+
H, W = zb.shape
|
| 186 |
+
ok = np.isfinite(uv).all(1)
|
| 187 |
+
v = np.zeros(len(uv), bool)
|
| 188 |
+
x = np.clip(np.round(uv[ok, 0]).astype(int), 0, W - 1)
|
| 189 |
+
y = np.clip(np.round(uv[ok, 1]).astype(int), 0, H - 1)
|
| 190 |
+
v[ok] = zb[y, x] >= z[ok] - Z_TOL
|
| 191 |
+
return v
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def frame_labels(ps, X, kp, cal, cam_ids, hw, scale):
|
| 195 |
+
"""Return ({cam: (uv (n, 2), visible (n,), depth (n,))}, P (n, 3)) for one frame."""
|
| 196 |
+
P = ps.points3d(X, kp)
|
| 197 |
+
out = {}
|
| 198 |
+
for c in cam_ids:
|
| 199 |
+
cam = camera(cal[c], scale)
|
| 200 |
+
uv, z = project(P, cam)
|
| 201 |
+
zb = zbuffer(X, ps.Fz, cam, hw)
|
| 202 |
+
out[c] = (uv, visible(uv, z, zb), z)
|
| 203 |
+
return out, P
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def world(regz, posz, kpz, f):
|
| 207 |
+
"""Return (registration in capture world, head pose, keypoints) of frame f from the zips."""
|
| 208 |
+
X = read_ply(regz.read(f"{f:06d}.ply"))
|
| 209 |
+
M = np.loadtxt(io.BytesIO(posz.read(f"{f:06d}.txt")))
|
| 210 |
+
kp = np.load(io.BytesIO(kpz.read(f"{f:06d}.npy")))
|
| 211 |
+
return X @ M[:3, :3].T + M[:3, 3], M, kp
|
prep/merge_data.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Merge an extra extraction (for example additional closed-eye frames) into the training data."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.merge_data <data root> <add root> <captures>
|
| 4 |
+
# [--tag closed] [--split split.json]
|
| 5 |
+
# <captures> is a split key (train / val / test) or a file of capture ids. For each listed capture
|
| 6 |
+
# present in <add root>: its labels.npz arrays are appended to the existing ones (the point classes
|
| 7 |
+
# must match), its jpgs are moved in (their names must be new), and the marker file
|
| 8 |
+
# <data root>/<capture>/merged_<tag>.txt makes a re-run a no-op.
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import shutil
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
DEFAULT_SPLIT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "split.json")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def merge_capture(root, add, cap, tag):
|
| 20 |
+
"""Merge one capture; return the number of images moved, or None when skipped."""
|
| 21 |
+
a, b = os.path.join(root, cap), os.path.join(add, cap)
|
| 22 |
+
mark = os.path.join(a, f"merged_{tag}.txt")
|
| 23 |
+
if not os.path.exists(os.path.join(b, "labels.npz")) or os.path.exists(mark):
|
| 24 |
+
return None
|
| 25 |
+
za, zb = np.load(os.path.join(a, "labels.npz")), np.load(os.path.join(b, "labels.npz"))
|
| 26 |
+
assert (za["cls"] == zb["cls"]).all(), cap
|
| 27 |
+
names_a = {f"{int(f):06d}_{int(c)}.jpg" for f, c in zip(za["frame"], za["cam"], strict=True)}
|
| 28 |
+
names_b = [f"{int(f):06d}_{int(c)}.jpg" for f, c in zip(zb["frame"], zb["cam"], strict=True)]
|
| 29 |
+
assert not names_a & set(names_b), f"{cap}: duplicate images"
|
| 30 |
+
out = {k: (za[k] if k == "cls" else np.concatenate([za[k], zb[k]])) for k in za.files}
|
| 31 |
+
for nm in names_b:
|
| 32 |
+
shutil.move(os.path.join(b, nm), os.path.join(a, nm))
|
| 33 |
+
np.savez_compressed(os.path.join(a, "labels.npz"), **out)
|
| 34 |
+
with open(mark, "w") as fh:
|
| 35 |
+
fh.write(f"{len(names_b)} images from {add}\n")
|
| 36 |
+
return len(names_b)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def main():
|
| 40 |
+
"""Merge the listed captures and print a summary."""
|
| 41 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 42 |
+
ap.add_argument("root", help="data root to merge into")
|
| 43 |
+
ap.add_argument("add", help="data root of the extra extraction")
|
| 44 |
+
ap.add_argument("captures", help="split key or file of capture ids")
|
| 45 |
+
ap.add_argument("--tag", default="extra", help="name of the merge marker")
|
| 46 |
+
ap.add_argument("--split", default=DEFAULT_SPLIT, help="identity split file")
|
| 47 |
+
a = ap.parse_args()
|
| 48 |
+
if os.path.exists(a.captures):
|
| 49 |
+
with open(a.captures) as fh:
|
| 50 |
+
caps = fh.read().split()
|
| 51 |
+
else:
|
| 52 |
+
with open(a.split) as fh:
|
| 53 |
+
caps = json.load(fh)[a.captures]
|
| 54 |
+
n_cap = n_img = n_skip = 0
|
| 55 |
+
for cap in caps:
|
| 56 |
+
n = merge_capture(a.root, a.add, cap, a.tag)
|
| 57 |
+
if n is None:
|
| 58 |
+
n_skip += 1
|
| 59 |
+
else:
|
| 60 |
+
n_cap, n_img = n_cap + 1, n_img + n
|
| 61 |
+
print(f"MERGED {n_cap} captures, {n_img} images (skipped {n_skip}) tag {a.tag}", flush=True)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
main()
|
prep/points_ava.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b0aa818ee773fdd77d737f0d59951cb01ce8769f8f6ff9aae1fa3ea2acb19532
|
| 3 |
+
size 19414
|
prep/sample_frames.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The training sample list: per Ava-256 capture, the most diverse frames and random cameras."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.sample_frames <out.json> [--expressions 50]
|
| 4 |
+
# [--cameras 20] [--seed 0] [--captures captures.txt]
|
| 5 |
+
# Expression diversity is measured on Ava-256's 3D keypoints in the head frame (ids present in at
|
| 6 |
+
# least 90 % of the capture's frames; an absent id takes its per-capture median; xyz concatenated)
|
| 7 |
+
# by farthest-point sampling from the capture's first frame. The cameras are a seeded random subset
|
| 8 |
+
# of the capture's 80 per selected frame (50 x 20 = 1,000 images per capture by default). The
|
| 9 |
+
# keypoint and head-pose zips are read whole per capture (about 50 MB). Output: {capture: {frames:
|
| 10 |
+
# [...], cams: {frame: [camera ids]}, n_frames, n_ids_used}}, or {skipped: reason}; resumable per
|
| 11 |
+
# capture through <out>.part/.
|
| 12 |
+
import argparse
|
| 13 |
+
import collections
|
| 14 |
+
import concurrent.futures as cf
|
| 15 |
+
import io
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
import random
|
| 19 |
+
import urllib.request
|
| 20 |
+
import zipfile
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
|
| 24 |
+
BASE = "https://fb-baas-f32eacb9-8abb-11eb-b2b8-4857dd089e15.s3.amazonaws.com/ava-256/4TB"
|
| 25 |
+
DEFAULT_CAPTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "captures.txt")
|
| 26 |
+
MIN_SHARE = 0.9 # a keypoint id must be present in this share of the frames to be used
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def fetch_zip(url):
|
| 30 |
+
"""Return a whole remote zip as a ZipFile."""
|
| 31 |
+
with urllib.request.urlopen(url, timeout=300) as r:
|
| 32 |
+
return zipfile.ZipFile(io.BytesIO(r.read()))
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def one_capture(cap, part, n_expr, n_cams, seed):
|
| 36 |
+
"""Return (capture, its sample dict), reading or writing <part>/<capture>.json."""
|
| 37 |
+
pf = os.path.join(part, cap + ".json")
|
| 38 |
+
if os.path.exists(pf):
|
| 39 |
+
with open(pf) as fh:
|
| 40 |
+
return cap, json.load(fh)
|
| 41 |
+
D = f"{BASE}/{cap}/decoder/"
|
| 42 |
+
try:
|
| 43 |
+
zk = fetch_zip(D + "keypoints_3d/keypoints_3d.zip")
|
| 44 |
+
zp = fetch_zip(D + "head_pose/head_pose.zip")
|
| 45 |
+
cal = json.load(urllib.request.urlopen(D + "camera_calibration.json", timeout=60))["KRT"]
|
| 46 |
+
except Exception as e: # noqa: BLE001 - a capture that cannot be read is skipped, not fatal
|
| 47 |
+
return cap, {"skipped": str(e)[:80]}
|
| 48 |
+
cams = [e["cameraId"] for e in cal]
|
| 49 |
+
frames = sorted(
|
| 50 |
+
int(n.split("/")[-1].split(".")[0]) for n in zk.namelist() if n.endswith(".npy")
|
| 51 |
+
)
|
| 52 |
+
poses = {int(n.split("/")[-1].split(".")[0]) for n in zp.namelist() if n.endswith(".txt")}
|
| 53 |
+
frames = [f for f in frames if f in poses]
|
| 54 |
+
K = {}
|
| 55 |
+
for f in frames:
|
| 56 |
+
k = np.load(io.BytesIO(zk.read(f"{f:06d}.npy")))
|
| 57 |
+
M = np.loadtxt(io.BytesIO(zp.read(f"{f:06d}.txt")))
|
| 58 |
+
P = (k[:, 1:4] - M[:3, 3]) @ M[:3, :3] # head frame
|
| 59 |
+
K[f] = dict(zip(k[:, 0].astype(int).tolist(), P.tolist(), strict=True))
|
| 60 |
+
cnt = collections.Counter(i for v in K.values() for i in v)
|
| 61 |
+
ids = sorted(i for i, n in cnt.items() if n >= MIN_SHARE * len(frames))
|
| 62 |
+
X = np.full((len(frames), len(ids), 3), np.nan)
|
| 63 |
+
for r, f in enumerate(frames):
|
| 64 |
+
for j, i in enumerate(ids):
|
| 65 |
+
if i in K[f]:
|
| 66 |
+
X[r, j] = K[f][i]
|
| 67 |
+
X = np.where(np.isnan(X), np.nanmedian(X, axis=0)[None], X).reshape(len(frames), -1)
|
| 68 |
+
chosen = [0]
|
| 69 |
+
d = np.linalg.norm(X - X[0], axis=1)
|
| 70 |
+
while len(chosen) < min(n_expr, len(frames)) and d.max() > 0:
|
| 71 |
+
j = int(np.argmax(d))
|
| 72 |
+
chosen.append(j)
|
| 73 |
+
d = np.minimum(d, np.linalg.norm(X - X[j], axis=1))
|
| 74 |
+
rng = random.Random(f"{seed}:{cap}")
|
| 75 |
+
sel = sorted(frames[j] for j in chosen)
|
| 76 |
+
res = {
|
| 77 |
+
"frames": sel,
|
| 78 |
+
"cams": {str(f): sorted(rng.sample(cams, min(n_cams, len(cams)))) for f in sel},
|
| 79 |
+
"n_frames": len(frames),
|
| 80 |
+
"n_ids_used": len(ids),
|
| 81 |
+
}
|
| 82 |
+
with open(pf, "w") as fh:
|
| 83 |
+
json.dump(res, fh)
|
| 84 |
+
return cap, res
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def main():
|
| 88 |
+
"""Sample every capture of the list and write the sample dict."""
|
| 89 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 90 |
+
ap.add_argument("out", help="output json")
|
| 91 |
+
ap.add_argument("--expressions", type=int, default=50, help="frames per capture")
|
| 92 |
+
ap.add_argument("--cameras", type=int, default=20, help="cameras per frame")
|
| 93 |
+
ap.add_argument("--seed", type=int, default=0)
|
| 94 |
+
ap.add_argument("--captures", default=DEFAULT_CAPTURES, help="file of Ava-256 capture ids")
|
| 95 |
+
a = ap.parse_args()
|
| 96 |
+
with open(a.captures) as fh:
|
| 97 |
+
caps = fh.read().split()
|
| 98 |
+
part = a.out + ".part"
|
| 99 |
+
os.makedirs(part, exist_ok=True)
|
| 100 |
+
R = {}
|
| 101 |
+
with cf.ThreadPoolExecutor(6) as ex:
|
| 102 |
+
jobs = (ex.submit(one_capture, c, part, a.expressions, a.cameras, a.seed) for c in caps)
|
| 103 |
+
for i, fu in enumerate(list(jobs)):
|
| 104 |
+
cap, r = fu.result()
|
| 105 |
+
R[cap] = r
|
| 106 |
+
if i % 16 == 0:
|
| 107 |
+
print(
|
| 108 |
+
f"{i + 1}/{len(caps)} {cap[-6:]} {r.get('skipped', '') or len(r['frames'])}",
|
| 109 |
+
flush=True,
|
| 110 |
+
)
|
| 111 |
+
with open(a.out, "w") as fh:
|
| 112 |
+
json.dump(R, fh)
|
| 113 |
+
ok = [c for c, r in R.items() if "skipped" not in r]
|
| 114 |
+
n_img = sum(len(v) for c in ok for v in R[c]["cams"].values())
|
| 115 |
+
n_fr = sum(len(R[c]["frames"]) for c in ok)
|
| 116 |
+
skipped = [c[-6:] for c in R if "skipped" in R[c]]
|
| 117 |
+
print(
|
| 118 |
+
f"samples: {len(ok)} captures, {n_fr} frames, {n_img} images; skipped {skipped} -> {a.out}"
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
if __name__ == "__main__":
|
| 123 |
+
main()
|
prep/samples.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
prep/samples_closed.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
prep/split.json
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": 1,
|
| 3 |
+
"created": "2026-10-10",
|
| 4 |
+
"rule": "split by identity (capture); frozen; test = the two bench people (never used for model selection); val = seed-0 sample of 24 (~10%); train = the rest",
|
| 5 |
+
"n": {
|
| 6 |
+
"train": 230,
|
| 7 |
+
"val": 24,
|
| 8 |
+
"test": 2
|
| 9 |
+
},
|
| 10 |
+
"train": [
|
| 11 |
+
"20210810--1306--FXN596",
|
| 12 |
+
"20210817--0900--NRE683",
|
| 13 |
+
"20210818--1332--CDR970",
|
| 14 |
+
"20210819--0903--DOT682",
|
| 15 |
+
"20210827--0906--KDA058",
|
| 16 |
+
"20210831--0815--XJT672",
|
| 17 |
+
"20210901--0833--LAS440",
|
| 18 |
+
"20210902--0817--IFG774",
|
| 19 |
+
"20210922--0839--UHV563",
|
| 20 |
+
"20210928--0843--PAK800",
|
| 21 |
+
"20210929--1334--SKB942",
|
| 22 |
+
"20211001--0855--KJJ701",
|
| 23 |
+
"20211001--1304--XMY093",
|
| 24 |
+
"20211006--0836--EID363",
|
| 25 |
+
"20220114--0820--JZE802",
|
| 26 |
+
"20220214--0846--HOS732",
|
| 27 |
+
"20220215--0801--ONK705",
|
| 28 |
+
"20220216--1324--ZSE894",
|
| 29 |
+
"20220303--1247--TAY503",
|
| 30 |
+
"20220304--1320--NNQ107",
|
| 31 |
+
"20220307--1342--RGA575",
|
| 32 |
+
"20220310--1128--ZSC414",
|
| 33 |
+
"20220311--1115--NFU480",
|
| 34 |
+
"20220314--1116--KMD200",
|
| 35 |
+
"20220315--0818--HDZ165",
|
| 36 |
+
"20220315--1053--FWC239",
|
| 37 |
+
"20220401--0841--PHT202",
|
| 38 |
+
"20220406--1314--PGO261",
|
| 39 |
+
"20220407--0816--OYQ923",
|
| 40 |
+
"20220412--1326--BGR645",
|
| 41 |
+
"20220426--1126--SCG995",
|
| 42 |
+
"20220510--0837--WFU321",
|
| 43 |
+
"20220510--1051--OZZ515",
|
| 44 |
+
"20220610--0901--TRP396",
|
| 45 |
+
"20220614--1135--DNM410",
|
| 46 |
+
"20220627--0812--BPM833",
|
| 47 |
+
"20220711--1048--UGU718",
|
| 48 |
+
"20220711--1300--BHC106",
|
| 49 |
+
"20220712--1040--JEH262",
|
| 50 |
+
"20220714--1031--NAB660",
|
| 51 |
+
"20220727--1059--TMA321",
|
| 52 |
+
"20220802--0838--IDD857",
|
| 53 |
+
"20220805--1649--NCU289",
|
| 54 |
+
"20220808--0809--DPE040",
|
| 55 |
+
"20220808--1036--IBX730",
|
| 56 |
+
"20220809--1034--BJM420",
|
| 57 |
+
"20220811--1704--HAZ933",
|
| 58 |
+
"20220815--0815--GZL442",
|
| 59 |
+
"20220815--1041--FZC097",
|
| 60 |
+
"20220815--1307--BMP511",
|
| 61 |
+
"20220815--1656--MOP211",
|
| 62 |
+
"20220818--1304--VCN655",
|
| 63 |
+
"20220818--1653--SSF476",
|
| 64 |
+
"20220819--0803--BUS250",
|
| 65 |
+
"20220819--1313--NYY338",
|
| 66 |
+
"20220824--0804--GTU803",
|
| 67 |
+
"20220829--0836--KSA222",
|
| 68 |
+
"20220830--1711--JZN879",
|
| 69 |
+
"20220831--0751--CMS162",
|
| 70 |
+
"20220831--1047--ZUS575",
|
| 71 |
+
"20220901--1305--LNP057",
|
| 72 |
+
"20221213--1425--VBY735",
|
| 73 |
+
"20230106--1047--QCW581",
|
| 74 |
+
"20230109--1327--NFS933",
|
| 75 |
+
"20230124--1415--KKF424",
|
| 76 |
+
"20230131--1229--WBX393",
|
| 77 |
+
"20230206--1354--MFS340",
|
| 78 |
+
"20230207--1619--DTW301",
|
| 79 |
+
"20230222--1639--KYL945",
|
| 80 |
+
"20230224--1102--UPC002",
|
| 81 |
+
"20230224--1359--CMZ386",
|
| 82 |
+
"20230227--1052--TXE513",
|
| 83 |
+
"20230227--1659--TRT983",
|
| 84 |
+
"20230228--1358--HUJ848",
|
| 85 |
+
"20230301--1646--VNS883",
|
| 86 |
+
"20230302--0808--SEI058",
|
| 87 |
+
"20230302--1343--UWK768",
|
| 88 |
+
"20230302--1654--LVY864",
|
| 89 |
+
"20230303--0802--EFQ599",
|
| 90 |
+
"20230303--1634--ZKN852",
|
| 91 |
+
"20230306--0812--CYQ521",
|
| 92 |
+
"20230306--1402--IXG477",
|
| 93 |
+
"20230307--0805--FQC133",
|
| 94 |
+
"20230307--1648--KUG325",
|
| 95 |
+
"20230308--1352--BDF920",
|
| 96 |
+
"20230308--1639--IKB041",
|
| 97 |
+
"20230309--0820--CPA784",
|
| 98 |
+
"20230309--1343--RIM072",
|
| 99 |
+
"20230310--1106--FCT871",
|
| 100 |
+
"20230313--0819--GCZ208",
|
| 101 |
+
"20230313--1653--RHL466",
|
| 102 |
+
"20230314--1632--PFN770",
|
| 103 |
+
"20230315--0816--TEA904",
|
| 104 |
+
"20230315--1638--UIS049",
|
| 105 |
+
"20230316--1103--BHK376",
|
| 106 |
+
"20230316--1356--GXR876",
|
| 107 |
+
"20230317--1054--GBQ558",
|
| 108 |
+
"20230317--1648--ONK818",
|
| 109 |
+
"20230320--0810--ZJN985",
|
| 110 |
+
"20230322--0820--QHY185",
|
| 111 |
+
"20230322--1706--IBQ026",
|
| 112 |
+
"20230323--1044--HRO745",
|
| 113 |
+
"20230323--1348--ZQK191",
|
| 114 |
+
"20230324--0820--AEY864",
|
| 115 |
+
"20230324--1118--ZZS850",
|
| 116 |
+
"20230324--1647--YWZ007",
|
| 117 |
+
"20230327--1036--RSC756",
|
| 118 |
+
"20230330--0823--LMT312",
|
| 119 |
+
"20230330--1421--UCD539",
|
| 120 |
+
"20230403--1022--RWS274",
|
| 121 |
+
"20230404--0804--GJI283",
|
| 122 |
+
"20230404--1217--DED409",
|
| 123 |
+
"20230405--1213--NWM763",
|
| 124 |
+
"20230406--1655--IEM175",
|
| 125 |
+
"20230410--0805--TWP078",
|
| 126 |
+
"20230410--1017--GVM433",
|
| 127 |
+
"20230410--1641--ENU192",
|
| 128 |
+
"20230412--1218--LCJ763",
|
| 129 |
+
"20230413--1031--PDG961",
|
| 130 |
+
"20230421--1716--ZWR419",
|
| 131 |
+
"20230424--1024--VOG927",
|
| 132 |
+
"20230501--1015--PHE887",
|
| 133 |
+
"20230502--0816--ORZ494",
|
| 134 |
+
"20230502--1030--NOG724",
|
| 135 |
+
"20230502--1418--RWA092",
|
| 136 |
+
"20230503--1006--HDM935",
|
| 137 |
+
"20230503--1400--JUP632",
|
| 138 |
+
"20230508--0828--XIK500",
|
| 139 |
+
"20230508--1015--XGD109",
|
| 140 |
+
"20230511--0810--DUA525",
|
| 141 |
+
"20230512--1009--ZEL435",
|
| 142 |
+
"20230515--0817--JUP244",
|
| 143 |
+
"20230515--1236--UII082",
|
| 144 |
+
"20230516--1240--UQD073",
|
| 145 |
+
"20230517--1211--QZP994",
|
| 146 |
+
"20230518--1009--RFZ285",
|
| 147 |
+
"20230525--1403--IHO061",
|
| 148 |
+
"20230526--1657--TLD140",
|
| 149 |
+
"20230530--1417--RKG241",
|
| 150 |
+
"20230531--1207--KTI779",
|
| 151 |
+
"20230601--1051--NEB896",
|
| 152 |
+
"20230601--1208--UJD603",
|
| 153 |
+
"20230601--1428--FJC651",
|
| 154 |
+
"20230602--0800--TOU998",
|
| 155 |
+
"20230602--1453--FMB793",
|
| 156 |
+
"20230605--1354--JJA915",
|
| 157 |
+
"20230607--0809--PZC599",
|
| 158 |
+
"20230607--1409--HVO722",
|
| 159 |
+
"20230607--1652--ZUE409",
|
| 160 |
+
"20230608--1009--YIK625",
|
| 161 |
+
"20230608--1209--EPY965",
|
| 162 |
+
"20230609--0959--LXC339",
|
| 163 |
+
"20230612--1211--JFS610",
|
| 164 |
+
"20230615--0808--WXT125",
|
| 165 |
+
"20230615--1702--DAT749",
|
| 166 |
+
"20230627--1045--UMJ550",
|
| 167 |
+
"20230627--1701--SIQ409",
|
| 168 |
+
"20230628--1405--IJB027",
|
| 169 |
+
"20230629--1004--QBN395",
|
| 170 |
+
"20230629--1159--APP152",
|
| 171 |
+
"20230630--1424--RSY180",
|
| 172 |
+
"20230630--1651--HTL154",
|
| 173 |
+
"20230703--0826--ZWG600",
|
| 174 |
+
"20230705--1657--HBT289",
|
| 175 |
+
"20230706--1016--CVR434",
|
| 176 |
+
"20230707--0817--KIX259",
|
| 177 |
+
"20230707--1200--MQL211",
|
| 178 |
+
"20230717--0836--PSV686",
|
| 179 |
+
"20230717--1225--QNG975",
|
| 180 |
+
"20230718--1021--UYF211",
|
| 181 |
+
"20230718--1434--YYA788",
|
| 182 |
+
"20230719--1655--MVB233",
|
| 183 |
+
"20230720--1025--CTP175",
|
| 184 |
+
"20230720--1415--MJX355",
|
| 185 |
+
"20230721--1207--WOZ406",
|
| 186 |
+
"20230724--0814--IIQ590",
|
| 187 |
+
"20230725--1218--KXI621",
|
| 188 |
+
"20230726--0805--GZU008",
|
| 189 |
+
"20230726--1016--RLB874",
|
| 190 |
+
"20230726--1657--AYE877",
|
| 191 |
+
"20230727--1211--RSJ093",
|
| 192 |
+
"20230728--0757--CRV122",
|
| 193 |
+
"20230728--1002--EXA831",
|
| 194 |
+
"20230728--1203--FGK870",
|
| 195 |
+
"20230728--1206--SVG441",
|
| 196 |
+
"20230728--1401--FPK790",
|
| 197 |
+
"20230802--1003--KAP399",
|
| 198 |
+
"20230802--1205--LPB114",
|
| 199 |
+
"20230804--1714--URP626",
|
| 200 |
+
"20230807--0839--TCE049",
|
| 201 |
+
"20230808--0759--YLC760",
|
| 202 |
+
"20230808--1157--YJF815",
|
| 203 |
+
"20230808--1656--QFA788",
|
| 204 |
+
"20230809--0814--LWN501",
|
| 205 |
+
"20230809--1645--ITH925",
|
| 206 |
+
"20230810--0812--OXS375",
|
| 207 |
+
"20230810--0819--LNL587",
|
| 208 |
+
"20230810--1201--NDZ630",
|
| 209 |
+
"20230810--1630--ANX726",
|
| 210 |
+
"20230811--1202--QWY097",
|
| 211 |
+
"20230811--1636--XUD838",
|
| 212 |
+
"20230814--0822--GJL377",
|
| 213 |
+
"20230814--1700--LVD531",
|
| 214 |
+
"20230815--1401--PWG860",
|
| 215 |
+
"20230816--0807--TYX812",
|
| 216 |
+
"20230816--1219--EPP420",
|
| 217 |
+
"20230817--0805--PDX936",
|
| 218 |
+
"20230817--1136--DZS003",
|
| 219 |
+
"20230829--1404--OKX508",
|
| 220 |
+
"20230831--0814--ADL311",
|
| 221 |
+
"20230831--1037--UTI042",
|
| 222 |
+
"20230901--1429--CPP930",
|
| 223 |
+
"20230905--1354--JOX535",
|
| 224 |
+
"20230906--0803--HBM931",
|
| 225 |
+
"20230906--1059--QLL112",
|
| 226 |
+
"20230908--1549--NCC597",
|
| 227 |
+
"20230908--1645--DHA971",
|
| 228 |
+
"20230912--1313--NJI020",
|
| 229 |
+
"20230914--1035--GBH400",
|
| 230 |
+
"20230914--1105--BXQ083",
|
| 231 |
+
"20230915--0810--EVY126",
|
| 232 |
+
"20230915--1648--QVU001",
|
| 233 |
+
"20230918--1040--NTA876",
|
| 234 |
+
"20230918--1402--UIQ957",
|
| 235 |
+
"20230919--1056--JEN167",
|
| 236 |
+
"20230919--1626--RHU956",
|
| 237 |
+
"20230920--1404--INQ807",
|
| 238 |
+
"20230925--1356--PKH444",
|
| 239 |
+
"20230925--1644--KWL586",
|
| 240 |
+
"20231003--1715--LGV914"
|
| 241 |
+
],
|
| 242 |
+
"val": [
|
| 243 |
+
"20210929--0827--MCR809",
|
| 244 |
+
"20220202--0844--THN461",
|
| 245 |
+
"20220218--0839--XIT043",
|
| 246 |
+
"20220310--1336--XRZ540",
|
| 247 |
+
"20220705--1055--JPG686",
|
| 248 |
+
"20220726--1314--HNU630",
|
| 249 |
+
"20220809--1321--UTC375",
|
| 250 |
+
"20220812--1031--KVG225",
|
| 251 |
+
"20230213--0817--KGV285",
|
| 252 |
+
"20230323--1656--MZL038",
|
| 253 |
+
"20230328--0800--BLY735",
|
| 254 |
+
"20230328--1214--IEY203",
|
| 255 |
+
"20230407--1229--UMQ592",
|
| 256 |
+
"20230516--0802--KOH994",
|
| 257 |
+
"20230710--0814--WWY595",
|
| 258 |
+
"20230725--1407--OWI326",
|
| 259 |
+
"20230801--1420--EJG940",
|
| 260 |
+
"20230804--1159--MZW030",
|
| 261 |
+
"20230807--1011--TWH159",
|
| 262 |
+
"20230814--0822--GTA798",
|
| 263 |
+
"20230815--0816--HWN858",
|
| 264 |
+
"20230914--1702--WOZ485",
|
| 265 |
+
"20230915--1145--ZEM686",
|
| 266 |
+
"20230925--0843--MGR740"
|
| 267 |
+
],
|
| 268 |
+
"test": [
|
| 269 |
+
"20230405--1635--AAN112",
|
| 270 |
+
"20230810--1355--AJR151"
|
| 271 |
+
],
|
| 272 |
+
"sha256_16_of_lists": "d1d9b81d2ea3d9f0"
|
| 273 |
+
}
|
prep/wild620_map.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:20cd2a77f527a6d267dde9cc3fbff81fd4d569e8a84194e84f2559bd4c9e0727
|
| 3 |
+
size 846902
|
prep/wild620_map.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The 620 head's pseudo-label maps on the wild faces, fitted on a model's own detections."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.wild620_map <detections dir> <out.npz> [--m 16]
|
| 4 |
+
# [--lam 1e-4]
|
| 5 |
+
# Input: detect_wild.py's <set>.npz per set (pts, gt, val, eyes). Fitted on the train faces the
|
| 6 |
+
# model got right (inner-label error through a first-pass map at most FAIL % of the inter-ocular
|
| 7 |
+
# distance), because a model's failures are not a reference.
|
| 8 |
+
# Per set: W (K, 620) writes each INNER hand label (all but the jaw contour) as a local affine
|
| 9 |
+
# combination of its M nearest detected surface points (weights sum to 1, ridge towards uniform);
|
| 10 |
+
# WFLW's two pupil labels map to the two iris points directly. The jaw contour gets no map: the
|
| 11 |
+
# detector's residual there leans inward with head turn, so a map would push a clean model
|
| 12 |
+
# outward. margin (620,): per surface point that neighbours the jaw (median |distance| under
|
| 13 |
+
# 10 % IOD, valid on at least half the faces), the 75th percentile of its signed distance to the
|
| 14 |
+
# labelled jaw. ucmed (620,): the under-chin points' median signed jaw distance, a two-sided
|
| 15 |
+
# target. D (620, K) and dscale: every point the inner map does not use as an affine combination
|
| 16 |
+
# of its ML nearest labels (sum 1), with its median per-coordinate residual as the scale (at
|
| 17 |
+
# least DFLOOR).
|
| 18 |
+
# The output is the trainer's map620= file (losses.Map620 documents the terms).
|
| 19 |
+
import argparse
|
| 20 |
+
import os
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import torch
|
| 24 |
+
|
| 25 |
+
from facepoints.prep.labels import DEFAULT_POINTS
|
| 26 |
+
from facepoints.train.losses import jaw_signed
|
| 27 |
+
|
| 28 |
+
FAIL = 8.0 # % IOD: faces with a worse first-pass inner error are left out of the fit
|
| 29 |
+
ML, DLAM, DFLOOR = 10, 1e-3, 1.5
|
| 30 |
+
JAW = {"wflw": 33, "300w": 17} # the jaw contour = the first J labels of each set
|
| 31 |
+
PUPILS = {"wflw": [96, 97], "300w": []}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def fit_local(P, G, eyes, inner, m, lam):
|
| 35 |
+
"""Return W (K, P): each inner label as an affine mix of its m nearest points (ridge lam)."""
|
| 36 |
+
c = G.mean(1, keepdims=True)
|
| 37 |
+
s = np.linalg.norm(G[:, eyes[0]] - G[:, eyes[1]], axis=1)[:, None, None]
|
| 38 |
+
Pn, Gn = (P - c) / s, (G - c) / s
|
| 39 |
+
mp, mg = Pn.mean(0), Gn.mean(0)
|
| 40 |
+
W = np.zeros((G.shape[1], P.shape[1]))
|
| 41 |
+
for k in np.flatnonzero(inner):
|
| 42 |
+
idx = np.argsort(((mp - mg[k]) ** 2).sum(-1))[:m]
|
| 43 |
+
A = np.concatenate([Pn[:, idx, 0], Pn[:, idx, 1]])
|
| 44 |
+
b = np.concatenate([Gn[:, k, 0], Gn[:, k, 1]])
|
| 45 |
+
H = np.zeros((m + 1, m + 1))
|
| 46 |
+
H[:m, :m] = A.T @ A / len(A) + lam * np.eye(m)
|
| 47 |
+
H[:m, m] = H[m, :m] = 1
|
| 48 |
+
W[k, idx] = np.linalg.solve(H, np.r_[A.T @ b / len(A) + lam / m, 1.0])[:m]
|
| 49 |
+
return W
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def fit_dense(P, G, eyes, pts):
|
| 53 |
+
"""Return D (P, K): each listed point as an affine mix of its ML nearest labels (sum 1)."""
|
| 54 |
+
c = G.mean(1, keepdims=True)
|
| 55 |
+
s = np.linalg.norm(G[:, eyes[0]] - G[:, eyes[1]], axis=1)[:, None, None] / 100
|
| 56 |
+
Pn, Gn = (P - c) / s, (G - c) / s
|
| 57 |
+
mp, mg = Pn.mean(0), Gn.mean(0)
|
| 58 |
+
D = np.zeros((P.shape[1], G.shape[1]))
|
| 59 |
+
for j in pts:
|
| 60 |
+
idx = np.argsort(((mg - mp[j]) ** 2).sum(-1))[:ML]
|
| 61 |
+
A = np.concatenate([Gn[:, idx, 0], Gn[:, idx, 1]])
|
| 62 |
+
b = np.concatenate([Pn[:, j, 0], Pn[:, j, 1]])
|
| 63 |
+
H = np.zeros((ML + 1, ML + 1))
|
| 64 |
+
H[:ML, :ML] = A.T @ A / len(A) + DLAM * 100 * np.eye(ML)
|
| 65 |
+
H[:ML, ML] = H[ML, :ML] = 1
|
| 66 |
+
D[j, idx] = np.linalg.solve(H, np.r_[A.T @ b / len(A) + DLAM * 100 / ML, 1.0])[:ML]
|
| 67 |
+
return D
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def fit_set(ds, z, cls, under, m, lam):
|
| 71 |
+
"""Return the map arrays of one set from its detections npz."""
|
| 72 |
+
eyes, J = tuple(int(e) for e in z["eyes"]), JAW[ds]
|
| 73 |
+
surface = np.isin(cls, ["lmk", "skin", "lid"])
|
| 74 |
+
iris = np.flatnonzero(cls == "iris")
|
| 75 |
+
P, G = z["pts"].astype(np.float64), z["gt"].astype(np.float64)
|
| 76 |
+
tr = np.flatnonzero(~z["val"] & np.isfinite(P[:, surface]).all((1, 2)))
|
| 77 |
+
K = G.shape[1]
|
| 78 |
+
inner = np.ones(K, bool)
|
| 79 |
+
inner[:J] = False
|
| 80 |
+
inner_fit = inner.copy()
|
| 81 |
+
inner_fit[PUPILS[ds]] = False
|
| 82 |
+
cand = np.flatnonzero(surface)
|
| 83 |
+
W = np.zeros((K, len(cls)))
|
| 84 |
+
W[:, cand] = fit_local(P[tr][:, cand], G[tr], eyes, inner_fit, m, lam)
|
| 85 |
+
io = np.linalg.norm(G[tr][:, eyes[0]] - G[tr][:, eyes[1]], axis=1)
|
| 86 |
+
e = (
|
| 87 |
+
np.linalg.norm(np.einsum("kj,njc->nkc", W, P[tr]) - G[tr], axis=2)[:, inner_fit].mean(1)
|
| 88 |
+
/ io
|
| 89 |
+
* 100
|
| 90 |
+
)
|
| 91 |
+
n0, tr = len(tr), tr[e <= FAIL]
|
| 92 |
+
W[:, cand] = fit_local(P[tr][:, cand], G[tr], eyes, inner_fit, m, lam)
|
| 93 |
+
print(f"{ds}: {len(tr)} of {n0} train faces kept (first-pass inner error <= {FAIL} % IOD)")
|
| 94 |
+
for k in PUPILS[ds]: # the iris point nearest this pupil label
|
| 95 |
+
j = iris[np.argmin([np.median(np.linalg.norm(P[tr, i] - G[tr, k], axis=1)) for i in iris])]
|
| 96 |
+
W[k, j] = 1.0
|
| 97 |
+
assert np.allclose(W[inner].sum(1), 1.0) and not W[~inner].any()
|
| 98 |
+
io = np.linalg.norm(G[tr][:, eyes[0]] - G[tr][:, eyes[1]], axis=1)
|
| 99 |
+
c = G[tr][:, inner].mean(1)
|
| 100 |
+
sd, v = jaw_signed(torch.tensor(P[tr][:, cand]), torch.tensor(G[tr][:, :J]), torch.tensor(c))
|
| 101 |
+
sdv = np.where(v.numpy(), (sd / torch.tensor(io)[:, None] * 100).numpy(), np.nan)
|
| 102 |
+
near = (np.nanmedian(np.abs(sdv), 0) < 10) & (v.numpy().mean(0) >= 0.5)
|
| 103 |
+
margin = np.full(len(cls), np.nan)
|
| 104 |
+
margin[cand[near]] = np.nanquantile(sdv[:, near], 0.75, axis=0)
|
| 105 |
+
ucmed = np.full(len(cls), np.nan)
|
| 106 |
+
pos = {int(j): q for q, j in enumerate(cand)}
|
| 107 |
+
for j in np.flatnonzero(under):
|
| 108 |
+
ucmed[j] = np.nanmedian(sdv[:, pos[int(j)]])
|
| 109 |
+
dense = ~(np.abs(W[inner]).sum(0) > 0) & ~under
|
| 110 |
+
D = fit_dense(P[tr], G[tr], eyes, np.flatnonzero(dense))
|
| 111 |
+
r = np.abs(np.einsum("jk,nkc->njc", D, G[tr]) - P[tr]).mean(-1) / io[:, None] * 100
|
| 112 |
+
dscale = np.full(len(cls), np.nan)
|
| 113 |
+
dscale[dense] = np.maximum(np.median(r[:, dense], 0), DFLOOR)
|
| 114 |
+
assert np.allclose(D[dense].sum(1), 1.0) and not D[~dense].any()
|
| 115 |
+
print(f"{ds}: inner {inner.sum()} labels, jaw neighbours {near.sum()}, margin median "
|
| 116 |
+
f"{np.nanmedian(margin):+.2f} %IOD; under-chin target median {np.nanmedian(ucmed):+.2f}; "
|
| 117 |
+
f"dense points {dense.sum()}, scale median {np.nanmedian(dscale):.2f}") # fmt: skip
|
| 118 |
+
return {
|
| 119 |
+
f"W_{ds}": W.astype(np.float32),
|
| 120 |
+
f"inner_{ds}": inner,
|
| 121 |
+
f"margin_{ds}": margin.astype(np.float32),
|
| 122 |
+
f"jaw_{ds}": J,
|
| 123 |
+
f"pupil_{ds}": np.array(PUPILS[ds], np.int64),
|
| 124 |
+
f"ucmed_{ds}": ucmed.astype(np.float32),
|
| 125 |
+
f"D_{ds}": D.astype(np.float32),
|
| 126 |
+
f"dense_{ds}": dense,
|
| 127 |
+
f"dscale_{ds}": dscale.astype(np.float32),
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def main():
|
| 132 |
+
"""Fit the maps of every set and write the trainer's map file."""
|
| 133 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 134 |
+
ap.add_argument("detections", help="detect_wild.py output folder")
|
| 135 |
+
ap.add_argument("out", help="output .npz (the trainer's map620=)")
|
| 136 |
+
ap.add_argument("--m", type=int, default=16, help="nearest points per inner label")
|
| 137 |
+
ap.add_argument("--lam", type=float, default=1e-4, help="ridge towards uniform weights")
|
| 138 |
+
ap.add_argument(
|
| 139 |
+
"--points", default=DEFAULT_POINTS, help="point classes and the under-chin mask"
|
| 140 |
+
)
|
| 141 |
+
ap.add_argument("--sets", default="wflw,300w")
|
| 142 |
+
a = ap.parse_args()
|
| 143 |
+
ps = np.load(a.points)
|
| 144 |
+
cls, under = ps["cls"].astype(str), ps["under_chin"]
|
| 145 |
+
save = {
|
| 146 |
+
"M": a.m,
|
| 147 |
+
"lam": a.lam,
|
| 148 |
+
"fail": FAIL,
|
| 149 |
+
"ml": ML,
|
| 150 |
+
"dlam": DLAM,
|
| 151 |
+
"dfloor": DFLOOR,
|
| 152 |
+
"under": under,
|
| 153 |
+
}
|
| 154 |
+
for ds in a.sets.split(","):
|
| 155 |
+
save.update(
|
| 156 |
+
fit_set(ds, np.load(os.path.join(a.detections, f"{ds}.npz")), cls, under, a.m, a.lam)
|
| 157 |
+
)
|
| 158 |
+
np.savez(a.out, **save)
|
| 159 |
+
print("wrote", a.out)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
main()
|
prep/wild_prep.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""In-the-wild training faces: WFLW and 300W faces framed like studio crops, plus backgrounds."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.prep.wild_prep <out dir> --w300 <dir> --wflw <dir>
|
| 4 |
+
# [--n-bg 6000] [--val 500]
|
| 5 |
+
# Expected layouts. 300W: dlib's ibug_300W_large_face_landmark_dataset folder
|
| 6 |
+
# (labels_ibug_300W_train.xml, labels_ibug_300W_test.xml and the image folders). WFLW:
|
| 7 |
+
# annotations/face_landmarks_wflw_{train,test}.json (COCO-style keypoints, as distributed for
|
| 8 |
+
# mmpose) and images/WIDER_{train,val}/images/.
|
| 9 |
+
# Only the TRAIN faces are used (WFLW 7,500; 300W 3,148); the test faces stay unseen. The last --val
|
| 10 |
+
# faces of each set (seeded shuffle) are held out for the side heads' validation metric.
|
| 11 |
+
# Framing: every face is cropped to 512 px with FRAMING, the median box of the detector's own
|
| 12 |
+
# framing (mesh-point extent x 1.25) relative to the hand-label extent, measured on faces an
|
| 13 |
+
# earlier checkpoint fitted well. Writes <out>/<set>/<i>.jpg and <out>/<set>/labels.npz (uv
|
| 14 |
+
# (N, K, 2) crop px, val (N,) bool, eyes (2,) outer eye corner ids, src, box) and <out>/bg/<i>.jpg:
|
| 15 |
+
# random square crops of WIDER_train images that host no WFLW test face, for the background swap.
|
| 16 |
+
import argparse
|
| 17 |
+
import json
|
| 18 |
+
import os
|
| 19 |
+
import xml.etree.ElementTree as ET
|
| 20 |
+
|
| 21 |
+
import cv2
|
| 22 |
+
import numpy as np
|
| 23 |
+
|
| 24 |
+
CROP = 512
|
| 25 |
+
EYES = {"300w": (36, 45), "wflw": (60, 72)} # outer eye corner ids per label set
|
| 26 |
+
# (side / label extent, centre offset in units of the extent)
|
| 27 |
+
FRAMING = {"wflw": (1.5031, (-0.0015, -0.0778)), "300w": (1.4934, (-0.0023, -0.0803))}
|
| 28 |
+
WFLW_SUBSETS = ("largepose", "expression", "illumination", "makeup", "occlusion", "blur")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def load_300w(root):
|
| 32 |
+
"""Return [(image path, split, (68, 2) labels)] from dlib's XML files without the mirrors."""
|
| 33 |
+
out, seen = [], set()
|
| 34 |
+
for x in ("labels_ibug_300W_train.xml", "labels_ibug_300W_test.xml"):
|
| 35 |
+
for im in ET.parse(os.path.join(root, x)).getroot().iter("image"):
|
| 36 |
+
f = im.get("file")
|
| 37 |
+
if "mirror" in f or f in seen:
|
| 38 |
+
continue
|
| 39 |
+
g = f.replace("\\", "/")
|
| 40 |
+
if g.startswith(("afw/", "helen/trainset/", "lfpw/trainset/")):
|
| 41 |
+
sp = "train"
|
| 42 |
+
elif g.startswith("ibug/") or "/testset/" in g:
|
| 43 |
+
sp = "test"
|
| 44 |
+
else:
|
| 45 |
+
continue
|
| 46 |
+
parts = im.find("box").findall("part")
|
| 47 |
+
if len(parts) != 68:
|
| 48 |
+
continue
|
| 49 |
+
pts = np.zeros((68, 2))
|
| 50 |
+
for p in parts:
|
| 51 |
+
pts[int(p.get("name"))] = float(p.get("x")), float(p.get("y"))
|
| 52 |
+
seen.add(f)
|
| 53 |
+
out.append((os.path.join(root, f), sp, pts))
|
| 54 |
+
return out
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def load_wflw(root):
|
| 58 |
+
"""Return [(image path, split, (98, 2) labels)] from the COCO-style WFLW annotations."""
|
| 59 |
+
ann = os.path.join(root, "annotations")
|
| 60 |
+
roots = [os.path.join(root, "images", d, "images") for d in ("WIDER_train", "WIDER_val")]
|
| 61 |
+
out = []
|
| 62 |
+
for name in ("train", "test"):
|
| 63 |
+
with open(os.path.join(ann, f"face_landmarks_wflw_{name}.json")) as fh:
|
| 64 |
+
d = json.load(fh)
|
| 65 |
+
files = {im["id"]: im["file_name"] for im in d["images"]}
|
| 66 |
+
for a in d["annotations"]:
|
| 67 |
+
path = next(
|
| 68 |
+
(
|
| 69 |
+
p
|
| 70 |
+
for p in (os.path.join(r, files[a["image_id"]]) for r in roots)
|
| 71 |
+
if os.path.exists(p)
|
| 72 |
+
),
|
| 73 |
+
None,
|
| 74 |
+
)
|
| 75 |
+
if path is None:
|
| 76 |
+
continue
|
| 77 |
+
out.append((path, name, np.array(a["keypoints"], np.float64).reshape(-1, 3)[:, :2]))
|
| 78 |
+
return out
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def crop(img, c, side):
|
| 82 |
+
"""Return (the CROP x CROP crop centred at c with the given side, its affine matrix)."""
|
| 83 |
+
s = CROP / side
|
| 84 |
+
M = np.array([[s, 0, CROP / 2 - c[0] * s], [0, s, CROP / 2 - c[1] * s]])
|
| 85 |
+
return cv2.warpAffine(img, M, (CROP, CROP), flags=cv2.INTER_AREA, borderValue=(0, 0, 0)), M
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def faces(out, ds, items, n_val):
|
| 89 |
+
"""Write one set's train faces as crops and labels.npz."""
|
| 90 |
+
ratio, off = FRAMING[ds]
|
| 91 |
+
L = [x for x in items if x[1] == "train"]
|
| 92 |
+
d = os.path.join(out, ds)
|
| 93 |
+
os.makedirs(d, exist_ok=True)
|
| 94 |
+
val = np.zeros(len(L), bool)
|
| 95 |
+
val[np.random.default_rng(0).permutation(len(L))[:n_val]] = True
|
| 96 |
+
UV, BOX, F = [], [], []
|
| 97 |
+
for i, (f, _, gt) in enumerate(L):
|
| 98 |
+
lo, hi = gt.min(0), gt.max(0)
|
| 99 |
+
side = float((hi - lo).max() * ratio)
|
| 100 |
+
c = (lo + hi) / 2 + np.asarray(off) * (hi - lo).max()
|
| 101 |
+
im, M = crop(cv2.imread(f), c, side)
|
| 102 |
+
cv2.imwrite(os.path.join(d, f"{i:05d}.jpg"), im, [cv2.IMWRITE_JPEG_QUALITY, 95])
|
| 103 |
+
UV.append(gt @ M[:, :2].T + M[:, 2])
|
| 104 |
+
BOX.append([c[0] - side / 2, c[1] - side / 2, c[0] + side / 2, c[1] + side / 2])
|
| 105 |
+
F.append(f)
|
| 106 |
+
arrays = dict(uv=np.array(UV, np.float32), val=val, eyes=np.array(EYES[ds]), src=np.array(F))
|
| 107 |
+
np.savez_compressed(os.path.join(d, "labels.npz"), box=np.array(BOX, np.float32), **arrays)
|
| 108 |
+
print(f"WILD {ds}: {len(L)} train faces ({val.sum()} held out for val) -> {d}", flush=True)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def backgrounds(out, wflw_root, wflw_items, n):
|
| 112 |
+
"""Write n random square crops of WIDER_train images that host no WFLW test face."""
|
| 113 |
+
test_hosts = {os.path.normcase(f) for f, sp, _ in wflw_items if sp == "test"}
|
| 114 |
+
root = os.path.join(wflw_root, "images", "WIDER_train", "images")
|
| 115 |
+
files = sorted(
|
| 116 |
+
os.path.join(dp, f) for dp, _, fs in os.walk(root) for f in fs if f.endswith(".jpg")
|
| 117 |
+
)
|
| 118 |
+
keep = [f for f in files if os.path.normcase(f) not in test_hosts]
|
| 119 |
+
d = os.path.join(out, "bg")
|
| 120 |
+
os.makedirs(d, exist_ok=True)
|
| 121 |
+
rng = np.random.default_rng(1)
|
| 122 |
+
for i in range(n):
|
| 123 |
+
im = cv2.imread(keep[int(rng.integers(len(keep)))])
|
| 124 |
+
h, w = im.shape[:2]
|
| 125 |
+
side = int(min(h, w) * rng.uniform(0.3, 1.0))
|
| 126 |
+
y, x = int(rng.integers(0, h - side + 1)), int(rng.integers(0, w - side + 1))
|
| 127 |
+
im = cv2.resize(im[y : y + side, x : x + side], (CROP, CROP), interpolation=cv2.INTER_AREA)
|
| 128 |
+
cv2.imwrite(os.path.join(d, f"{i:05d}.jpg"), im, [cv2.IMWRITE_JPEG_QUALITY, 90])
|
| 129 |
+
excluded = len(files) - len(keep)
|
| 130 |
+
print(f"BACKGROUNDS {n} from {len(keep)} WIDER train images ({excluded} WFLW-test hosts "
|
| 131 |
+
f"excluded) "
|
| 132 |
+
f"-> {d}", flush=True) # fmt: skip
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def main():
|
| 136 |
+
"""Write both face sets and the backgrounds."""
|
| 137 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 138 |
+
ap.add_argument("out", help="output root (the trainer's wild=<dir>)")
|
| 139 |
+
ap.add_argument("--w300", required=True, help="300W root (dlib's XML distribution)")
|
| 140 |
+
ap.add_argument(
|
| 141 |
+
"--wflw", required=True, help="WFLW root (COCO-style annotations + WIDER images)"
|
| 142 |
+
)
|
| 143 |
+
ap.add_argument("--n-bg", type=int, default=6000, help="background crops")
|
| 144 |
+
ap.add_argument("--val", type=int, default=500, help="faces per set held out for validation")
|
| 145 |
+
a = ap.parse_args()
|
| 146 |
+
wflw = load_wflw(a.wflw)
|
| 147 |
+
faces(a.out, "wflw", wflw, a.val)
|
| 148 |
+
faces(a.out, "300w", load_300w(a.w300), a.val)
|
| 149 |
+
backgrounds(a.out, a.wflw, wflw, a.n_bg)
|
| 150 |
+
print("WILD PREP DONE", flush=True)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
if __name__ == "__main__":
|
| 154 |
+
main()
|
pyproject.toml
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "facepoints"
|
| 3 |
+
version = "1.0.0"
|
| 4 |
+
description = "620 face points defined on the GNM Head mesh, plus the 68 / 98-point sets, from one photo"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.10"
|
| 7 |
+
license = { file = "LICENSE" }
|
| 8 |
+
authors = [{ name = "Myned AI", email = "opensource@myned.ai" }]
|
| 9 |
+
dependencies = [
|
| 10 |
+
"numpy<2",
|
| 11 |
+
"opencv-python-headless>=4.8",
|
| 12 |
+
"safetensors>=0.4",
|
| 13 |
+
"timm>=0.9",
|
| 14 |
+
"torch>=2.0",
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
[project.optional-dependencies]
|
| 18 |
+
prep = ["pillow>=10", "pillow-avif-plugin>=1.4"]
|
| 19 |
+
test = ["pytest"]
|
| 20 |
+
|
| 21 |
+
[project.urls]
|
| 22 |
+
Homepage = "https://huggingface.co/myned-ai/facepoints"
|
| 23 |
+
|
| 24 |
+
[build-system]
|
| 25 |
+
requires = ["setuptools>=68"]
|
| 26 |
+
build-backend = "setuptools.build_meta"
|
| 27 |
+
|
| 28 |
+
[tool.setuptools]
|
| 29 |
+
packages = ["facepoints", "facepoints.train", "facepoints.train.data", "facepoints.prep"]
|
| 30 |
+
package-dir = { "facepoints" = "." }
|
| 31 |
+
|
| 32 |
+
[tool.setuptools.package-data]
|
| 33 |
+
facepoints = ["points_gnm.json", "prep/*.npz", "prep/*.json", "prep/*.txt"]
|
| 34 |
+
|
| 35 |
+
[tool.ruff]
|
| 36 |
+
line-length = 100
|
| 37 |
+
|
| 38 |
+
[tool.ruff.lint]
|
| 39 |
+
select = ["E", "F", "W", "I"]
|
| 40 |
+
|
| 41 |
+
[tool.ruff.lint.isort]
|
| 42 |
+
known-first-party = ["facepoints"]
|
| 43 |
+
|
| 44 |
+
[tool.ruff.format]
|
| 45 |
+
skip-magic-trailing-comma = true
|
run.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Detect the 620 points on every face OpenCV's YuNet finds in a photo and draw them."""
|
| 2 |
+
|
| 3 |
+
# Usage: python run.py photo.jpg [out.jpg] [--model DIR]
|
| 4 |
+
# The model folder defaults to ./model next to this file (model.safetensors + config.json).
|
| 5 |
+
# Works from a clone of the repository without installing it; YuNet is downloaded once into the
|
| 6 |
+
# temp dir. Installed, the same script is `python -m facepoints.run`.
|
| 7 |
+
import argparse
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
import tempfile
|
| 11 |
+
import urllib.request
|
| 12 |
+
|
| 13 |
+
import cv2
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 17 |
+
if __package__ in (None, ""): # run as a file, not installed: make the clone importable
|
| 18 |
+
sys.path.insert(0, os.path.dirname(HERE))
|
| 19 |
+
try:
|
| 20 |
+
from facepoints import Detector, square_box # noqa: E402
|
| 21 |
+
except ModuleNotFoundError:
|
| 22 |
+
sys.exit("facepoints is not importable: keep the folder named facepoints, or pip install -e .")
|
| 23 |
+
|
| 24 |
+
YUNET_URL = (
|
| 25 |
+
"https://github.com/opencv/opencv_zoo/raw/main/models/face_detection_yunet/"
|
| 26 |
+
"face_detection_yunet_2023mar.onnx"
|
| 27 |
+
)
|
| 28 |
+
CONFIDENT_PX = 4.0 # points with a smaller std are drawn green, the rest orange
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def main(photo, out, model_dir):
|
| 32 |
+
"""Find the faces, detect their points and write the drawing."""
|
| 33 |
+
yunet = os.path.join(tempfile.gettempdir(), "yunet_2023mar.onnx")
|
| 34 |
+
if not os.path.exists(yunet):
|
| 35 |
+
urllib.request.urlretrieve(YUNET_URL, yunet)
|
| 36 |
+
finder = cv2.FaceDetectorYN.create(yunet, "", (320, 320), score_threshold=0.5)
|
| 37 |
+
det = Detector(model_dir)
|
| 38 |
+
img = cv2.imread(photo)
|
| 39 |
+
finder.setInputSize((img.shape[1], img.shape[0]))
|
| 40 |
+
_, faces = finder.detect(img)
|
| 41 |
+
for f in [] if faces is None else faces:
|
| 42 |
+
res = det.detect(img, square_box(*f[:4]))
|
| 43 |
+
for (x, y), s in zip(res["points"], res["std"], strict=True):
|
| 44 |
+
colour = (0, 255, 0) if s < CONFIDENT_PX else (0, 165, 255)
|
| 45 |
+
cv2.circle(img, (int(round(x)), int(round(y))), 2, colour, -1)
|
| 46 |
+
box = np.round(f[:4]).astype(int).tolist()
|
| 47 |
+
print(f"face at {box}: median std {np.median(res['std']):.2f} px")
|
| 48 |
+
cv2.imwrite(out, img)
|
| 49 |
+
print("wrote", out)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
if __name__ == "__main__":
|
| 53 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 54 |
+
ap.add_argument("photo")
|
| 55 |
+
ap.add_argument("out", nargs="?", help="default: <photo>_points.jpg")
|
| 56 |
+
ap.add_argument("--model", default=os.path.join(HERE, "model"), help="the model folder")
|
| 57 |
+
a = ap.parse_args()
|
| 58 |
+
main(a.photo, a.out or os.path.splitext(a.photo)[0] + "_points.jpg", a.model)
|
tests/test_detect.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests: the point definitions always; the model when model/ exists or FACEPOINTS_MODEL is set."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pytest
|
| 7 |
+
|
| 8 |
+
from facepoints import gnm_positions, load_points
|
| 9 |
+
|
| 10 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 11 |
+
MODEL = os.environ.get("FACEPOINTS_MODEL") or os.path.join(os.path.dirname(HERE), "model")
|
| 12 |
+
MODEL = MODEL if os.path.isdir(MODEL) else None # the shipped model/ folder, when present
|
| 13 |
+
PHOTO = os.environ.get("FACEPOINTS_PHOTO")
|
| 14 |
+
CLASSES = {"lmk": 416, "skin": 150, "lid": 36, "teeth_u": 7, "teeth_l": 9, "iris": 2}
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def test_points_file():
|
| 18 |
+
"""The point file has 620 rows in order, normalised weights and the documented class counts."""
|
| 19 |
+
pts = load_points()
|
| 20 |
+
assert len(pts) == 620 and [p["index"] for p in pts] == list(range(620))
|
| 21 |
+
assert all(abs(sum(p["weights"]) - 1) < 1e-5 for p in pts)
|
| 22 |
+
counts = {c: sum(p["class"] == c for p in pts) for c in CLASSES}
|
| 23 |
+
assert counts == CLASSES
|
| 24 |
+
X = np.random.default_rng(0).normal(size=(17821, 3))
|
| 25 |
+
P = gnm_positions(X, pts)
|
| 26 |
+
assert P.shape == (620, 3) and np.isfinite(P).all()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@pytest.mark.skipif(not MODEL, reason="set FACEPOINTS_MODEL to the folder with model.safetensors")
|
| 30 |
+
def test_model_classes_match_points():
|
| 31 |
+
"""The released model's point classes equal the point file's, row by row."""
|
| 32 |
+
import json
|
| 33 |
+
|
| 34 |
+
with open(os.path.join(MODEL, "config.json")) as fh:
|
| 35 |
+
config = json.load(fh)
|
| 36 |
+
assert config["classes"] == [p["class"] for p in load_points()]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@pytest.mark.skipif(not (MODEL and PHOTO), reason="set FACEPOINTS_MODEL and FACEPOINTS_PHOTO")
|
| 40 |
+
def test_detector():
|
| 41 |
+
"""The detector returns finite points, positive std and both side sets for one face."""
|
| 42 |
+
import cv2
|
| 43 |
+
|
| 44 |
+
from facepoints import Detector
|
| 45 |
+
|
| 46 |
+
det = Detector(MODEL)
|
| 47 |
+
img = cv2.imread(PHOTO)
|
| 48 |
+
assert img is not None
|
| 49 |
+
h, w = img.shape[:2]
|
| 50 |
+
side = 0.8 * min(h, w)
|
| 51 |
+
out = det.detect(img, (w / 2 - side / 2, h / 2 - side / 2, side))
|
| 52 |
+
assert out["points"].shape == (620, 2) and out["std"].shape == (620,)
|
| 53 |
+
assert np.isfinite(out["points"]).all() and (out["std"] > 0).all()
|
| 54 |
+
assert out["300w"].shape == (68, 2) and out["wflw"].shape == (98, 2)
|
train/README.md
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# How the model was trained
|
| 2 |
+
|
| 3 |
+
Everything here runs from a clone with the package installed (`pip install -e ".[prep]"`); every
|
| 4 |
+
command is `python -m facepoints.<module>`. Seed 0 throughout, one run per stage, no repeats.
|
| 5 |
+
|
| 6 |
+
## 1. Studio data (Ava-256)
|
| 7 |
+
|
| 8 |
+
Ava-256 is public (Meta, CC BY-NC 4.0); the scripts read its S3 zips by HTTP range, nothing is
|
| 9 |
+
downloaded whole except the per-capture keypoint and head-pose zips. The identity split
|
| 10 |
+
([prep/split.json](../prep/split.json)) has 230 training, 24 validation and 2 test captures.
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
# the sample lists used for the release are shipped: prep/samples.json (50 expressions x 20 cameras
|
| 14 |
+
# per capture) and prep/samples_closed.json (15 closed + 5 half-closed eye frames x 20 cameras);
|
| 15 |
+
# prep.sample_frames and prep.closed_frames regenerate them
|
| 16 |
+
python -m facepoints.prep.extract facepoints/prep/samples.json data --workers 16
|
| 17 |
+
python -m facepoints.prep.iris_labels data # rows 618 / 619 from the iris rings
|
| 18 |
+
python -m facepoints.prep.extract_masks data # person masks for the background swap
|
| 19 |
+
python -m facepoints.prep.extract facepoints/prep/samples_closed.json data_closed
|
| 20 |
+
python -m facepoints.prep.iris_labels data_closed
|
| 21 |
+
python -m facepoints.prep.merge_data data data_closed train --tag closed # training captures only
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
Result: `data/<capture>/{labels.npz, <frame>_<cam>.jpg, <frame>_<cam>_m.png}`; 345,671 training
|
| 25 |
+
images. The Ava template OBJ is fetched once from the ava-256 repository (or set `AVA_TOPOLOGY`).
|
| 26 |
+
|
| 27 |
+
## 2. In-the-wild data (300W, WFLW)
|
| 28 |
+
|
| 29 |
+
Download 300W (dlib's `ibug_300W_large_face_landmark_dataset`) and WFLW with its COCO-style
|
| 30 |
+
annotations (`annotations/face_landmarks_wflw_{train,test}.json`, images under `images/WIDER_*/`).
|
| 31 |
+
|
| 32 |
+
```bash
|
| 33 |
+
python -m facepoints.prep.wild_prep wild --w300 <300W dir> --wflw <WFLW dir>
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
Result: `wild/{wflw,300w}/{labels.npz, <i>.jpg}` (7,500 and 3,148 train faces, 500 of each held out
|
| 37 |
+
for validation) and `wild/bg/` (6,000 WIDER FACE background crops). The test faces are never used.
|
| 38 |
+
|
| 39 |
+
## 3. The three stages
|
| 40 |
+
|
| 41 |
+
```bash
|
| 42 |
+
# stage 1: studio only, 10.45 h on one A100; keep best.pt (lowest held-out visible-point error,
|
| 43 |
+
# reached at iteration 174,000)
|
| 44 |
+
python -m facepoints.train data out/stage1 batch=64 iters=183308 val_every=2000 val_n=4096 \
|
| 45 |
+
workers=20 log_every=50 nll=laplace
|
| 46 |
+
|
| 47 |
+
# stage 2: + wild faces for the side heads, 20,000 iterations
|
| 48 |
+
python -m facepoints.train data out/stage2 iters=20000 val_every=2000 val_n=4096 workers=6 \
|
| 49 |
+
log_every=200 nll=laplace pretrained=0 init=out/stage1/best.pt lr=3e-5 lr_bb=3e-6 warmup=200 \
|
| 50 |
+
batch=48 fresh_aug=1 wild=wild wild_bs=12 wild_w=1.0 bg_p=0.5 relight_p=0.3 occ2_p=0.2 \
|
| 51 |
+
glasses_p=0.1 wild_mix=0.5,0.5
|
| 52 |
+
|
| 53 |
+
# stage 3: Student-t likelihood, refinement stage on the side heads, mild/heavy augmentation mix,
|
| 54 |
+
# the 620 head supervised on the wild faces through the map; 12,000 iterations (stages 2 + 3
|
| 55 |
+
# together 5.4 h on one V100). ckpt.pt at the end is the released model.
|
| 56 |
+
python -m facepoints.train data out/stage3 iters=12000 val_every=2000 val_n=4096 workers=6 \
|
| 57 |
+
log_every=200 nll=studentt nu=10 pretrained=0 init=out/stage2/ckpt.pt lr=3e-5 lr_bb=1e-5 \
|
| 58 |
+
warmup=200 batch=48 fresh_aug=1 wild=wild wild_bs=12 wild_w=1.0 bg_p=0.5 relight_p=0.3 \
|
| 59 |
+
occ2_p=0.2 glasses_p=0.1 wild_mix=0.5,0.5 mild_p=0.5 flip=1 side_rf=1 lr_rf=3e-4 \
|
| 60 |
+
map620=facepoints/prep/wild620_map.npz w620=1 w620_jaw=1 w620_dense=0.3 w620_uc=1
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
Every other key is at its default ([config.py](config.py)); every key prints an `ANNOUNCE` line and
|
| 64 |
+
an unknown key aborts. Before each long run a 120-iteration smoke on the same machine checked that
|
| 65 |
+
every key announced itself, that `STUDIO FRESH` reported a non-zero difference, that the wild sets
|
| 66 |
+
and masks were found and that the first validation wrote `val.json` and a picture grid. A twin of
|
| 67 |
+
stage 3 with `nll=laplace` scored the same on every instrument and was not released.
|
| 68 |
+
|
| 69 |
+
**The pseudo-label map** ([prep/wild620_map.npz](../prep/wild620_map.npz)) is shipped as used: it was
|
| 70 |
+
fitted on the detections of an earlier checkpoint of this detector on the wild training crops. To
|
| 71 |
+
refit it from the released model instead:
|
| 72 |
+
|
| 73 |
+
```bash
|
| 74 |
+
python -m facepoints.prep.detect_wild <model dir> wild detections
|
| 75 |
+
python -m facepoints.prep.wild620_map detections wild620_map.npz
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
## 4. Release files
|
| 79 |
+
|
| 80 |
+
```bash
|
| 81 |
+
python -m facepoints.train.export out/stage3/ckpt.pt release/ # model.safetensors + config.json
|
| 82 |
+
```
|
| 83 |
+
|
| 84 |
+
`config.json` holds the architecture, the 620 point classes in output order, the likelihood and
|
| 85 |
+
its std factor, the side heads and the training keys as run; `Detector` reads the folder.
|
train/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Training: the trainer and its configuration, losses, validation, datasets and augmentation."""
|
train/__main__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""`python -m facepoints.train <data root> <out dir> [key=value ...]` runs the trainer."""
|
| 2 |
+
|
| 3 |
+
from facepoints.train.trainer import run_as_script
|
| 4 |
+
|
| 5 |
+
run_as_script()
|
train/config.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Training configuration: every trainer key, typed and documented, read from key=value args."""
|
| 2 |
+
|
| 3 |
+
# Every field prints an ANNOUNCE line at start-up and an unknown key aborts, so a misspelt flag can
|
| 4 |
+
# never be silently ignored. Defaults are stage 1 of the released recipe (train/README.md) except
|
| 5 |
+
# the run length, the loader workers and the logging cadence.
|
| 6 |
+
import os
|
| 7 |
+
import sys
|
| 8 |
+
from dataclasses import asdict, dataclass, fields
|
| 9 |
+
|
| 10 |
+
PREP = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "prep")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class TrainConfig:
|
| 15 |
+
# --- data: extracted Ava-256 captures (<root>/<capture>/{labels.npz, <frame>_<cam>.jpg})
|
| 16 |
+
split: str = os.path.join(PREP, "split.json") # identity split: {train, val, test: [captures]}
|
| 17 |
+
train_caps: str = (
|
| 18 |
+
"train" # a split key, ALL (every capture under root) or a file of capture ids
|
| 19 |
+
)
|
| 20 |
+
val_caps: str = "val"
|
| 21 |
+
val_n: int = 2048 # validation images (evenly spaced over the val captures)
|
| 22 |
+
min_vis: float = 0.2 # drop images with a smaller visible fraction of points (rear cameras)
|
| 23 |
+
points: str = os.path.join(PREP, "points_ava.npz") # point classes and regions (prep/)
|
| 24 |
+
workers: int = 6 # loader workers for the studio set (the wild set gets max(2, workers // 4))
|
| 25 |
+
device: str = "cuda"
|
| 26 |
+
# --- model
|
| 27 |
+
backbone: str = "hrnet_w48" # any timm backbone with feature maps at strides 4 and `stride`
|
| 28 |
+
res: int = 256 # input side in px (the labels come from 512-px crops)
|
| 29 |
+
pretrained: int = 1 # timm ImageNet weights at construction (0 when init= loads a checkpoint)
|
| 30 |
+
neck: int = 32 # channels of the 1x1 conv on the feature grid before the flattened head
|
| 31 |
+
stride: int = 8 # the feature map the head reads (hrnet_w48 at 256 px: 32 x 32)
|
| 32 |
+
side_rf: int = 0 # 1 = a local refinement stage on each side head
|
| 33 |
+
init: str = "" # start from this checkpoint's weights with a fresh optimiser
|
| 34 |
+
resume: str = "" # continue this checkpoint (weights, optimiser, iteration)
|
| 35 |
+
# --- loss: per-point (x, y, log scale) negative log-likelihood
|
| 36 |
+
nll: str = "laplace" # gauss | laplace | studentt
|
| 37 |
+
nu: float = 10.0 # Student-t degrees of freedom
|
| 38 |
+
w_lid: float = 1.0 # loss weight of eyelid-edge points
|
| 39 |
+
w_teeth: float = 1.0 # loss weight of teeth points
|
| 40 |
+
w_occ: float = 1.0 # loss weight of labelled but mesh-occluded points
|
| 41 |
+
# --- schedule
|
| 42 |
+
batch: int = 64 # studio images per step
|
| 43 |
+
iters: int = 100000
|
| 44 |
+
lr: float = 1e-4 # heads
|
| 45 |
+
lr_bb: float = 1e-5 # backbone
|
| 46 |
+
lr_rf: float = 3e-4 # refinement stages (they start from zero)
|
| 47 |
+
wd: float = 0.05 # AdamW weight decay
|
| 48 |
+
warmup: int = 500 # linear warm-up iterations
|
| 49 |
+
flat: float = 0.5 # fraction of the run at the full learning rate before the cosine decay
|
| 50 |
+
amp: int = 1 # float16 autocast
|
| 51 |
+
seed: int = 0
|
| 52 |
+
# --- studio augmentation (data.augment): affine, low-resolution branch, photometric, occluders
|
| 53 |
+
aff_smin: float = 0.5 # scale range
|
| 54 |
+
aff_smax: float = 1.5
|
| 55 |
+
aff_rot: float = 80.0 # rotation range, degrees
|
| 56 |
+
aff_rot_p: float = 0.6 # probability of a rotation
|
| 57 |
+
aff_shift: float = 0.1 # shift range, fraction of the side
|
| 58 |
+
lowres_min: int = 48 # smallest simulated face size, px
|
| 59 |
+
lowres_p: float = 0.5 # probability of the low-resolution branch
|
| 60 |
+
lowres_jpeg: float = 0.5 # probability of JPEG re-encoding at the small size
|
| 61 |
+
lowres_q: int = 30 # lowest JPEG quality of that re-encoding
|
| 62 |
+
occ_p: float = 1.0 # probability of the synthetic occluders
|
| 63 |
+
grey_p: float = 0.05 # probability of greyscale
|
| 64 |
+
studio_fresh: int = 1 # 1 = a new augmentation draw per read (0 = one fixed draw per image)
|
| 65 |
+
# --- in-the-wild faces (prep/wild_prep.py output) for the 68 / 98-point side heads
|
| 66 |
+
wild: str = "" # dir with <set>/{labels.npz, <i>.jpg} and bg/ backgrounds; "" = studio only
|
| 67 |
+
wild_sets: str = "wflw,300w" # one side head per set
|
| 68 |
+
wild_bs: int = 16 # wild faces per step, added to the same forward pass
|
| 69 |
+
wild_w: float = 1.0 # weight of the side heads' loss
|
| 70 |
+
wild_mix: str = "" # draw shares per set, e.g. 0.5,0.5 ("" = uniform over faces)
|
| 71 |
+
bg_p: float = 0.0 # background swap behind the person mask (studio images with a mask)
|
| 72 |
+
relight_p: float = 0.0 # directional relighting and cast shadows
|
| 73 |
+
occ2_p: float = 0.0 # real-texture occluders cut from the backgrounds
|
| 74 |
+
glasses_p: float = 0.0 # sunglasses over the eyes
|
| 75 |
+
flip: int = 0 # 1 = mirror the 300W faces (iBUG-68 left-right map)
|
| 76 |
+
fresh_aug: int = 0 # 1 = a new augmentation draw per read for the wild faces
|
| 77 |
+
mild_p: float = 0.0 # share of wild faces drawing the mild augmentation instead
|
| 78 |
+
# --- the 620 points on the wild faces (prep/wild620_map.py output)
|
| 79 |
+
map620: str = "" # the map file; "" = the 620 head is supervised on studio images only
|
| 80 |
+
w620: float = 0.0 # inner points through the map to the hand labels
|
| 81 |
+
w620_jaw: float = 0.0 # jaw-neighbour points kept inside the labelled jaw line
|
| 82 |
+
w620_dense: float = 0.0 # every other point against its pseudo-label
|
| 83 |
+
w620_uc: float = 0.0 # under-chin points held at their reference distance to the jaw
|
| 84 |
+
# --- logging
|
| 85 |
+
val_every: int = 2000
|
| 86 |
+
log_every: int = 20
|
| 87 |
+
|
| 88 |
+
@classmethod
|
| 89 |
+
def from_argv(cls, argv):
|
| 90 |
+
"""Return (data root, out dir, config) from argv; unknown keys abort; fields announce."""
|
| 91 |
+
if len(argv) < 3:
|
| 92 |
+
sys.exit(
|
| 93 |
+
"usage: python -m facepoints.train <data root> <out dir> [key=value ...]"
|
| 94 |
+
)
|
| 95 |
+
root, out, cfg = argv[1], argv[2], cls()
|
| 96 |
+
types = {f.name: f.type for f in fields(cls)}
|
| 97 |
+
for kv in argv[3:]:
|
| 98 |
+
k, _, v = kv.partition("=")
|
| 99 |
+
if k not in types:
|
| 100 |
+
sys.exit(f"unknown key {k}")
|
| 101 |
+
setattr(cfg, k, types[k](v))
|
| 102 |
+
for k, v in asdict(cfg).items():
|
| 103 |
+
print(f"ANNOUNCE {k}={v}", flush=True)
|
| 104 |
+
return root, out, cfg
|
| 105 |
+
|
| 106 |
+
PATH_KEYS = ("split", "points", "init", "resume", "wild", "map620", "train_caps", "val_caps")
|
| 107 |
+
|
| 108 |
+
def public(self):
|
| 109 |
+
"""Return the fields without local file paths, for the release config.json."""
|
| 110 |
+
return {k: v for k, v in asdict(self).items() if k not in self.PATH_KEYS}
|
train/data/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Datasets and augmentation: studio set (ava), in-the-wild sets (wild), augmentation (augment)."""
|
| 2 |
+
|
| 3 |
+
# What the trainer imports; everything else is reached through the submodules.
|
| 4 |
+
from facepoints.train.data.augment import Affine, LowRes # noqa: F401
|
| 5 |
+
from facepoints.train.data.ava import PointData # noqa: F401
|
| 6 |
+
from facepoints.train.data.wild import FLIP68, MILD_BIT, Wild, WildData # noqa: F401
|
train/data/augment.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Augmentation of a face crop and its labels: affine, low resolution, photometric, occluders."""
|
| 2 |
+
|
| 3 |
+
# Training only. One random affine that the labels follow, with a constant random border;
|
| 4 |
+
# photometric jitter (brightness, contrast, per-channel gain, gamma, hue/saturation, grey, gaussian
|
| 5 |
+
# and directional motion blur, noise, JPEG re-encode); zero to two occluders. The in-the-wild extras
|
| 6 |
+
# (background swap behind the person mask, relighting with cast shadows, sunglasses, real-texture
|
| 7 |
+
# occluders) are applied through a hook right after the affine (data.wild.Wild).
|
| 8 |
+
import functools
|
| 9 |
+
from typing import NamedTuple
|
| 10 |
+
|
| 11 |
+
import cv2
|
| 12 |
+
import numpy as np
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class Affine(NamedTuple):
|
| 16 |
+
"""The random affine: scale range, rotation range in degrees and its probability."""
|
| 17 |
+
|
| 18 |
+
scale_min: float
|
| 19 |
+
scale_max: float
|
| 20 |
+
rotation_deg: float
|
| 21 |
+
rotation_p: float
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class LowRes(NamedTuple):
|
| 25 |
+
"""Low-resolution branch: smallest face size (px), probability, JPEG probability and quality."""
|
| 26 |
+
|
| 27 |
+
min_px: int
|
| 28 |
+
p: float
|
| 29 |
+
jpeg_p: float
|
| 30 |
+
jpeg_q: int
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def photometric(im, rng, grey_p=0.05):
|
| 34 |
+
"""Return the image with random photometric changes: brightness to JPEG re-encoding."""
|
| 35 |
+
x = im.astype(np.float32) / 255.0
|
| 36 |
+
if rng.random() < 0.9:
|
| 37 |
+
x = x * rng.uniform(0.6, 1.4)
|
| 38 |
+
m = x.mean()
|
| 39 |
+
x = (x - m) * rng.uniform(0.6, 1.4) + m
|
| 40 |
+
x = x * rng.uniform(0.85, 1.15, 3).astype(np.float32)
|
| 41 |
+
x = np.clip(x, 0, 1) ** rng.uniform(0.7, 1.4)
|
| 42 |
+
if rng.random() < 0.5: # hue / saturation
|
| 43 |
+
hsv = cv2.cvtColor((np.clip(x, 0, 1) * 255).astype(np.uint8), cv2.COLOR_BGR2HSV)
|
| 44 |
+
hsv = hsv.astype(np.float32)
|
| 45 |
+
hsv[:, :, 0] = (hsv[:, :, 0] + rng.uniform(-8, 8)) % 180
|
| 46 |
+
hsv[:, :, 1] = np.clip(hsv[:, :, 1] * rng.uniform(0.7, 1.3), 0, 255)
|
| 47 |
+
x = cv2.cvtColor(hsv.astype(np.uint8), cv2.COLOR_HSV2BGR).astype(np.float32) / 255.0
|
| 48 |
+
if rng.random() < grey_p:
|
| 49 |
+
x = np.repeat(x.mean(2, keepdims=True), 3, axis=2)
|
| 50 |
+
if rng.random() < 0.3:
|
| 51 |
+
x = cv2.GaussianBlur(x, (0, 0), rng.uniform(0.5, 2.0))
|
| 52 |
+
if rng.random() < 0.2: # directional motion blur, length proportional to the side
|
| 53 |
+
n = int(rng.integers(3, max(4, x.shape[0] // 20)))
|
| 54 |
+
k = np.zeros((n, n), np.float32)
|
| 55 |
+
cv2.line(k, (0, n // 2), (n - 1, n // 2), 1.0, 1)
|
| 56 |
+
k = cv2.warpAffine(
|
| 57 |
+
k, cv2.getRotationMatrix2D((n / 2, n / 2), rng.uniform(0, 180), 1), (n, n)
|
| 58 |
+
)
|
| 59 |
+
x = cv2.filter2D(x, -1, k / max(k.sum(), 1e-6))
|
| 60 |
+
if rng.random() < 0.3:
|
| 61 |
+
x = x + rng.normal(0, rng.uniform(0.01, 0.05), x.shape).astype(np.float32)
|
| 62 |
+
x = (np.clip(x, 0, 1) * 255).astype(np.uint8)
|
| 63 |
+
if rng.random() < 0.3:
|
| 64 |
+
x = jpeg(x, int(rng.integers(25, 90)))
|
| 65 |
+
return x
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def jpeg(im, quality):
|
| 69 |
+
"""Return the image after a JPEG encode-decode round trip at the given quality."""
|
| 70 |
+
return cv2.imdecode(
|
| 71 |
+
cv2.imencode(".jpg", im, [cv2.IMWRITE_JPEG_QUALITY, quality])[1], cv2.IMREAD_COLOR
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@functools.lru_cache(maxsize=4)
|
| 76 |
+
def grid(R):
|
| 77 |
+
"""Return the (yy, xx) pixel coordinates of an R x R image as float32 (cached per size)."""
|
| 78 |
+
return np.mgrid[0:R, 0:R].astype(np.float32)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def swap_bg(im, mask, bg, rng):
|
| 82 |
+
"""Return the person (mask 255) over a background crop, with a feathered edge."""
|
| 83 |
+
a = cv2.GaussianBlur(mask.astype(np.float32) / 255.0, (0, 0), rng.uniform(0.3, 1.0))[:, :, None]
|
| 84 |
+
if rng.random() < 0.5:
|
| 85 |
+
bg = bg[:, ::-1]
|
| 86 |
+
return (im * a + bg * (1 - a)).astype(np.uint8)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def relight(im, rng, mask=None):
|
| 90 |
+
"""Return the image relit by a directional light with random level, colour cast and shadow."""
|
| 91 |
+
# gain ramp e^(k d.p) across the image, level 0.25-1.1 of the original, with p 0.3 a soft-edged
|
| 92 |
+
# shadow polygon; applied to the person only when a mask is given
|
| 93 |
+
R = im.shape[0]
|
| 94 |
+
yy, xx = grid(R) / (R - 1) * 2 - 1
|
| 95 |
+
t = rng.uniform(0, 2 * np.pi)
|
| 96 |
+
g = np.exp(rng.uniform(0.3, 1.6) * (np.cos(t) * xx + np.sin(t) * yy))
|
| 97 |
+
g = g / g.mean() * rng.uniform(0.25, 1.1)
|
| 98 |
+
if rng.random() < 0.3:
|
| 99 |
+
m = np.zeros((R, R), np.uint8)
|
| 100 |
+
pts = (rng.uniform(-0.2, 1.2, (int(rng.integers(3, 6)), 2)) * R).astype(np.int32)
|
| 101 |
+
cv2.fillPoly(m, [cv2.convexHull(pts)], 1)
|
| 102 |
+
m = cv2.GaussianBlur(m.astype(np.float32), (0, 0), rng.uniform(1, 6))
|
| 103 |
+
g = g * (1 - m * rng.uniform(0.3, 0.7))
|
| 104 |
+
g = g[:, :, None] * rng.uniform(0.8, 1.2, 3).astype(np.float32)
|
| 105 |
+
lit = np.clip(im.astype(np.float32) * g, 0, 255)
|
| 106 |
+
if mask is not None:
|
| 107 |
+
a = (mask.astype(np.float32) / 255.0)[:, :, None]
|
| 108 |
+
lit = lit * a + im * (1 - a)
|
| 109 |
+
return lit.astype(np.uint8)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def glasses(im, rng, e1, e2):
|
| 113 |
+
"""Return the image with dark sunglasses over the two eye centres, one highlight on a lens."""
|
| 114 |
+
d = float(np.linalg.norm(e2 - e1))
|
| 115 |
+
if not (np.isfinite(d) and d > 4):
|
| 116 |
+
return im
|
| 117 |
+
ang = float(np.degrees(np.arctan2(e2[1] - e1[1], e2[0] - e1[0])))
|
| 118 |
+
ax = (int(d * rng.uniform(0.3, 0.45)), int(d * rng.uniform(0.2, 0.35)))
|
| 119 |
+
m = np.zeros(im.shape[:2], np.uint8)
|
| 120 |
+
for e in (e1, e2):
|
| 121 |
+
cv2.ellipse(m, (int(e[0]), int(e[1])), ax, ang, 0, 360, 1, -1)
|
| 122 |
+
cv2.line(m, tuple(int(v) for v in e1), tuple(int(v) for v in e2), 1, max(1, int(d * 0.06)))
|
| 123 |
+
lens = np.full(im.shape, rng.integers(0, 70, 3), np.float32)
|
| 124 |
+
yy, xx = grid(im.shape[0])
|
| 125 |
+
hl = np.exp(-(((xx - e1[0]) * 0.7 + (yy - e1[1])) ** 2) / (2 * (0.15 * d) ** 2))
|
| 126 |
+
lens = lens + hl[:, :, None] * rng.uniform(0, 120)
|
| 127 |
+
a = m[:, :, None].astype(np.float32) * rng.uniform(0.8, 1.0)
|
| 128 |
+
return np.clip(im * (1 - a) + lens * a, 0, 255).astype(np.uint8)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def occlude_real(im, rng, src, centre, size):
|
| 132 |
+
"""Return the image with a real-texture occluder (ellipse, polygon or band) near the face."""
|
| 133 |
+
# the occluder spans 20-50 % of the face size and sits within half a face size of the centre
|
| 134 |
+
R = im.shape[0]
|
| 135 |
+
m = np.zeros((R, R), np.uint8)
|
| 136 |
+
c = centre + rng.uniform(-0.5, 0.5, 2) * size
|
| 137 |
+
w, h = rng.uniform(0.1, 0.25, 2) * size
|
| 138 |
+
k = rng.random()
|
| 139 |
+
if k < 0.4:
|
| 140 |
+
cv2.ellipse(
|
| 141 |
+
m, (int(c[0]), int(c[1])), (int(w), int(h)), float(rng.uniform(0, 180)), 0, 360, 1, -1
|
| 142 |
+
)
|
| 143 |
+
elif k < 0.8:
|
| 144 |
+
pts = c + rng.uniform(-1, 1, (int(rng.integers(4, 8)), 2)) * [w, h]
|
| 145 |
+
cv2.fillPoly(m, [cv2.convexHull(pts.astype(np.int32))], 1)
|
| 146 |
+
else: # a band from one side, its edge 10-45 % of the face size off-centre (scarf, collar)
|
| 147 |
+
t = rng.uniform(0, 2 * np.pi)
|
| 148 |
+
n = np.array([np.cos(t), np.sin(t)])
|
| 149 |
+
e = centre + n * rng.uniform(0.1, 0.45) * size
|
| 150 |
+
yy, xx = grid(R)
|
| 151 |
+
m = (((xx - e[0]) * n[0] + (yy - e[1]) * n[1]) > 0).astype(np.uint8)
|
| 152 |
+
a = cv2.GaussianBlur(m.astype(np.float32), (0, 0), rng.uniform(0.3, 1.0))[:, :, None]
|
| 153 |
+
return (im * (1 - a) + src * a).astype(np.uint8)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def augment(im, uv, rng, res, train, aff, lowres, hook=None, shift=0.1, occ_p=1.0, grey_p=0.05):
|
| 157 |
+
"""Return (res x res image, labels) after affine, hook, low-res, photometric and occluders."""
|
| 158 |
+
# aff: Affine; lowres: LowRes; hook(im, uv, M) runs after the affine (data.wild.Wild.apply).
|
| 159 |
+
# shift, occ_p and grey_p at their defaults reproduce the studio recipe's random stream exactly.
|
| 160 |
+
S = im.shape[0]
|
| 161 |
+
if train:
|
| 162 |
+
s = rng.uniform(aff.scale_min, aff.scale_max)
|
| 163 |
+
a = (
|
| 164 |
+
rng.uniform(-aff.rotation_deg, aff.rotation_deg)
|
| 165 |
+
if rng.random() < aff.rotation_p
|
| 166 |
+
else 0.0
|
| 167 |
+
)
|
| 168 |
+
t = rng.uniform(-shift, shift, 2) * res
|
| 169 |
+
else:
|
| 170 |
+
s, a, t = 1.0, 0.0, np.zeros(2)
|
| 171 |
+
M = cv2.getRotationMatrix2D((S / 2, S / 2), a, s * res / S)
|
| 172 |
+
M[:, 2] += (res - S) / 2 + t
|
| 173 |
+
border = [int(v) for v in rng.integers(0, 256, 3)] if train else [0, 0, 0]
|
| 174 |
+
im = cv2.warpAffine(im, M, (res, res), flags=cv2.INTER_LINEAR, borderValue=border)
|
| 175 |
+
uv = uv @ M[:, :2].T + M[:, 2]
|
| 176 |
+
if hook is not None:
|
| 177 |
+
im = hook(im, uv, M)
|
| 178 |
+
if train:
|
| 179 |
+
if lowres.min_px < res and rng.random() < lowres.p:
|
| 180 |
+
# a small face in phone video: down to s px and back up (labels unmoved); with
|
| 181 |
+
# probability jpeg_p, JPEG-compress AT the small size (video is encoded at its own
|
| 182 |
+
# resolution, so its blocks are large after upscaling)
|
| 183 |
+
s = int(rng.integers(lowres.min_px, res))
|
| 184 |
+
im = cv2.resize(im, (s, s), interpolation=cv2.INTER_AREA)
|
| 185 |
+
if lowres.jpeg_p > 0 and rng.random() < lowres.jpeg_p:
|
| 186 |
+
im = jpeg(im, int(rng.integers(lowres.jpeg_q, 91)))
|
| 187 |
+
im = cv2.resize(im, (res, res))
|
| 188 |
+
im = photometric(im, rng, grey_p)
|
| 189 |
+
if occ_p >= 1.0 or rng.random() < occ_p:
|
| 190 |
+
im = occlude(im, rng)
|
| 191 |
+
return im, uv
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def occlude(im, rng):
|
| 195 |
+
"""Return the image with zero to two random occluders (solid or noise; rectangle or ellipse)."""
|
| 196 |
+
if rng.random() > 0.35:
|
| 197 |
+
return im
|
| 198 |
+
R = im.shape[0]
|
| 199 |
+
for _ in range(int(rng.integers(1, 3))):
|
| 200 |
+
w, h = (rng.uniform(0.1, 0.35, 2) * R).astype(int)
|
| 201 |
+
cx, cy = (rng.uniform(0.1, 0.9, 2) * R).astype(int)
|
| 202 |
+
if rng.random() < 0.6:
|
| 203 |
+
patch = np.full((h, w, 3), rng.integers(0, 256, 3), np.uint8)
|
| 204 |
+
else:
|
| 205 |
+
patch = rng.integers(0, 256, (h, w, 3)).astype(np.uint8)
|
| 206 |
+
mask = np.zeros((R, R), np.uint8)
|
| 207 |
+
if rng.random() < 0.5:
|
| 208 |
+
cv2.rectangle(mask, (cx - w // 2, cy - h // 2), (cx + w // 2, cy + h // 2), 1, -1)
|
| 209 |
+
else:
|
| 210 |
+
cv2.ellipse(mask, (cx, cy), (w // 2, h // 2), float(rng.uniform(0, 180)), 0, 360, 1, -1)
|
| 211 |
+
full = np.zeros_like(im)
|
| 212 |
+
y0, x0 = max(cy - h // 2, 0), max(cx - w // 2, 0)
|
| 213 |
+
ph, pw = min(h, R - y0), min(w, R - x0)
|
| 214 |
+
full[y0 : y0 + ph, x0 : x0 + pw] = patch[:ph, :pw]
|
| 215 |
+
im = np.where(mask[:, :, None] > 0, full, im)
|
| 216 |
+
return im
|
train/data/ava.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The studio set: items over extracted Ava-256 captures (<root>/<capture>/labels.npz + jpgs)."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
import cv2
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from facepoints.model import to_tensor
|
| 10 |
+
from facepoints.train.data.augment import Affine, LowRes, augment
|
| 11 |
+
|
| 12 |
+
DEPTH_SPAN_MAX_MM = 400.0 # a real head spans 75-347 mm; a broken registration spans 1e5
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class PointData(torch.utils.data.Dataset):
|
| 16 |
+
"""Studio items: (image (3, res, res), uv (P, 2) px, ok (P,) labelled, vis (P,) visible, k)."""
|
| 17 |
+
|
| 18 |
+
# Seeded per (seed, epoch, item) and, with fresh=True, per call too: without the per-call
|
| 19 |
+
# counter
|
| 20 |
+
# every image keeps ONE augmentation for the whole run, because train.py never advances this
|
| 21 |
+
# set's epoch and persistent loader workers hold their own copy of the dataset object anyway.
|
| 22 |
+
def __init__(
|
| 23 |
+
self,
|
| 24 |
+
root,
|
| 25 |
+
captures,
|
| 26 |
+
res=256,
|
| 27 |
+
train=True,
|
| 28 |
+
limit=0,
|
| 29 |
+
seed=0,
|
| 30 |
+
min_vis=0.0,
|
| 31 |
+
lowres=LowRes(64, 0.5, 0.0, 30),
|
| 32 |
+
aff=Affine(0.5, 1.5, 80.0, 0.6),
|
| 33 |
+
wild=None,
|
| 34 |
+
fresh=False,
|
| 35 |
+
):
|
| 36 |
+
self.root, self.res, self.train, self.seed, self.epoch = root, res, train, seed, 0
|
| 37 |
+
self.fresh, self.calls = fresh, 0
|
| 38 |
+
self.lowres, self.aff = lowres, aff
|
| 39 |
+
self.wild = wild if (wild is not None and wild.on() and train) else None
|
| 40 |
+
self.lab, self.items, self.dropped = {}, [], 0
|
| 41 |
+
for cap in captures:
|
| 42 |
+
p = os.path.join(root, cap, "labels.npz")
|
| 43 |
+
if not os.path.exists(p):
|
| 44 |
+
continue
|
| 45 |
+
z = np.load(p)
|
| 46 |
+
self.lab[cap] = {k: z[k] for k in z.files}
|
| 47 |
+
keep = z["vis"].mean(1) >= min_vis # rear cameras see no face point at all
|
| 48 |
+
zz = np.where(np.isfinite(z["z"]), z["z"], np.nan)
|
| 49 |
+
with np.errstate(all="ignore"):
|
| 50 |
+
keep &= (np.nanmax(zz, 1) - np.nanmin(zz, 1)) < DEPTH_SPAN_MAX_MM
|
| 51 |
+
self.dropped += int((~keep).sum())
|
| 52 |
+
self.items += [(cap, i) for i in np.flatnonzero(keep)]
|
| 53 |
+
if limit and len(self.items) > limit: # evenly spaced subset
|
| 54 |
+
step = len(self.items) / limit
|
| 55 |
+
self.items = [self.items[int(j * step)] for j in range(limit)]
|
| 56 |
+
self.cls = next(iter(self.lab.values()))["cls"] if self.lab else None
|
| 57 |
+
self.masks = set() # (capture, "<frame>_<cam>") with a <frame>_<cam>_m.png person mask
|
| 58 |
+
if self.wild is not None and self.wild.bg_p > 0:
|
| 59 |
+
for cap in self.lab:
|
| 60 |
+
for f in os.listdir(os.path.join(root, cap)):
|
| 61 |
+
if f.endswith("_m.png"):
|
| 62 |
+
self.masks.add((cap, f[:-6]))
|
| 63 |
+
self.n_masked = (
|
| 64 |
+
sum(
|
| 65 |
+
(c, f"{int(self.lab[c]['frame'][i]):06d}_{int(self.lab[c]['cam'][i])}")
|
| 66 |
+
in self.masks
|
| 67 |
+
for c, i in self.items
|
| 68 |
+
)
|
| 69 |
+
if self.masks
|
| 70 |
+
else 0
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
def __len__(self):
|
| 74 |
+
"""Return the number of items."""
|
| 75 |
+
return len(self.items)
|
| 76 |
+
|
| 77 |
+
def meta(self, k):
|
| 78 |
+
"""Return (capture, row, that capture's label arrays) of item k."""
|
| 79 |
+
cap, i = self.items[k]
|
| 80 |
+
return cap, i, self.lab[cap]
|
| 81 |
+
|
| 82 |
+
def __getitem__(self, k):
|
| 83 |
+
"""Return (image, uv, ok, vis, k) for item k, augmented when training."""
|
| 84 |
+
cv2.setNumThreads(0)
|
| 85 |
+
cap, i, z = self.meta(k)
|
| 86 |
+
stem = f"{int(z['frame'][i]):06d}_{int(z['cam'][i])}"
|
| 87 |
+
im = cv2.imread(os.path.join(self.root, cap, stem + ".jpg"), cv2.IMREAD_COLOR)
|
| 88 |
+
uv = z["uv"][i].astype(np.float64)
|
| 89 |
+
self.calls += 1
|
| 90 |
+
rng = np.random.default_rng(
|
| 91 |
+
[self.seed, self.epoch, k] + ([self.calls] if self.fresh else [])
|
| 92 |
+
)
|
| 93 |
+
hook = None
|
| 94 |
+
if self.wild is not None: # off (every probability 0) leaves the random stream unchanged
|
| 95 |
+
mask = None
|
| 96 |
+
if (cap, stem) in self.masks:
|
| 97 |
+
mask = cv2.imread(
|
| 98 |
+
os.path.join(self.root, cap, stem + "_m.png"), cv2.IMREAD_GRAYSCALE
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
def hook(im2, uv2, M): # after the affine: the mask follows it, border = background
|
| 102 |
+
R = im2.shape[0]
|
| 103 |
+
m2 = None if mask is None else cv2.warpAffine(mask, M, (R, R), borderValue=0)
|
| 104 |
+
return self.wild.apply(im2, uv2, rng, m2)
|
| 105 |
+
|
| 106 |
+
im, uv = augment(im, uv, rng, self.res, self.train, self.aff, self.lowres, hook)
|
| 107 |
+
ok = np.isfinite(uv).all(1)
|
| 108 |
+
uv = np.where(ok[:, None], uv, 0.0).astype(np.float32)
|
| 109 |
+
return (
|
| 110 |
+
to_tensor(im),
|
| 111 |
+
torch.from_numpy(uv),
|
| 112 |
+
torch.from_numpy(ok),
|
| 113 |
+
torch.from_numpy(z["vis"][i].copy()),
|
| 114 |
+
k,
|
| 115 |
+
)
|
train/data/wild.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""In-the-wild faces for the 68 / 98-point side heads, and the in-the-wild augmentation extras."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
import cv2
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from facepoints.model import to_tensor
|
| 10 |
+
from facepoints.train.data.augment import (
|
| 11 |
+
Affine,
|
| 12 |
+
LowRes,
|
| 13 |
+
augment,
|
| 14 |
+
glasses,
|
| 15 |
+
occlude_real,
|
| 16 |
+
relight,
|
| 17 |
+
swap_bg,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Wild:
|
| 22 |
+
"""The in-the-wild extras: background swap, relighting, real occluders, sunglasses."""
|
| 23 |
+
|
| 24 |
+
def __init__(self, bg_dir="", bg_p=0.0, relight_p=0.0, occ2_p=0.0, glasses_p=0.0, eyes=None):
|
| 25 |
+
self.bg = sorted(os.path.join(bg_dir, f) for f in os.listdir(bg_dir)) if bg_dir else []
|
| 26 |
+
self.bg_p, self.relight_p, self.occ2_p, self.glasses_p = bg_p, relight_p, occ2_p, glasses_p
|
| 27 |
+
self.eyes = eyes # (point rows of one eye, point rows of the other) for the sunglasses
|
| 28 |
+
if (bg_p > 0 or occ2_p > 0) and not self.bg:
|
| 29 |
+
raise ValueError("bg_p / occ2_p need a backgrounds dir")
|
| 30 |
+
|
| 31 |
+
def on(self):
|
| 32 |
+
"""Return whether any extra has a non-zero probability."""
|
| 33 |
+
return max(self.bg_p, self.relight_p, self.occ2_p, self.glasses_p) > 0
|
| 34 |
+
|
| 35 |
+
def background(self, rng):
|
| 36 |
+
"""Return a random background crop (BGR)."""
|
| 37 |
+
return cv2.imread(self.bg[int(rng.integers(len(self.bg)))], cv2.IMREAD_COLOR)
|
| 38 |
+
|
| 39 |
+
def apply(self, im, uv, rng, mask=None, eyes=None):
|
| 40 |
+
"""Return the image with the enabled extras drawn (after the affine; labels unchanged)."""
|
| 41 |
+
eyes = self.eyes if eyes is None else eyes
|
| 42 |
+
if mask is not None and self.bg_p > 0 and rng.random() < self.bg_p:
|
| 43 |
+
bg = cv2.resize(self.background(rng), im.shape[1::-1])
|
| 44 |
+
im = swap_bg(im, mask, bg, rng)
|
| 45 |
+
if self.relight_p > 0 and rng.random() < self.relight_p:
|
| 46 |
+
im = relight(im, rng, mask)
|
| 47 |
+
if self.glasses_p > 0 and eyes is not None and rng.random() < self.glasses_p:
|
| 48 |
+
e1, e2 = (np.nanmean(uv[r], 0) for r in eyes)
|
| 49 |
+
im = glasses(im, rng, e1, e2)
|
| 50 |
+
if self.occ2_p > 0 and rng.random() < self.occ2_p:
|
| 51 |
+
f = uv[np.isfinite(uv).all(1)]
|
| 52 |
+
if len(f):
|
| 53 |
+
lo, hi = f.min(0), f.max(0)
|
| 54 |
+
src = cv2.resize(self.background(rng), im.shape[1::-1])
|
| 55 |
+
im = occlude_real(im, rng, src, (lo + hi) / 2, float((hi - lo).max()))
|
| 56 |
+
return im
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
KMAX = 98 # the widest wild label set (WFLW); 300W's 68 are padded with ok = False
|
| 60 |
+
MILD_BIT = 1 << 30 # added to the returned item index when the item drew the mild augmentation
|
| 61 |
+
# the mild augmentation (mild_p): scale +-10 %, rotation +-18 deg always, shift +-5 %, no
|
| 62 |
+
# low-resolution branch, no occluders, grey 20 %, no in-the-wild extras
|
| 63 |
+
MILD = dict(
|
| 64 |
+
aff=Affine(0.9, 1.1, 18.0, 1.0),
|
| 65 |
+
lowres=LowRes(64, 0.0, 0.0, 30),
|
| 66 |
+
shift=0.05,
|
| 67 |
+
occ_p=0.0,
|
| 68 |
+
grey_p=0.2,
|
| 69 |
+
)
|
| 70 |
+
# iBUG 68 mirror map (jaw 0-16, brows 17-26, nose 27-35, eyes 36-47, mouth 48-67)
|
| 71 |
+
FLIP68 = np.array(
|
| 72 |
+
list(range(16, -1, -1))
|
| 73 |
+
+ list(range(26, 16, -1))
|
| 74 |
+
+ [27, 28, 29, 30, 35, 34, 33, 32, 31]
|
| 75 |
+
+ [45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, 40]
|
| 76 |
+
+ [54, 53, 52, 51, 50, 49, 48, 59, 58, 57, 56, 55, 64, 63, 62, 61, 60, 67, 66, 65]
|
| 77 |
+
)
|
| 78 |
+
WILD_EYES = { # eye contour rows per label set (the sunglasses)
|
| 79 |
+
"wflw": (np.arange(60, 68), np.arange(68, 76)),
|
| 80 |
+
"300w": (np.arange(36, 42), np.arange(42, 48)),
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class WildData(torch.utils.data.Dataset):
|
| 85 |
+
"""Wild items: (image, uv (KMAX, 2) px, ok (KMAX,), set index, item index [+ MILD_BIT])."""
|
| 86 |
+
|
| 87 |
+
# Layout (prep/wild_prep.py): <root>/<name>/{labels.npz, <i>.jpg}, one side head per name. The
|
| 88 |
+
# same affine / low-resolution / photometric / occluder recipe as the studio crops, plus the
|
| 89 |
+
# extras except the background swap (the photo is its own background).
|
| 90 |
+
def __init__(
|
| 91 |
+
self,
|
| 92 |
+
root,
|
| 93 |
+
names,
|
| 94 |
+
res,
|
| 95 |
+
train,
|
| 96 |
+
seed=0,
|
| 97 |
+
lowres=LowRes(64, 0.5, 0.0, 30),
|
| 98 |
+
aff=Affine(0.5, 1.5, 80.0, 0.6),
|
| 99 |
+
wild=None,
|
| 100 |
+
flip=None,
|
| 101 |
+
shift=0.1,
|
| 102 |
+
occ_p=1.0,
|
| 103 |
+
grey_p=0.05,
|
| 104 |
+
fresh=False,
|
| 105 |
+
mild_p=0.0,
|
| 106 |
+
):
|
| 107 |
+
self.root, self.names, self.res, self.train = root, names, res, train
|
| 108 |
+
self.seed, self.epoch, self.lowres, self.aff = seed, 0, lowres, aff
|
| 109 |
+
self.flip = flip # {set name: mirror index map}, training only
|
| 110 |
+
self.shift, self.occ_p, self.grey_p = shift, occ_p, grey_p
|
| 111 |
+
self.fresh, self.calls, self.mild_p = fresh, 0, mild_p # fresh: see PointData
|
| 112 |
+
self.wild = wild if (wild is not None and wild.on() and train) else None
|
| 113 |
+
self.lab, self.items = {}, []
|
| 114 |
+
for j, n in enumerate(names):
|
| 115 |
+
z = np.load(os.path.join(root, n, "labels.npz"))
|
| 116 |
+
self.lab[n] = {k: z[k] for k in z.files}
|
| 117 |
+
keep = ~z["val"] if train else z["val"]
|
| 118 |
+
self.items += [(j, int(i)) for i in np.flatnonzero(keep)]
|
| 119 |
+
self.K = [int(self.lab[n]["uv"].shape[1]) for n in names] # points per set
|
| 120 |
+
|
| 121 |
+
def __len__(self):
|
| 122 |
+
"""Return the number of items."""
|
| 123 |
+
return len(self.items)
|
| 124 |
+
|
| 125 |
+
def __getitem__(self, k):
|
| 126 |
+
"""Return (image, uv, ok, set index, item index) for item k, augmented when training."""
|
| 127 |
+
cv2.setNumThreads(0)
|
| 128 |
+
j, i = self.items[k]
|
| 129 |
+
z = self.lab[self.names[j]]
|
| 130 |
+
im = cv2.imread(os.path.join(self.root, self.names[j], f"{i:05d}.jpg"), cv2.IMREAD_COLOR)
|
| 131 |
+
uv = z["uv"][i].astype(np.float64)
|
| 132 |
+
self.calls += 1
|
| 133 |
+
rng = np.random.default_rng(
|
| 134 |
+
[self.seed, self.epoch, k, 7] + ([self.calls] if self.fresh else [])
|
| 135 |
+
)
|
| 136 |
+
fm = self.flip.get(self.names[j]) if self.flip else None
|
| 137 |
+
if self.train and fm is not None and rng.random() < 0.5:
|
| 138 |
+
im = np.ascontiguousarray(im[:, ::-1])
|
| 139 |
+
uv[:, 0] = im.shape[1] - 1 - uv[:, 0]
|
| 140 |
+
uv = uv[fm]
|
| 141 |
+
mild = self.train and self.mild_p > 0 and rng.random() < self.mild_p
|
| 142 |
+
hook = None
|
| 143 |
+
if self.wild is not None and not mild:
|
| 144 |
+
eyes = WILD_EYES[self.names[j]]
|
| 145 |
+
|
| 146 |
+
def hook(im2, uv2, M):
|
| 147 |
+
return self.wild.apply(im2, uv2, rng, eyes=eyes)
|
| 148 |
+
|
| 149 |
+
if mild:
|
| 150 |
+
im, uv = augment(im, uv, rng, self.res, True, MILD["aff"], MILD["lowres"], None,
|
| 151 |
+
MILD["shift"], MILD["occ_p"], MILD["grey_p"]) # fmt: skip
|
| 152 |
+
else:
|
| 153 |
+
im, uv = augment(im, uv, rng, self.res, self.train, self.aff, self.lowres, hook,
|
| 154 |
+
self.shift, self.occ_p, self.grey_p) # fmt: skip
|
| 155 |
+
out = np.zeros((KMAX, 2), np.float32)
|
| 156 |
+
ok = np.zeros(KMAX, bool)
|
| 157 |
+
out[: len(uv)], ok[: len(uv)] = uv, np.isfinite(uv).all(1)
|
| 158 |
+
return (
|
| 159 |
+
to_tensor(im),
|
| 160 |
+
torch.from_numpy(out),
|
| 161 |
+
torch.from_numpy(ok),
|
| 162 |
+
j,
|
| 163 |
+
k + (MILD_BIT if mild else 0),
|
| 164 |
+
)
|
train/evaluate.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Validation: the 620 points against mesh labels (px, mm), the side heads against hand labels."""
|
| 2 |
+
|
| 3 |
+
# Each function returns one report dict per call; the trainer appends it to val.json and prints the
|
| 4 |
+
# VAL lines. Both write a picture grid through draw_grid().
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
import cv2
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
from facepoints.model import LOG_SCALE_RANGE, MEAN, STD
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def autocast(dev, enabled):
|
| 15 |
+
"""Return a float16 autocast context for device dev (a no-op on the CPU or when disabled)."""
|
| 16 |
+
kind = dev.split(":")[0]
|
| 17 |
+
return torch.autocast(kind, dtype=torch.float16, enabled=bool(enabled) and kind == "cuda")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def mm_scale(ds, ks):
|
| 21 |
+
"""Return (n, P) px-to-mm factors at label depth for items ks (NaN without a focal length)."""
|
| 22 |
+
S = []
|
| 23 |
+
for k in ks:
|
| 24 |
+
_, i, z = ds.meta(k)
|
| 25 |
+
side = float(z["box"][i][2] - z["box"][i][0])
|
| 26 |
+
fx = float(z["fx"][i]) if "fx" in z else np.nan
|
| 27 |
+
S.append(side / ds.res * z["z"][i] / fx)
|
| 28 |
+
return np.array(S)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def draw_grid(shown, res, path, scale_every=0):
|
| 32 |
+
"""Write up to 8 panels of labels (green) and predictions (red), with scale circles if asked."""
|
| 33 |
+
T = res
|
| 34 |
+
sheet = np.zeros((2 * T, 4 * T, 3), np.uint8)
|
| 35 |
+
for j, (x, uv, ok, o) in enumerate(shown[:8]):
|
| 36 |
+
im = ((x.transpose(1, 2, 0) * STD + MEAN) * 255).clip(0, 255)[:, :, ::-1]
|
| 37 |
+
im = np.ascontiguousarray(im.astype(np.uint8))
|
| 38 |
+
for p in np.flatnonzero(ok):
|
| 39 |
+
cv2.circle(im, (int(uv[p, 0]), int(uv[p, 1])), 1, (0, 255, 0), -1)
|
| 40 |
+
cv2.circle(im, (int(o[p, 0]), int(o[p, 1])), 1, (0, 0, 255), -1)
|
| 41 |
+
if scale_every and p % scale_every == 0:
|
| 42 |
+
rad = int(min(np.exp(np.clip(o[p, 2], *LOG_SCALE_RANGE)), T / 4))
|
| 43 |
+
cv2.circle(im, (int(o[p, 0]), int(o[p, 1])), rad, (0, 0, 255), 1)
|
| 44 |
+
sheet[(j // 4) * T : (j // 4 + 1) * T, (j % 4) * T : (j % 4 + 1) * T] = im
|
| 45 |
+
cv2.imwrite(path, sheet, [cv2.IMWRITE_JPEG_QUALITY, 88])
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def evaluate(net, ds, cfg, dev, cls, region, out_dir, it):
|
| 49 |
+
"""Return per class and region the median error (px, mm) over visible and over all labelled."""
|
| 50 |
+
net.eval()
|
| 51 |
+
dl = torch.utils.data.DataLoader(ds, cfg.batch, num_workers=min(cfg.workers, 4))
|
| 52 |
+
E, OK, VIS, MM, SC, shown = [], [], [], [], [], []
|
| 53 |
+
with torch.no_grad():
|
| 54 |
+
for x, uv, ok, vis, ks in dl:
|
| 55 |
+
with autocast(dev, cfg.amp):
|
| 56 |
+
o = net(x.to(dev, non_blocking=True)).float().cpu()
|
| 57 |
+
E.append((o[..., :2] - uv).norm(dim=-1).numpy())
|
| 58 |
+
OK.append(ok.numpy())
|
| 59 |
+
VIS.append(vis.numpy())
|
| 60 |
+
MM.append(mm_scale(ds, ks.tolist()))
|
| 61 |
+
SC.append(torch.exp(o[..., 2].clamp(*LOG_SCALE_RANGE)).numpy())
|
| 62 |
+
if len(shown) < 8:
|
| 63 |
+
for j in range(min(8 - len(shown), len(x))):
|
| 64 |
+
shown.append((x[j].numpy(), uv[j].numpy(), ok[j].numpy(), o[j].numpy()))
|
| 65 |
+
E, OK, VIS, MM, SC = (np.concatenate(a) for a in (E, OK, VIS, MM, SC))
|
| 66 |
+
MMe = E * MM
|
| 67 |
+
groups = {"all": np.ones(len(cls), bool)}
|
| 68 |
+
for c in dict.fromkeys(cls):
|
| 69 |
+
groups[str(c)] = cls == c
|
| 70 |
+
for r in dict.fromkeys(region):
|
| 71 |
+
if r:
|
| 72 |
+
groups[str(r)] = region == r
|
| 73 |
+
rep = {"it": it, "n_images": int(len(E))}
|
| 74 |
+
for g, m in groups.items():
|
| 75 |
+
sel_v = OK & VIS & m[None]
|
| 76 |
+
sel_a = OK & m[None]
|
| 77 |
+
if sel_a.sum() == 0:
|
| 78 |
+
continue
|
| 79 |
+
rep[g] = {
|
| 80 |
+
"vis_px": float(np.median(E[sel_v])) if sel_v.any() else None,
|
| 81 |
+
"vis_mm": float(np.nanmedian(MMe[sel_v])) if sel_v.any() else None,
|
| 82 |
+
"all_px": float(np.median(E[sel_a])),
|
| 83 |
+
"all_mm": float(np.nanmedian(MMe[sel_a])),
|
| 84 |
+
"sigma_vis_px": float(np.median(SC[sel_v])) if sel_v.any() else None,
|
| 85 |
+
"n_vis": int(sel_v.sum()),
|
| 86 |
+
"n_all": int(sel_a.sum()),
|
| 87 |
+
}
|
| 88 |
+
draw_grid(shown, ds.res, os.path.join(out_dir, f"pred_grid_{it}.jpg"), scale_every=8)
|
| 89 |
+
net.train()
|
| 90 |
+
return rep
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def evaluate_wild(net, side, ds, cfg, dev, out_dir, it, side_rf=None, mp=None):
|
| 94 |
+
"""Return per wild set the side head's NME and FR@10 (refined too) and the 620 head's terms."""
|
| 95 |
+
# NME = mean point error / outer-eye-corner distance x 100; FR@10 = share of faces above 10.
|
| 96 |
+
# With mp (losses.Map620): medians of its four terms per face, the under-chin points' median
|
| 97 |
+
# signed distance to the jaw (% IOD, + = below) and, where pupils are labelled, the iris error.
|
| 98 |
+
net.eval()
|
| 99 |
+
dl = torch.utils.data.DataLoader(ds, cfg.batch, num_workers=min(cfg.workers, 4))
|
| 100 |
+
E = {n: [] for n in ds.names}
|
| 101 |
+
ER = {n: [] for n in ds.names}
|
| 102 |
+
T = {n: {k: [] for k in ("map", "jaw", "chin", "pupil", "dense", "uc")} for n in ds.names}
|
| 103 |
+
shown = []
|
| 104 |
+
with torch.no_grad():
|
| 105 |
+
for x, uv, _, j, _ in dl:
|
| 106 |
+
with autocast(dev, cfg.amp):
|
| 107 |
+
_, zt, fs = net.run(x.to(dev, non_blocking=True), 0)
|
| 108 |
+
for k, n in enumerate(ds.names):
|
| 109 |
+
sel = j == k
|
| 110 |
+
if not bool(sel.any()):
|
| 111 |
+
continue
|
| 112 |
+
K, ey = ds.K[k], ds.lab[n]["eyes"]
|
| 113 |
+
sd = sel.to(dev)
|
| 114 |
+
oc = side[n](zt[sd].float()).view(-1, K, 3)
|
| 115 |
+
o = oc.cpu()
|
| 116 |
+
g = uv[sel, :K]
|
| 117 |
+
io = (g[:, int(ey[0])] - g[:, int(ey[1])]).norm(dim=-1)
|
| 118 |
+
E[n].append(((o[..., :2] - g).norm(dim=-1).mean(1) / io * 100).numpy())
|
| 119 |
+
if side_rf is not None and len(side_rf):
|
| 120 |
+
r = side_rf[n]([f[sd].float() for f in fs], oc).cpu()
|
| 121 |
+
ER[n].append(((r[..., :2] - g).norm(dim=-1).mean(1) / io * 100).numpy())
|
| 122 |
+
if mp is not None:
|
| 123 |
+
mu6 = net.h[-1](zt[sd].float()).view(-1, net.n_points, 3)[..., :2]
|
| 124 |
+
gd = g.to(dev)
|
| 125 |
+
okd = torch.isfinite(gd).all(-1)
|
| 126 |
+
tmap, tjaw, tdense, tuc = mp.terms(n, mu6, gd, okd, per_face=True)
|
| 127 |
+
chin, pupil = mp.probe(n, mu6, gd, okd)
|
| 128 |
+
terms = (("map", tmap * mp.B0), ("jaw", tjaw * mp.B0), ("chin", chin),
|
| 129 |
+
("pupil", pupil), ("dense", tdense), ("uc", tuc * mp.B0)) # fmt: skip
|
| 130 |
+
for key, v in terms:
|
| 131 |
+
T[n][key].append(v.cpu().numpy().astype(np.float64))
|
| 132 |
+
for q in range(min(int(sel.sum()), 4 - sum(s[0] == n for s in shown))):
|
| 133 |
+
shown.append((n, x[sel][q].numpy(), g[q].numpy(), o[q].numpy()))
|
| 134 |
+
rep = {}
|
| 135 |
+
for n, e in E.items():
|
| 136 |
+
if not e:
|
| 137 |
+
continue
|
| 138 |
+
e = np.concatenate(e)
|
| 139 |
+
rep[n] = {"nme": float(e.mean()), "fr10": float((e > 10).mean() * 100), "n": len(e)}
|
| 140 |
+
if ER[n]:
|
| 141 |
+
rep[n]["nme_rf"] = float(np.concatenate(ER[n]).mean())
|
| 142 |
+
if T[n]["map"]:
|
| 143 |
+
t = {k: np.concatenate(v) for k, v in T[n].items()}
|
| 144 |
+
rep[n].update(
|
| 145 |
+
m620=float(np.median(t["map"])),
|
| 146 |
+
j620=float(np.nanmedian(t["jaw"])),
|
| 147 |
+
chin620=float(np.nanmedian(t["chin"])),
|
| 148 |
+
d620=float(np.nanmedian(t["dense"])),
|
| 149 |
+
u620=float(np.nanmedian(t["uc"])),
|
| 150 |
+
)
|
| 151 |
+
if np.isfinite(t["pupil"]).any():
|
| 152 |
+
rep[n]["pupil620"] = float(np.nanmedian(t["pupil"]))
|
| 153 |
+
panels = [(x, g, np.ones(len(g), bool), o) for _, x, g, o in shown[:8]]
|
| 154 |
+
draw_grid(panels, ds.res, os.path.join(out_dir, f"wild_grid_{it}.jpg"))
|
| 155 |
+
net.train()
|
| 156 |
+
return rep
|
train/export.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Export a training checkpoint to the release format: model.safetensors + config.json."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.train.export <ckpt.pt or best.pt> <out dir>
|
| 4 |
+
# Tensors are stored flat with the prefixes net. / side.<set>. / side_rf.<set>. ; config.json holds
|
| 5 |
+
# the architecture, the point classes in output order, the likelihood and its std factor, the side
|
| 6 |
+
# heads and the training keys as run (without local file paths). model.load_release reads them.
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
from dataclasses import fields
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from safetensors.torch import save_file
|
| 15 |
+
|
| 16 |
+
from facepoints.model import CONFIG, FORMAT, LOG_SCALE_RANGE, WEIGHTS, std_factor
|
| 17 |
+
from facepoints.train.config import TrainConfig
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def export_release(checkpoint_path, out_dir):
|
| 21 |
+
"""Write model.safetensors and config.json to out_dir from a checkpoint; return the config."""
|
| 22 |
+
ck = torch.load(checkpoint_path, map_location="cpu", weights_only=False) # a trainer checkpoint
|
| 23 |
+
a, classes = ck["args"], [str(c) for c in ck["cls"]]
|
| 24 |
+
tensors = {f"net.{k}": v.contiguous() for k, v in ck["model"].items()}
|
| 25 |
+
tensors.update({f"side.{k}": v.contiguous() for k, v in ck.get("side", {}).items()})
|
| 26 |
+
tensors.update({f"side_rf.{k}": v.contiguous() for k, v in ck.get("side_rf", {}).items()})
|
| 27 |
+
keep = {f.name for f in fields(TrainConfig)} - set(TrainConfig.PATH_KEYS)
|
| 28 |
+
config = {
|
| 29 |
+
"format": FORMAT,
|
| 30 |
+
"backbone": a["backbone"],
|
| 31 |
+
"input_px": int(a["res"]),
|
| 32 |
+
"neck": int(a["neck"]),
|
| 33 |
+
"stride": int(a.get("stride", 8)),
|
| 34 |
+
"n_points": len(classes),
|
| 35 |
+
"classes": classes,
|
| 36 |
+
"likelihood": a["nll"],
|
| 37 |
+
"nu": float(a.get("nu", 10.0)),
|
| 38 |
+
"std_factor": std_factor(a["nll"], float(a.get("nu", 10.0))),
|
| 39 |
+
"log_scale_range": list(LOG_SCALE_RANGE),
|
| 40 |
+
"side_heads": {k: int(v) for k, v in ck.get("side_k", {}).items()},
|
| 41 |
+
"side_refine": "side_rf" in ck,
|
| 42 |
+
"iterations": int(ck["it"]),
|
| 43 |
+
"training": {k: v for k, v in a.items() if k in keep},
|
| 44 |
+
}
|
| 45 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 46 |
+
save_file(tensors, os.path.join(out_dir, WEIGHTS), metadata={"format": "pt"})
|
| 47 |
+
with open(os.path.join(out_dir, CONFIG), "w") as fh:
|
| 48 |
+
json.dump(config, fh, indent=1)
|
| 49 |
+
return config
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def sha256(path):
|
| 53 |
+
"""Return the SHA-256 hex digest of a file."""
|
| 54 |
+
h = hashlib.sha256()
|
| 55 |
+
with open(path, "rb") as fh:
|
| 56 |
+
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
| 57 |
+
h.update(chunk)
|
| 58 |
+
return h.hexdigest()
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def main():
|
| 62 |
+
"""Export a checkpoint and print the weights file's size and SHA-256."""
|
| 63 |
+
if len(sys.argv) != 3:
|
| 64 |
+
sys.exit("usage: python -m facepoints.train.export <ckpt.pt> <out dir>")
|
| 65 |
+
config = export_release(sys.argv[1], sys.argv[2])
|
| 66 |
+
w = os.path.join(sys.argv[2], WEIGHTS)
|
| 67 |
+
mb = os.path.getsize(w) / 1e6
|
| 68 |
+
print(f"wrote {w} ({mb:.1f} MB, sha256 {sha256(w)}) and {CONFIG}: {config['n_points']} points, "
|
| 69 |
+
f"side heads {config['side_heads']}", flush=True) # fmt: skip
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
if __name__ == "__main__":
|
| 73 |
+
main()
|
train/losses.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Losses: the per-point likelihood, and the 620 head's terms on in-the-wild faces."""
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
|
| 6 |
+
from facepoints.model import LOG_SCALE_RANGE
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def gnll(out, uv, w, kind="gauss", nu=10.0):
|
| 10 |
+
"""Return the weighted mean negative log-likelihood of uv (B, P, 2) under out (B, P, 3)."""
|
| 11 |
+
# out = (x, y, log scale) per point; w (B, P) weights the points. kind: "gauss" (isotropic),
|
| 12 |
+
# "laplace" (per coordinate, scale b: std = sqrt(2) b) or "studentt" (per coordinate, nu degrees
|
| 13 |
+
# of freedom: std = s sqrt(nu / (nu - 2))). Consumers read the kind from the model's config.
|
| 14 |
+
mu, ls = out[..., :2], out[..., 2].clamp(*LOG_SCALE_RANGE)
|
| 15 |
+
if kind == "laplace":
|
| 16 |
+
nll = (mu - uv).abs().sum(-1) / torch.exp(ls) + 2 * ls
|
| 17 |
+
elif kind == "studentt":
|
| 18 |
+
z2 = ((mu - uv) / torch.exp(ls)[..., None]) ** 2
|
| 19 |
+
nll = 0.5 * (nu + 1) * torch.log1p(z2 / nu).sum(-1) + 2 * ls
|
| 20 |
+
else:
|
| 21 |
+
nll = ((mu - uv) ** 2).sum(-1) / (2 * torch.exp(2 * ls)) + ls
|
| 22 |
+
return (nll * w).sum() / w.sum().clamp(min=1.0)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def jaw_signed(p, jaw, c):
|
| 26 |
+
"""Return (signed distance (n, C) of p to the jaw polyline, valid (n, C)); + = outside."""
|
| 27 |
+
# p (n, C, 2), jaw (n, J, 2), c (n, 2) face centre: each segment's normal points away from c.
|
| 28 |
+
# A point is valid when its nearest polyline point is not past either end of the jaw (points
|
| 29 |
+
# beside the temples have no jaw to cross).
|
| 30 |
+
a, ab = jaw[:, :-1], jaw[:, 1:] - jaw[:, :-1] # (n, S, 2)
|
| 31 |
+
t = ((p[:, :, None] - a[:, None]) * ab[:, None]).sum(-1)
|
| 32 |
+
t = t / (ab * ab).sum(-1).clamp(min=1e-6)[:, None]
|
| 33 |
+
q = (
|
| 34 |
+
a[:, None] + t.clamp(0, 1)[..., None] * ab[:, None]
|
| 35 |
+
) # (n, C, S, 2) nearest point per segment
|
| 36 |
+
s = (p[:, :, None] - q).norm(dim=-1).argmin(-1) # (n, C)
|
| 37 |
+
nrm = torch.stack([ab[..., 1], -ab[..., 0]], -1)
|
| 38 |
+
nrm = torch.where((((a + ab / 2) - c[:, None]) * nrm).sum(-1, keepdim=True) < 0, -nrm, nrm)
|
| 39 |
+
nrm = nrm / nrm.norm(dim=-1, keepdim=True).clamp(min=1e-6)
|
| 40 |
+
qs = q.gather(2, s[..., None, None].expand(-1, -1, 1, 2))[:, :, 0]
|
| 41 |
+
ns = nrm.gather(1, s[..., None].expand(-1, -1, 2))
|
| 42 |
+
ts = t.gather(2, s[..., None])[..., 0]
|
| 43 |
+
S = ab.shape[1]
|
| 44 |
+
valid = ~(((s == 0) & (ts <= 0)) | ((s == S - 1) & (ts >= 1)))
|
| 45 |
+
return ((p - qs) * ns).sum(-1), valid
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class Map620:
|
| 49 |
+
"""The 620 head's four terms on in-the-wild faces, from the map file (prep/wild620_map.py)."""
|
| 50 |
+
|
| 51 |
+
# Per wild set: W (K, 620) writes each INNER hand label as a local affine combination of the
|
| 52 |
+
# 620 points (weights sum to 1, so the map is crop-equivariant); margin (620,) a jaw margin per
|
| 53 |
+
# point that neighbours the jaw (NaN elsewhere). Terms, all in % of the inter-ocular distance
|
| 54 |
+
# divided by B0 (a Laplace-like scale): map = mean |W mu - label| over the inner labelled
|
| 55 |
+
# coordinates; jaw = mean relu(signed jaw distance - margin) over valid jaw neighbours; dense =
|
| 56 |
+
# every point the map does not use against its own pseudo-label (D (620, K): each point as an
|
| 57 |
+
# affine combination of its nearest labels), |error| / that point's reference scale; uc = the
|
| 58 |
+
# under-chin points' |signed jaw distance - their reference median| (two-sided: the map and jaw
|
| 59 |
+
# terms alone let the under-chin drift downwards, nothing holds it from inside).
|
| 60 |
+
B0 = 3.0
|
| 61 |
+
|
| 62 |
+
def __init__(self, path, names, K, eyes, dev):
|
| 63 |
+
z = np.load(path)
|
| 64 |
+
self.W, self.inner, self.marg, self.J, self.eyes = {}, {}, {}, {}, {}
|
| 65 |
+
for n, k in zip(names, K, strict=True):
|
| 66 |
+
self.W[n] = torch.tensor(z[f"W_{n}"], dtype=torch.float32, device=dev)
|
| 67 |
+
assert self.W[n].shape[0] == k, (n, self.W[n].shape, k)
|
| 68 |
+
self.inner[n] = torch.tensor(z[f"inner_{n}"], device=dev)
|
| 69 |
+
m = z[f"margin_{n}"]
|
| 70 |
+
f = np.isfinite(m)
|
| 71 |
+
self.marg[n] = (
|
| 72 |
+
torch.tensor(np.flatnonzero(f), device=dev),
|
| 73 |
+
torch.tensor(m[f], dtype=torch.float32, device=dev),
|
| 74 |
+
)
|
| 75 |
+
self.J[n], self.eyes[n] = int(z[f"jaw_{n}"]), [int(e) for e in eyes[n]]
|
| 76 |
+
print(
|
| 77 |
+
f"MAP620 {n} K {k} inner {int(self.inner[n].sum())} jaw {self.J[n]} neighbours "
|
| 78 |
+
f"{len(self.marg[n][0])} margin median {float(np.nanmedian(m)):+.2f} %IOD",
|
| 79 |
+
flush=True,
|
| 80 |
+
)
|
| 81 |
+
self.under = torch.tensor(np.flatnonzero(z["under"]), device=dev)
|
| 82 |
+
self.D, self.dix, self.dsc, self.ucix, self.ucm = {}, {}, {}, {}, {}
|
| 83 |
+
for n in names:
|
| 84 |
+
if f"D_{n}" not in z.files:
|
| 85 |
+
continue
|
| 86 |
+
dm, u = z[f"dense_{n}"], z[f"ucmed_{n}"]
|
| 87 |
+
self.dix[n] = torch.tensor(np.flatnonzero(dm), device=dev)
|
| 88 |
+
self.D[n] = torch.tensor(z[f"D_{n}"][dm], dtype=torch.float32, device=dev)
|
| 89 |
+
self.dsc[n] = torch.tensor(z[f"dscale_{n}"][dm], dtype=torch.float32, device=dev)
|
| 90 |
+
self.ucix[n] = torch.tensor(np.flatnonzero(np.isfinite(u)), device=dev)
|
| 91 |
+
self.ucm[n] = torch.tensor(u[np.isfinite(u)], dtype=torch.float32, device=dev)
|
| 92 |
+
sm = float(np.nanmedian(z[f"dscale_{n}"]))
|
| 93 |
+
print(
|
| 94 |
+
f"MAP620 {n} dense {int(dm.sum())} scale median {sm:.2f} "
|
| 95 |
+
f"under-chin target {len(self.ucm[n])} median {float(np.nanmedian(u)):+.2f} %IOD",
|
| 96 |
+
flush=True,
|
| 97 |
+
)
|
| 98 |
+
self.pupil = {
|
| 99 |
+
n: [(int(k), int(np.argmax(z[f"W_{n}"][k]))) for k in z[f"pupil_{n}"]] for n in names
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
def terms(self, n, mu, g, ok, per_face=False):
|
| 103 |
+
"""Return (map, jaw, dense, uc) for set n: scalars, or per face (m,), NaN if unscored."""
|
| 104 |
+
# mu (m, 620, 2) our points, g (m, K, 2) hand labels, ok (m, K) label present
|
| 105 |
+
e0, e1 = self.eyes[n]
|
| 106 |
+
io = (g[:, e0] - g[:, e1]).norm(dim=-1).clamp(min=1.0) # (m,)
|
| 107 |
+
okm = ok & self.inner[n][None]
|
| 108 |
+
d = (torch.einsum("kj,mjc->mkc", self.W[n], mu) - g).abs() / io[:, None, None] * 100
|
| 109 |
+
d = d.sum(-1) * okm
|
| 110 |
+
if per_face:
|
| 111 |
+
lmap = d.sum(1) / (2 * okm.sum(1)).clamp(min=1) / self.B0
|
| 112 |
+
else:
|
| 113 |
+
lmap = d.sum() / (2 * okm.sum()).clamp(min=1) / self.B0
|
| 114 |
+
nan = torch.full_like(io, float("nan"))
|
| 115 |
+
zero = lmap * 0
|
| 116 |
+
|
| 117 |
+
def agg(h, v, rows=None): # mean over valid entries: per face (NaN-filled) or over all
|
| 118 |
+
if not per_face:
|
| 119 |
+
return h.sum() / v.sum().clamp(min=1)
|
| 120 |
+
out = nan.clone()
|
| 121 |
+
r = slice(None) if rows is None else rows
|
| 122 |
+
out[r] = torch.where(v.sum(1) > 0, h.sum(1) / v.sum(1).clamp(min=1), torch.nan)
|
| 123 |
+
return out
|
| 124 |
+
|
| 125 |
+
ldense = nan.clone() if per_face else zero
|
| 126 |
+
if n in self.D:
|
| 127 |
+
used = (self.D[n] != 0).float() # (nd, K): the labels each dense point is built from
|
| 128 |
+
okd = (ok.float() @ used.T) == used.sum(1)[None] # (m, nd) all of them labelled
|
| 129 |
+
pseudo = torch.einsum("jk,mkc->mjc", self.D[n], g)
|
| 130 |
+
dd = (mu[:, self.dix[n]] - pseudo).abs().mean(-1) / io[:, None] * 100
|
| 131 |
+
dd = dd / self.dsc[n][None]
|
| 132 |
+
ldense = agg(dd * okd, okd)
|
| 133 |
+
jo = ok[:, : self.J[n]].all(1) # faces with the whole jaw labelled
|
| 134 |
+
if not bool(jo.any()):
|
| 135 |
+
return lmap, (nan if per_face else zero), ldense, (nan if per_face else zero)
|
| 136 |
+
ix, mg = self.marg[n]
|
| 137 |
+
c = (g[jo] * okm[jo][..., None]).sum(1) / okm[jo].sum(1, keepdim=True).clamp(min=1)
|
| 138 |
+
sd, v = jaw_signed(mu[jo][:, ix], g[jo, : self.J[n]], c)
|
| 139 |
+
ljaw = agg(torch.relu(sd / io[jo, None] * 100 - mg[None]) * v, v, jo) / self.B0
|
| 140 |
+
luc = nan.clone() if per_face else zero
|
| 141 |
+
if n in self.ucm:
|
| 142 |
+
su, vu = jaw_signed(mu[jo][:, self.ucix[n]], g[jo, : self.J[n]], c)
|
| 143 |
+
luc = agg((su / io[jo, None] * 100 - self.ucm[n][None]).abs() * vu, vu, jo) / self.B0
|
| 144 |
+
return lmap, ljaw, ldense, luc
|
| 145 |
+
|
| 146 |
+
@torch.no_grad()
|
| 147 |
+
def probe(self, n, mu, g, ok):
|
| 148 |
+
"""Return per face (m,): under-chin median signed jaw distance, pupil error; both % IOD."""
|
| 149 |
+
e0, e1 = self.eyes[n]
|
| 150 |
+
io = (g[:, e0] - g[:, e1]).norm(dim=-1).clamp(min=1.0)
|
| 151 |
+
chin = torch.full_like(io, float("nan"))
|
| 152 |
+
jo = ok[:, : self.J[n]].all(1)
|
| 153 |
+
if bool(jo.any()):
|
| 154 |
+
okm = (ok & self.inner[n][None])[jo]
|
| 155 |
+
c = (g[jo] * okm[..., None]).sum(1) / okm.sum(1, keepdim=True).clamp(min=1)
|
| 156 |
+
sd, v = jaw_signed(mu[jo][:, self.under], g[jo, : self.J[n]], c)
|
| 157 |
+
md = torch.nanmedian(torch.where(v, sd, torch.nan), 1).values / io[jo] * 100
|
| 158 |
+
chin[jo] = torch.where(v.sum(1) >= 5, md, torch.nan)
|
| 159 |
+
pu = torch.full_like(io, float("nan"))
|
| 160 |
+
if len(self.pupil[n]) == 2:
|
| 161 |
+
(k0, j0), (k1, j1) = self.pupil[n]
|
| 162 |
+
e = ((mu[:, j0] - g[:, k0]).norm(dim=-1) + (mu[:, j1] - g[:, k1]).norm(dim=-1)) / 2
|
| 163 |
+
pu = torch.where(ok[:, k0] & ok[:, k1], e / io * 100, torch.nan)
|
| 164 |
+
return chin, pu
|
train/trainer.py
ADDED
|
@@ -0,0 +1,454 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train the detector: a timm backbone, the 620-point head and the 68 / 98-point side heads."""
|
| 2 |
+
|
| 3 |
+
# Usage: python -m facepoints.train <data root> <out dir> [key=value ...]
|
| 4 |
+
# The keys are TrainConfig's fields (config.py); every one prints an ANNOUNCE line and an unknown
|
| 5 |
+
# key aborts. train/README.md has the three invocations that produced the released model.
|
| 6 |
+
# Data: extracted Ava-256 captures (prep/extract.py). With wild=<dir>, in-the-wild faces join every
|
| 7 |
+
# step for the side heads (one Linear each on the shared 1024-d layer, plus a local refinement
|
| 8 |
+
# stage with side_rf=1); with map620=<file> the 620 head is supervised on those faces too
|
| 9 |
+
# (losses.Map620).
|
| 10 |
+
# Loss: per labelled point, the (x, y, log scale) negative log-likelihood (losses.gnll); hidden
|
| 11 |
+
# points are weighted w_occ.
|
| 12 |
+
# Writes val.json (one report per validation), ckpt.pt (last), best.pt (lowest held-out
|
| 13 |
+
# visible-point error), pred_grid_<it>.jpg and wild_grid_<it>.jpg. The lines ANNOUNCE / MODEL /
|
| 14 |
+
# DATA / WILD / STUDIO FRESH / it / VAL / BEST / TRAIN DONE are read by smoke checks.
|
| 15 |
+
import json
|
| 16 |
+
import math
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
import time
|
| 20 |
+
from dataclasses import asdict
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import torch
|
| 24 |
+
from torch import nn
|
| 25 |
+
|
| 26 |
+
from facepoints.model import Net, side_head, side_refine
|
| 27 |
+
from facepoints.train.config import TrainConfig
|
| 28 |
+
from facepoints.train.data import (
|
| 29 |
+
FLIP68,
|
| 30 |
+
MILD_BIT,
|
| 31 |
+
Affine,
|
| 32 |
+
LowRes,
|
| 33 |
+
PointData,
|
| 34 |
+
Wild,
|
| 35 |
+
WildData,
|
| 36 |
+
)
|
| 37 |
+
from facepoints.train.evaluate import autocast, evaluate, evaluate_wild
|
| 38 |
+
from facepoints.train.losses import Map620, gnll
|
| 39 |
+
|
| 40 |
+
LABEL_CROP_PX = 512.0 # the labels' crop side (prep/extract.py); positions scale to cfg.res
|
| 41 |
+
SCALE_INIT_PX = 8.0 # every head starts predicting this scale ...
|
| 42 |
+
HEAD_INIT_SCALE = 0.01 # ... and the mean label position (its weights shrunk, its bias set)
|
| 43 |
+
GRAD_CLIP = 5.0
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def captures(spec, split):
|
| 47 |
+
"""Return the capture ids for spec: a split key, ALL (None = every capture) or a file of ids."""
|
| 48 |
+
if spec == "ALL":
|
| 49 |
+
return None
|
| 50 |
+
if os.path.exists(spec):
|
| 51 |
+
with open(spec) as fh:
|
| 52 |
+
return fh.read().split()
|
| 53 |
+
with open(split) as fh:
|
| 54 |
+
return json.load(fh)[spec]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def build_data(cfg, root):
|
| 58 |
+
"""Return (studio train, studio val, wild train or None, wild val or None, point regions)."""
|
| 59 |
+
all_caps = sorted(
|
| 60 |
+
d for d in os.listdir(root) if os.path.exists(os.path.join(root, d, "labels.npz"))
|
| 61 |
+
)
|
| 62 |
+
lowres = LowRes(cfg.lowres_min, cfg.lowres_p, cfg.lowres_jpeg, cfg.lowres_q)
|
| 63 |
+
aff = Affine(cfg.aff_smin, cfg.aff_smax, cfg.aff_rot, cfg.aff_rot_p)
|
| 64 |
+
if not os.path.exists(cfg.points):
|
| 65 |
+
sys.exit(f"points file not found: {cfg.points}")
|
| 66 |
+
ps = np.load(cfg.points)
|
| 67 |
+
region = ps["region"].astype(str)
|
| 68 |
+
w_ava = wild_tr = wild_va = None
|
| 69 |
+
if cfg.wild:
|
| 70 |
+
eyes = tuple(np.flatnonzero(np.isin(region, [f"{e}_upper", f"{e}_lower"])) for e in "LR")
|
| 71 |
+
bg = os.path.join(cfg.wild, "bg")
|
| 72 |
+
w_ava = Wild(bg, cfg.bg_p, cfg.relight_p, cfg.occ2_p, cfg.glasses_p, eyes)
|
| 73 |
+
w_wild = Wild(bg, 0.0, cfg.relight_p, cfg.occ2_p, cfg.glasses_p)
|
| 74 |
+
sets = cfg.wild_sets.split(",")
|
| 75 |
+
flip = {"300w": FLIP68} if cfg.flip and "300w" in sets else None
|
| 76 |
+
wild_tr = WildData(cfg.wild, sets, cfg.res, True, cfg.seed, lowres, aff, w_wild, flip=flip,
|
| 77 |
+
shift=cfg.aff_shift, occ_p=cfg.occ_p, grey_p=cfg.grey_p,
|
| 78 |
+
fresh=bool(cfg.fresh_aug), mild_p=cfg.mild_p) # fmt: skip
|
| 79 |
+
wild_va = WildData(cfg.wild, sets, cfg.res, False, cfg.seed)
|
| 80 |
+
elif max(cfg.bg_p, cfg.relight_p, cfg.occ2_p, cfg.glasses_p) > 0:
|
| 81 |
+
sys.exit("bg_p / relight_p / occ2_p / glasses_p need wild=<dir> (its bg/ backgrounds)")
|
| 82 |
+
tr_caps = captures(cfg.train_caps, cfg.split) or all_caps
|
| 83 |
+
tr = PointData(root, tr_caps, cfg.res, True, 0, cfg.seed, cfg.min_vis, lowres, aff, w_ava,
|
| 84 |
+
bool(cfg.studio_fresh)) # fmt: skip
|
| 85 |
+
if cfg.studio_fresh: # by content: the same item twice must differ (else the flag is a no-op)
|
| 86 |
+
d = float((tr[0][0] - tr[0][0]).abs().mean())
|
| 87 |
+
print(f"STUDIO FRESH item 0 drawn twice: mean |diff| {d:.4f}", flush=True)
|
| 88 |
+
if d <= 0.01:
|
| 89 |
+
sys.exit("studio_fresh=1 but the augmentation repeats")
|
| 90 |
+
va = PointData(root, captures(cfg.val_caps, cfg.split) or all_caps, cfg.res, False, cfg.val_n,
|
| 91 |
+
cfg.seed, cfg.min_vis) # fmt: skip
|
| 92 |
+
print(f"DATA train {len(tr)} items / {len(tr.lab)} captures ({tr.dropped} dropped below "
|
| 93 |
+
f"min_vis), val {len(va)} items / {len(va.lab)} captures ({va.dropped} dropped), "
|
| 94 |
+
f"{len(all_caps)} extracted under {root}", flush=True) # fmt: skip
|
| 95 |
+
if wild_tr is not None:
|
| 96 |
+
per = {n: sum(j == k for j, _ in wild_tr.items) for k, n in enumerate(wild_tr.names)}
|
| 97 |
+
print(f"WILD train {len(wild_tr)} faces {per}, val {len(wild_va)}, K {wild_tr.K}, "
|
| 98 |
+
f"backgrounds {len(w_ava.bg)}, Ava masks {tr.n_masked}/{len(tr)} "
|
| 99 |
+
f"({tr.n_masked / len(tr):.1%}), {cfg.wild_bs} wild per step, w {cfg.wild_w}",
|
| 100 |
+
flush=True) # fmt: skip
|
| 101 |
+
if cfg.bg_p > 0 and tr.n_masked == 0:
|
| 102 |
+
sys.exit("bg_p > 0 but no Ava crop has a mask - run prep/extract_masks.py first")
|
| 103 |
+
if len(tr) == 0 or len(va) == 0:
|
| 104 |
+
sys.exit("no data")
|
| 105 |
+
assert (ps["cls"] == tr.cls).all(), "points file / labels class mismatch"
|
| 106 |
+
return tr, va, wild_tr, wild_va, region
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def init_head(linear, uv_mean):
|
| 110 |
+
"""Start a (x, y, log scale) head at the mean label position with scale SCALE_INIT_PX."""
|
| 111 |
+
with torch.no_grad():
|
| 112 |
+
b = linear.bias.view(-1, 3)
|
| 113 |
+
b[:, :2] = torch.from_numpy(uv_mean.astype(np.float32))
|
| 114 |
+
b[:, 2] = math.log(SCALE_INIT_PX)
|
| 115 |
+
linear.weight.mul_(HEAD_INIT_SCALE)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def build_model(cfg, tr, wild_tr, dev):
|
| 119 |
+
"""Return (net, side heads, side refinement stages, Map620 or None), initialised, on dev."""
|
| 120 |
+
if cfg.side_rf and wild_tr is None:
|
| 121 |
+
sys.exit("side_rf needs wild=<dir> (it refines the wild side heads)")
|
| 122 |
+
net = Net(cfg.backbone, cfg.res, len(tr.cls), bool(cfg.pretrained), cfg.neck, cfg.stride,
|
| 123 |
+
bool(cfg.side_rf)) # fmt: skip
|
| 124 |
+
uv0 = np.concatenate([z["uv"] for z in tr.lab.values()]) * (cfg.res / LABEL_CROP_PX)
|
| 125 |
+
init_head(net.h[-1], np.nan_to_num(np.nanmean(uv0, axis=0), nan=cfg.res / 2))
|
| 126 |
+
side = nn.ModuleDict()
|
| 127 |
+
if wild_tr is not None: # one Linear per wild set on the shared layer, same initialisation
|
| 128 |
+
for k, n in enumerate(wild_tr.names):
|
| 129 |
+
side[n] = side_head(net, wild_tr.K[k])
|
| 130 |
+
u = wild_tr.lab[n]["uv"][~wild_tr.lab[n]["val"]] * (cfg.res / LABEL_CROP_PX)
|
| 131 |
+
init_head(side[n], np.nanmean(u, axis=0))
|
| 132 |
+
side_rf = nn.ModuleDict()
|
| 133 |
+
if cfg.side_rf:
|
| 134 |
+
for k, n in enumerate(wild_tr.names):
|
| 135 |
+
side_rf[n] = side_refine(net, wild_tr.K[k], cfg.res)
|
| 136 |
+
n_rf = sum(p.numel() for p in side_rf.parameters())
|
| 137 |
+
print(f"MODEL side refine {list(side_rf)} on maps {net.b.feature_info.channels()} ch "
|
| 138 |
+
f"({n_rf} params)", flush=True) # fmt: skip
|
| 139 |
+
mp = None
|
| 140 |
+
if max(cfg.w620, cfg.w620_jaw, cfg.w620_dense, cfg.w620_uc) > 0:
|
| 141 |
+
if wild_tr is None or not cfg.map620:
|
| 142 |
+
sys.exit("w620 / w620_jaw / w620_dense / w620_uc need wild=<dir> and map620=<file>")
|
| 143 |
+
eyes6 = {n: wild_tr.lab[n]["eyes"] for n in wild_tr.names}
|
| 144 |
+
mp = Map620(cfg.map620, wild_tr.names, wild_tr.K, eyes6, dev)
|
| 145 |
+
if cfg.init: # weights only, with a fresh optimiser and schedule
|
| 146 |
+
ck = torch.load(cfg.init, map_location="cpu", weights_only=False) # our own checkpoint
|
| 147 |
+
net.load_state_dict(ck["model"])
|
| 148 |
+
if "side" in ck and len(side):
|
| 149 |
+
side.load_state_dict(ck["side"])
|
| 150 |
+
print(f"INIT {cfg.init} (it {ck.get('it')}, side heads {'side' in ck})", flush=True)
|
| 151 |
+
if len(side_rf):
|
| 152 |
+
print(
|
| 153 |
+
f"INIT side refine: {len(side_rf.state_dict())} new tensors (zero-init output)",
|
| 154 |
+
flush=True,
|
| 155 |
+
)
|
| 156 |
+
side, side_rf = side.to(dev), side_rf.to(dev)
|
| 157 |
+
net = net.to(dev).to(memory_format=torch.channels_last)
|
| 158 |
+
n_par = sum(p.numel() for p in net.parameters()) / 1e6
|
| 159 |
+
n_bb = sum(p.numel() for p in net.b.parameters()) / 1e6
|
| 160 |
+
print(f"MODEL {cfg.backbone} params {n_par:.1f}M (backbone {n_bb:.1f}M)", flush=True)
|
| 161 |
+
return net, side, side_rf, mp
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def build_optimizer(cfg, net, side, side_rf):
|
| 165 |
+
"""Return (AdamW, base learning rates) with groups backbone / heads / refinement stages."""
|
| 166 |
+
bb = list(net.b.parameters())
|
| 167 |
+
hd = [p for n, p in net.named_parameters() if not n.startswith("b.")] + list(side.parameters())
|
| 168 |
+
groups = [{"params": bb, "lr": cfg.lr_bb}, {"params": hd, "lr": cfg.lr}]
|
| 169 |
+
base = [cfg.lr_bb, cfg.lr]
|
| 170 |
+
rfp = list(side_rf.parameters())
|
| 171 |
+
if rfp: # new stages learn from zero at their own, higher rate
|
| 172 |
+
groups.append({"params": rfp, "lr": cfg.lr_rf})
|
| 173 |
+
base.append(cfg.lr_rf)
|
| 174 |
+
return torch.optim.AdamW(groups, weight_decay=cfg.wd), base
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def lr_factor(it, cfg):
|
| 178 |
+
"""Return the learning-rate factor at iteration it: linear warm-up, flat, then cosine decay."""
|
| 179 |
+
t0 = cfg.flat * cfg.iters
|
| 180 |
+
ph = max(0.0, it - t0) / max(1.0, cfg.iters - t0)
|
| 181 |
+
f = 0.5 * (1 + math.cos(math.pi * ph))
|
| 182 |
+
return f * min(1.0, (it + 1) / cfg.warmup)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def wild_loader(cfg, wild_tr):
|
| 186 |
+
"""Return an endless iterator over the wild faces (seeded draws per set by wild_mix)."""
|
| 187 |
+
sampler = None
|
| 188 |
+
if cfg.wild_mix: # draw shares per set, seeded, with replacement
|
| 189 |
+
mix = [float(v) for v in cfg.wild_mix.split(",")]
|
| 190 |
+
assert len(mix) == len(wild_tr.names), "wild_mix needs one share per wild set"
|
| 191 |
+
cnt = np.bincount([j for j, _ in wild_tr.items], minlength=len(mix))
|
| 192 |
+
wts = [mix[j] / cnt[j] for j, _ in wild_tr.items]
|
| 193 |
+
gen = torch.Generator().manual_seed(cfg.seed)
|
| 194 |
+
sampler = torch.utils.data.WeightedRandomSampler(wts, len(wild_tr), True, gen)
|
| 195 |
+
wdl = torch.utils.data.DataLoader(wild_tr, cfg.wild_bs, shuffle=sampler is None,
|
| 196 |
+
sampler=sampler, num_workers=max(2, cfg.workers // 4),
|
| 197 |
+
drop_last=True, pin_memory=True,
|
| 198 |
+
persistent_workers=True) # fmt: skip
|
| 199 |
+
|
| 200 |
+
def endless():
|
| 201 |
+
while True:
|
| 202 |
+
wild_tr.epoch += 1
|
| 203 |
+
yield from wdl
|
| 204 |
+
|
| 205 |
+
return endless()
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
class Trainer:
|
| 209 |
+
"""One training run: the step, the log line, validation and checkpoints."""
|
| 210 |
+
|
| 211 |
+
def __init__(self, cfg, root, out):
|
| 212 |
+
self.cfg, self.out, self.dev = cfg, out, cfg.device
|
| 213 |
+
torch.manual_seed(cfg.seed)
|
| 214 |
+
np.random.seed(cfg.seed)
|
| 215 |
+
self.tr, self.va, self.wild_tr, self.wild_va, self.region = build_data(cfg, root)
|
| 216 |
+
cls = self.tr.cls
|
| 217 |
+
w_cls = np.ones(len(cls), np.float32)
|
| 218 |
+
w_cls[cls == "lid"] = cfg.w_lid
|
| 219 |
+
w_cls[np.isin(cls, ["teeth_u", "teeth_l"])] = cfg.w_teeth
|
| 220 |
+
self.w_cls = torch.tensor(w_cls, device=self.dev)
|
| 221 |
+
self.net, self.side, self.side_rf, self.mp = build_model(
|
| 222 |
+
cfg, self.tr, self.wild_tr, self.dev
|
| 223 |
+
)
|
| 224 |
+
self.opt, self.base_lr = build_optimizer(cfg, self.net, self.side, self.side_rf)
|
| 225 |
+
self.scaler = torch.amp.GradScaler(enabled=bool(cfg.amp))
|
| 226 |
+
self.it, self.best = 0, float("inf")
|
| 227 |
+
if cfg.resume:
|
| 228 |
+
ck = torch.load(cfg.resume, map_location=self.dev, weights_only=False)
|
| 229 |
+
self.net.load_state_dict(ck["model"])
|
| 230 |
+
if len(self.side):
|
| 231 |
+
self.side.load_state_dict(ck["side"])
|
| 232 |
+
if len(self.side_rf):
|
| 233 |
+
self.side_rf.load_state_dict(ck["side_rf"])
|
| 234 |
+
self.opt.load_state_dict(ck["opt"])
|
| 235 |
+
self.it, self.best = ck["it"], ck.get("best", self.best)
|
| 236 |
+
print(f"RESUME {cfg.resume} at it {self.it}", flush=True)
|
| 237 |
+
self.dl = torch.utils.data.DataLoader(self.tr, cfg.batch, shuffle=True,
|
| 238 |
+
num_workers=cfg.workers, drop_last=True,
|
| 239 |
+
pin_memory=True,
|
| 240 |
+
persistent_workers=cfg.workers > 0) # fmt: skip
|
| 241 |
+
self.wl = wild_loader(cfg, self.wild_tr) if self.wild_tr is not None else None
|
| 242 |
+
# running statistics for the log line: loss, wild NLL, side refinement NLL, the Map620 terms
|
| 243 |
+
self.hist = {"loss": [], "wild": [], "srf": [], "m620": []}
|
| 244 |
+
self.side_move = None # how far the side refinement moved its points, last wild step
|
| 245 |
+
self.n_img = self.n_wild = self.n_mild = 0
|
| 246 |
+
self.n_set = np.zeros(len(self.wild_tr.names) if self.wild_tr is not None else 0, np.int64)
|
| 247 |
+
self.t0 = time.time()
|
| 248 |
+
|
| 249 |
+
def set_lr(self):
|
| 250 |
+
"""Set every parameter group's learning rate for the current iteration."""
|
| 251 |
+
f = lr_factor(self.it, self.cfg)
|
| 252 |
+
for g, b0 in zip(self.opt.param_groups, self.base_lr, strict=True):
|
| 253 |
+
g["lr"] = b0 * f
|
| 254 |
+
|
| 255 |
+
def to_dev(self, x):
|
| 256 |
+
"""Return the image batch on the device in channels-last layout."""
|
| 257 |
+
return x.to(self.dev, non_blocking=True).to(memory_format=torch.channels_last)
|
| 258 |
+
|
| 259 |
+
def step(self, x, uv, ok, vis):
|
| 260 |
+
"""Run one optimiser step on a studio (+ wild) batch; return (points, uv, ok) to log."""
|
| 261 |
+
cfg, net = self.cfg, self.net
|
| 262 |
+
x = self.to_dev(x)
|
| 263 |
+
uv, ok, vis = uv.to(self.dev), ok.to(self.dev), vis.to(self.dev)
|
| 264 |
+
w = ok.float() * self.w_cls[None] * torch.where(vis, 1.0, cfg.w_occ)
|
| 265 |
+
B = len(x)
|
| 266 |
+
if self.wl is not None:
|
| 267 |
+
xw, uvw, okw, jw, kw = next(self.wl)
|
| 268 |
+
self.n_mild += int((kw >= MILD_BIT).sum())
|
| 269 |
+
x = torch.cat([x, self.to_dev(xw)])
|
| 270 |
+
maps = None
|
| 271 |
+
with autocast(self.dev, cfg.amp):
|
| 272 |
+
if len(self.side_rf):
|
| 273 |
+
o, zt, maps = net.run(x, B)
|
| 274 |
+
else:
|
| 275 |
+
zt, _ = net.shared(x)
|
| 276 |
+
o = net.h[-1](zt[:B]).view(-1, net.n_points, 3)
|
| 277 |
+
loss = gnll(o.float(), uv, w, cfg.nll, cfg.nu)
|
| 278 |
+
if self.wl is not None: # side heads: each wild set's NLL on its own items, item-weighted
|
| 279 |
+
lw, lws, nw, moves = 0.0, 0.0, 0, []
|
| 280 |
+
uvw, okw = uvw.to(self.dev), okw.to(self.dev)
|
| 281 |
+
for k, n in enumerate(self.wild_tr.names):
|
| 282 |
+
sel = (jw == k).to(self.dev)
|
| 283 |
+
if not bool(sel.any()):
|
| 284 |
+
continue
|
| 285 |
+
K, m = self.wild_tr.K[k], int(sel.sum())
|
| 286 |
+
ow = self.side[n](zt[B:][sel].float()).view(-1, K, 3)
|
| 287 |
+
lw = lw + gnll(ow, uvw[sel, :K], okw[sel, :K].float(), cfg.nll, cfg.nu) * m
|
| 288 |
+
if maps is not None: # the refinement stage on this set's coarse points
|
| 289 |
+
owr = self.side_rf[n]([f[B:][sel].float() for f in maps], ow)
|
| 290 |
+
lws = lws + gnll(owr, uvw[sel, :K], okw[sel, :K].float(), cfg.nll, cfg.nu) * m
|
| 291 |
+
moves.append(float((owr[..., :2] - ow[..., :2]).norm(dim=-1).mean()))
|
| 292 |
+
nw += m
|
| 293 |
+
lw, lws = lw / max(nw, 1), lws / max(nw, 1)
|
| 294 |
+
loss = loss + cfg.wild_w * (lw + lws)
|
| 295 |
+
if self.mp is not None: # the 620 points on the same wild faces, item-weighted
|
| 296 |
+
ow6 = net.h[-1](zt[B:].float()).view(-1, net.n_points, 3)[..., :2]
|
| 297 |
+
t6 = [0.0, 0.0, 0.0, 0.0] # map, jaw, dense, under-chin
|
| 298 |
+
for k, n in enumerate(self.wild_tr.names):
|
| 299 |
+
sel = (jw == k).to(self.dev)
|
| 300 |
+
if bool(sel.any()):
|
| 301 |
+
K, m = self.wild_tr.K[k], int(sel.sum())
|
| 302 |
+
tt = self.mp.terms(n, ow6[sel], uvw[sel, :K], okw[sel, :K].bool())
|
| 303 |
+
t6 = [a + b * m for a, b in zip(t6, tt, strict=True)]
|
| 304 |
+
t6 = [a / max(nw, 1) for a in t6]
|
| 305 |
+
loss = loss + cfg.w620 * t6[0] + cfg.w620_jaw * t6[1]
|
| 306 |
+
loss = loss + cfg.w620_dense * t6[2] + cfg.w620_uc * t6[3]
|
| 307 |
+
self.hist["m620"].append(tuple(float(a) for a in t6))
|
| 308 |
+
self.hist["wild"].append(float(lw))
|
| 309 |
+
if maps is not None:
|
| 310 |
+
self.hist["srf"].append(float(lws))
|
| 311 |
+
self.side_move = float(np.mean(moves)) if moves else self.side_move
|
| 312 |
+
self.n_wild += len(xw)
|
| 313 |
+
self.n_set += np.bincount(jw.numpy(), minlength=len(self.n_set))
|
| 314 |
+
self.opt.zero_grad(set_to_none=True)
|
| 315 |
+
self.scaler.scale(loss).backward()
|
| 316 |
+
self.scaler.unscale_(self.opt)
|
| 317 |
+
torch.nn.utils.clip_grad_norm_(net.parameters(), GRAD_CLIP)
|
| 318 |
+
self.scaler.step(self.opt)
|
| 319 |
+
self.scaler.update()
|
| 320 |
+
self.it += 1
|
| 321 |
+
self.n_img += B
|
| 322 |
+
self.hist["loss"].append(float(loss))
|
| 323 |
+
return o, uv, ok
|
| 324 |
+
|
| 325 |
+
def log(self, o, uv, ok):
|
| 326 |
+
"""Print the `it` line: running losses, pixel error, learning rate, throughput, memory."""
|
| 327 |
+
h, n = self.hist, self.cfg.log_every
|
| 328 |
+
with torch.no_grad():
|
| 329 |
+
px = ((o[..., :2].float() - uv).norm(dim=-1) * ok).sum() / ok.sum()
|
| 330 |
+
dt = time.time() - self.t0
|
| 331 |
+
txt = ""
|
| 332 |
+
if h["srf"]: # side refinement: its wild NLL and how far it moves the points
|
| 333 |
+
move = self.side_move if self.side_move is not None else float("nan")
|
| 334 |
+
txt = f" srf {np.mean(h['srf'][-n:]):.4f} smove {move:.2e}"
|
| 335 |
+
wtxt = f" wild {np.mean(h['wild'][-n:]):.4f}{txt}" if h["wild"] else txt
|
| 336 |
+
wrate = f" (+{self.n_wild / dt:.1f} wild/s)" if self.n_wild else ""
|
| 337 |
+
mtxt = ""
|
| 338 |
+
if h["m620"]:
|
| 339 |
+
mtxt = " | m620 {:.4f} jaw {:.4f} dense {:.4f} uc {:.4f}".format(
|
| 340 |
+
*np.mean(h["m620"][-n:], 0)
|
| 341 |
+
)
|
| 342 |
+
mem = torch.cuda.max_memory_allocated() / 1e9 if self.dev.startswith("cuda") else 0.0
|
| 343 |
+
print(f"it {self.it} loss {np.mean(h['loss'][-n:]):.4f}{wtxt} px {float(px):.2f} "
|
| 344 |
+
f"lr {self.opt.param_groups[1]['lr']:.2e} {self.n_img / dt:.1f} img/s{wrate} "
|
| 345 |
+
f"mem {mem:.1f} GB" + mtxt, flush=True) # fmt: skip
|
| 346 |
+
|
| 347 |
+
def validate(self):
|
| 348 |
+
"""Validate on the studio and wild held-out sets, append to val.json, save checkpoints."""
|
| 349 |
+
cfg, it = self.cfg, self.it
|
| 350 |
+
rep = evaluate(self.net, self.va, cfg, self.dev, self.tr.cls, self.region, self.out, it)
|
| 351 |
+
a = rep["all"]
|
| 352 |
+
print(f"VAL it {it} vis px {a['vis_px']:.2f} mm {a['vis_mm']:.3f} | all px "
|
| 353 |
+
f"{a['all_px']:.2f} | sigma {a['sigma_vis_px']:.2f} px | n {rep['n_images']}",
|
| 354 |
+
flush=True) # fmt: skip
|
| 355 |
+
for g in ("lmk", "skin", "lid", "teeth_u", "teeth_l"):
|
| 356 |
+
if g in rep and rep[g]["vis_px"] is not None:
|
| 357 |
+
r = rep[g]
|
| 358 |
+
print(
|
| 359 |
+
f"VAL {g:8s} vis px {r['vis_px']:.2f} mm {r['vis_mm']:.3f} n {r['n_vis']}",
|
| 360 |
+
flush=True,
|
| 361 |
+
)
|
| 362 |
+
if self.wild_va is not None:
|
| 363 |
+
rep["wild"] = evaluate_wild(self.net, self.side, self.wild_va, cfg, self.dev, self.out,
|
| 364 |
+
it, self.side_rf, self.mp) # fmt: skip
|
| 365 |
+
for n, r in rep["wild"].items():
|
| 366 |
+
rf_ = f" refined {r['nme_rf']:.2f}" if "nme_rf" in r else ""
|
| 367 |
+
print(
|
| 368 |
+
f"VAL WILD {n:5s} NME {r['nme']:.2f}{rf_} FR10 {r['fr10']:.1f} % n {r['n']}",
|
| 369 |
+
flush=True,
|
| 370 |
+
)
|
| 371 |
+
if "m620" in r: # the 620 head on the same held-out wild faces
|
| 372 |
+
pu = f" pupil {r['pupil620']:.2f}" if "pupil620" in r else ""
|
| 373 |
+
print(f"VAL WILD620 {n:5s} map {r['m620']:.3f} jaw {r['j620']:.3f} "
|
| 374 |
+
f"dense {r['d620']:.3f} uc {r['u620']:.3f} under-chin "
|
| 375 |
+
f"{r['chin620']:+.2f} %IOD{pu}", flush=True) # fmt: skip
|
| 376 |
+
with open(os.path.join(self.out, "val.json"), "a") as fh:
|
| 377 |
+
fh.write(json.dumps(rep) + "\n")
|
| 378 |
+
self.save(a)
|
| 379 |
+
|
| 380 |
+
def save(self, a):
|
| 381 |
+
"""Write ckpt.pt, and best.pt when the held-out visible-point error improved."""
|
| 382 |
+
ck = {"model": self.net.state_dict(), "opt": self.opt.state_dict(), "it": self.it,
|
| 383 |
+
"args": asdict(self.cfg), "cls": self.tr.cls, "best": self.best} # fmt: skip
|
| 384 |
+
if len(self.side):
|
| 385 |
+
ck.update(
|
| 386 |
+
side=self.side.state_dict(),
|
| 387 |
+
side_k=dict(zip(self.wild_tr.names, self.wild_tr.K, strict=True)),
|
| 388 |
+
)
|
| 389 |
+
if len(self.side_rf):
|
| 390 |
+
ck.update(side_rf=self.side_rf.state_dict())
|
| 391 |
+
torch.save(ck, os.path.join(self.out, "ckpt.pt"))
|
| 392 |
+
score = a["vis_mm"] if a["vis_mm"] == a["vis_mm"] else a["vis_px"]
|
| 393 |
+
if score < self.best:
|
| 394 |
+
self.best = score
|
| 395 |
+
ck["best"] = self.best
|
| 396 |
+
torch.save(ck, os.path.join(self.out, "best.pt"))
|
| 397 |
+
print(f"BEST it {self.it} score {self.best:.4f}", flush=True)
|
| 398 |
+
|
| 399 |
+
def run(self):
|
| 400 |
+
"""Train for cfg.iters iterations, logging and validating on their cadence."""
|
| 401 |
+
cfg = self.cfg
|
| 402 |
+
self.net.train()
|
| 403 |
+
while self.it < cfg.iters:
|
| 404 |
+
self.tr.epoch += 1
|
| 405 |
+
for x, uv, ok, vis, _ in self.dl:
|
| 406 |
+
if self.it >= cfg.iters:
|
| 407 |
+
break
|
| 408 |
+
self.set_lr()
|
| 409 |
+
o, uv, ok = self.step(x, uv, ok, vis)
|
| 410 |
+
if self.it % cfg.log_every == 0 or self.it == 1:
|
| 411 |
+
self.log(o, uv, ok)
|
| 412 |
+
if self.it % cfg.val_every == 0 or self.it == cfg.iters:
|
| 413 |
+
self.validate()
|
| 414 |
+
if self.wild_tr is not None:
|
| 415 |
+
seen = {n: int(c) for n, c in zip(self.wild_tr.names, self.n_set, strict=True)}
|
| 416 |
+
print(f"WILD SEEN {seen} mild {self.n_mild}/{self.n_wild}", flush=True)
|
| 417 |
+
dt = time.time() - self.t0
|
| 418 |
+
print(
|
| 419 |
+
f"TRAIN DONE it {self.it} {self.n_img} images (+{self.n_wild} wild) {dt:.0f} s",
|
| 420 |
+
flush=True,
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def main():
|
| 425 |
+
"""Parse the command line and run one training."""
|
| 426 |
+
root, out, cfg = TrainConfig.from_argv(sys.argv)
|
| 427 |
+
os.makedirs(out, exist_ok=True)
|
| 428 |
+
Trainer(cfg, root, out).run()
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def run_as_script():
|
| 432 |
+
"""Run main() and leave through os._exit, so a stuck loader worker cannot hang the exit."""
|
| 433 |
+
# (a swallowed CUDA out-of-memory error would otherwise leave a silent live process)
|
| 434 |
+
try:
|
| 435 |
+
main()
|
| 436 |
+
except SystemExit as e: # a usage message or an explicit exit: pass it through as is
|
| 437 |
+
code = e.code if isinstance(e.code, int) else 0 if e.code is None else 1
|
| 438 |
+
if code == 1 and not isinstance(e.code, int):
|
| 439 |
+
print(e.code, file=sys.stderr)
|
| 440 |
+
sys.stdout.flush()
|
| 441 |
+
sys.stderr.flush()
|
| 442 |
+
os._exit(code)
|
| 443 |
+
except BaseException: # noqa: BLE001
|
| 444 |
+
import traceback
|
| 445 |
+
|
| 446 |
+
traceback.print_exc()
|
| 447 |
+
sys.stdout.flush()
|
| 448 |
+
os._exit(1)
|
| 449 |
+
sys.stdout.flush()
|
| 450 |
+
os._exit(0)
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
if __name__ == "__main__":
|
| 454 |
+
run_as_script()
|