Spaces:
Running
Running
fix git bug
Browse files- BUSINESS_BACKUP.md +4 -0
- business_recovery_export_regression.py +81 -0
- tools/business_export.py +75 -10
- tools/export_schema.py +14 -0
BUSINESS_BACKUP.md
CHANGED
|
@@ -2,6 +2,10 @@
|
|
| 2 |
|
| 3 |
## Scope and integration
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
Default-disabled Frontend-only feature. No Worker, evaluation, scoring, Queue producer, quota, identity, publication or GPU lifecycle changes. No terminal/per-Case callbacks were inserted into business functions.
|
| 6 |
|
| 7 |
Changed files:
|
|
|
|
| 2 |
|
| 3 |
## Scope and integration
|
| 4 |
|
| 5 |
+
Historical `operator_compatible_resume` records are accepted as backup audit data without rewriting them into the newer `operator_compatibility_resume` protocol. Their integer retained count and digest must match the successful Progress results completed strictly before the resume timestamp. Terminal/History and Progress recovery evidence must agree. Reused Case-start IDs require both original occurrences in Progress, an intervening resume, the unchanged attempt and a matching final start/result timestamp. Identical recovery occurrences in History remain preserved; contradictory occurrences, unknown fields, unsupported count types and missing evidence fail closed. This compatibility does not certify evaluator equivalence, alter scoring or authorize Worker recovery.
|
| 6 |
+
|
| 7 |
+
Deploy `tools/export_schema.py` and `tools/business_export.py` together on the Frontend before reopening an exhausted backup operation. Keep the existing baseline, destination, receipts and confirmed Git history. A local offline validation success does not mean the live GitHub backup has succeeded; confirm a fresh live backup receipt separately. No Worker restart or rebuild is required for these exporter-only changes.
|
| 8 |
+
|
| 9 |
Default-disabled Frontend-only feature. No Worker, evaluation, scoring, Queue producer, quota, identity, publication or GPU lifecycle changes. No terminal/per-Case callbacks were inserted into business functions.
|
| 10 |
|
| 11 |
Changed files:
|
business_recovery_export_regression.py
CHANGED
|
@@ -180,6 +180,87 @@ class RecoveryExportChecks(unittest.TestCase):
|
|
| 180 |
progress = next(row["data"] for row in rows if row["source"] == f"queue/progress/{self.sid}.json")
|
| 181 |
self.assertEqual(progress["submission_events"], clean(self.progress["submission_events"], ("list", SCHEMAS["audit"])))
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
def test_case_start_identity_conflict_without_reset_remains_rejected(self):
|
| 184 |
self.add_case_reset()
|
| 185 |
for event in self.progress["submission_events"]:
|
|
|
|
| 180 |
progress = next(row["data"] for row in rows if row["source"] == f"queue/progress/{self.sid}.json")
|
| 181 |
self.assertEqual(progress["submission_events"], clean(self.progress["submission_events"], ("list", SCHEMAS["audit"])))
|
| 182 |
|
| 183 |
+
def legacy_recovery(self, cancelled=False, provenance=True):
|
| 184 |
+
self.add_case_reset()
|
| 185 |
+
recovery = {key: self.recovery[key] for key in (
|
| 186 |
+
"timestamp", "reason", "backup_directory", "retained_results_digest")}
|
| 187 |
+
recovery.update(event="operator_compatible_resume", retained_scenes=80,
|
| 188 |
+
event_id=f"{self.sid}:operator_compatible_resume:{self.resumed}")
|
| 189 |
+
self.events[2] = copy.deepcopy(recovery)
|
| 190 |
+
if cancelled:
|
| 191 |
+
self.events[1]["event"] = "cancelled"
|
| 192 |
+
self.events.insert(1, dict(event="operator_priority_pause", event_id=f"{self.sid}:pause",
|
| 193 |
+
timestamp=self.failed, reason="Synthetic priority pause",
|
| 194 |
+
backup_directory="C:/synthetic/private-backup"))
|
| 195 |
+
for index, event in enumerate(self.progress["submission_events"]):
|
| 196 |
+
if event.get("event") == "operator_compatibility_resume":
|
| 197 |
+
self.progress["submission_events"][index] = copy.deepcopy(recovery)
|
| 198 |
+
self.record["evaluation_provenance"]["compatibility_recoveries"] = [copy.deepcopy(recovery)] if provenance else []
|
| 199 |
+
self.record["submission_events"] = self.events
|
| 200 |
+
self.save_records()
|
| 201 |
+
position = next(index for index, event in enumerate(self.events)
|
| 202 |
+
if event.get("event") == "operator_compatible_resume")
|
| 203 |
+
case_events = [event for event in self.progress["submission_events"]
|
| 204 |
+
if event.get("event") != "operator_compatible_resume"]
|
| 205 |
+
combined = (self.events[:position] + [event for event in case_events if event["timestamp"] < self.resumed]
|
| 206 |
+
+ [copy.deepcopy(recovery)] + self.events[position:]
|
| 207 |
+
+ [event for event in case_events if event["timestamp"] >= self.resumed])
|
| 208 |
+
history = self.source / f"results/{TASKS[1]}/_history.2026-10.jsonl"
|
| 209 |
+
history.write_bytes(json.dumps(dict(self.record, submission_events=combined)).encode() + b"\n")
|
| 210 |
+
return recovery
|
| 211 |
+
|
| 212 |
+
def assert_legacy_round_trip(self, cancelled=False, provenance=True):
|
| 213 |
+
self.legacy_recovery(cancelled=cancelled, provenance=provenance)
|
| 214 |
+
export.export(self.source, self.root / "export", attempts=1)
|
| 215 |
+
export.verify(self.root / "export")
|
| 216 |
+
export.reconstruct(self.root / "export", self.root / "reconstructed")
|
| 217 |
+
rows = [json.loads(raw) for raw in (self.root / "reconstructed/business-records.jsonl").read_bytes().splitlines()]
|
| 218 |
+
progress = next(row["data"] for row in rows if row["source"] == f"queue/progress/{self.sid}.json")
|
| 219 |
+
self.assertEqual(progress["submission_events"], clean(self.progress["submission_events"], ("list", SCHEMAS["audit"])))
|
| 220 |
+
self.assertEqual(progress["scene_results"], self.progress["scene_results"])
|
| 221 |
+
|
| 222 |
+
def test_legacy_count_resume_preserves_raw_events_and_results(self):
|
| 223 |
+
self.assert_legacy_round_trip()
|
| 224 |
+
|
| 225 |
+
def test_legacy_priority_pause_without_provenance_preserves_records(self):
|
| 226 |
+
self.assert_legacy_round_trip(cancelled=True, provenance=False)
|
| 227 |
+
|
| 228 |
+
def test_legacy_history_requires_original_progress_reset_evidence(self):
|
| 229 |
+
self.legacy_recovery()
|
| 230 |
+
history = json.loads((self.source / f"results/{TASKS[1]}/_history.2026-10.jsonl").read_bytes())
|
| 231 |
+
history = clean(history, SCHEMAS["history"])
|
| 232 |
+
with self.assertRaisesRegex(ExportError, "conflicting_event_identity"):
|
| 233 |
+
export._events(history["submission_events"], self.sid, submission=history)
|
| 234 |
+
|
| 235 |
+
def test_legacy_count_and_digest_cannot_bypass_retained_validation(self):
|
| 236 |
+
recovery = self.legacy_recovery()
|
| 237 |
+
for change in ({"retained_scenes": 79}, {"retained_results_digest": "a" * 64}):
|
| 238 |
+
with self.subTest(change=change):
|
| 239 |
+
malformed = dict(recovery, **change)
|
| 240 |
+
with self.assertRaisesRegex(ExportError, "recovery_retained_results_mismatch"):
|
| 241 |
+
export._legacy_retained_results(malformed, self.progress["scene_results"])
|
| 242 |
+
|
| 243 |
+
def test_legacy_unknown_fields_and_invalid_count_are_rejected(self):
|
| 244 |
+
recovery = self.legacy_recovery()
|
| 245 |
+
for change, error in (({"unknown": "refuse"}, "unclassified_field"),
|
| 246 |
+
({"retained_scenes": True}, "invalid_number"),
|
| 247 |
+
({"retained_scenes": 0}, "invalid_operator_resume_event")):
|
| 248 |
+
with self.subTest(change=change), self.assertRaisesRegex(ExportError, error):
|
| 249 |
+
clean(dict(recovery, **change), SCHEMAS["audit"])
|
| 250 |
+
|
| 251 |
+
def test_legacy_wrong_submission_and_progress_proof_are_rejected(self):
|
| 252 |
+
self.legacy_recovery()
|
| 253 |
+
events = copy.deepcopy(self.events)
|
| 254 |
+
events[2]["event_id"] = "other:operator_compatible_resume:synthetic"
|
| 255 |
+
with self.assertRaisesRegex(ExportError, "invalid_operator_resume_event"):
|
| 256 |
+
self.validate_events(events=events)
|
| 257 |
+
for event in self.progress["submission_events"]:
|
| 258 |
+
if event.get("event") == "operator_compatible_resume":
|
| 259 |
+
event["reason"] = "Contradictory proof"
|
| 260 |
+
self.save_records()
|
| 261 |
+
with self.assertRaisesRegex(ExportError, "case_reset_recovery_evidence_mismatch"):
|
| 262 |
+
export.export(self.source, self.root / "export", attempts=1)
|
| 263 |
+
|
| 264 |
def test_case_start_identity_conflict_without_reset_remains_rejected(self):
|
| 265 |
self.add_case_reset()
|
| 266 |
for event in self.progress["submission_events"]:
|
tools/business_export.py
CHANGED
|
@@ -420,14 +420,47 @@ def _retained_results_digest(value):
|
|
| 420 |
return digest.hexdigest()
|
| 421 |
|
| 422 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 423 |
def _recovery_start_reuse(events, index, submission):
|
| 424 |
if not isinstance(submission, dict) or index < 2 or submission.get("status") != "done":
|
| 425 |
return None
|
| 426 |
running, recovery, failure = events[index], events[index - 1], events[index - 2]
|
| 427 |
identity = submission.get("submission_id")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 428 |
if (not identity or running.get("event") != "running" or running.get("case") or running.get("case_id")
|
| 429 |
-
or recovery.get("event")
|
| 430 |
-
or
|
|
|
|
|
|
|
|
|
|
| 431 |
or running.get("timestamp") != submission.get("started_at")
|
| 432 |
or recovery.get("timestamp") != submission.get("requeued_at")
|
| 433 |
or type(submission.get("worker_attempt")) is not int
|
|
@@ -436,7 +469,7 @@ def _recovery_start_reuse(events, index, submission):
|
|
| 436 |
or type(submission.get("scenes_total")) is not int or submission["scenes_total"] <= 0
|
| 437 |
or submission.get("scenes_done") != submission.get("scenes_total") or submission.get("scenes_failed") != 0):
|
| 438 |
return None
|
| 439 |
-
originals = [event for event in events[:
|
| 440 |
and event.get("event_id") == f"{identity}:running:1" and event.get("timestamp") == running["timestamp"]]
|
| 441 |
if len(originals) != 1:
|
| 442 |
return None
|
|
@@ -446,11 +479,13 @@ def _recovery_start_reuse(events, index, submission):
|
|
| 446 |
return None
|
| 447 |
matches = [event for event in proofs
|
| 448 |
if event.get("event_id") == recovery.get("event_id")]
|
| 449 |
-
if len(matches) != 1 or
|
|
|
|
| 450 |
return None
|
| 451 |
try:
|
| 452 |
clean(recovery, ("operator_resume_record",))
|
| 453 |
-
|
|
|
|
| 454 |
return None
|
| 455 |
start, failed, resumed = [datetime.fromisoformat(event["timestamp"].replace("Z", "+00:00"))
|
| 456 |
for event in (running, failure, recovery)]
|
|
@@ -473,18 +508,27 @@ def _reset_case_start(events, index, previous, progress):
|
|
| 473 |
or case not in (progress.get("scene_results") or {}) or case in (progress.get("scene_failures") or {})
|
| 474 |
or any(previous.get(key) != current.get(key) for key in (previous.keys() | current.keys()) - {"timestamp", "actor_worker_id"})):
|
| 475 |
return None
|
| 476 |
-
matches = [event for event in events[:index]
|
| 477 |
-
and event.get("submission_id") == identity
|
| 478 |
-
|
|
|
|
|
|
|
|
|
|
| 479 |
if len(matches) != 1:
|
| 480 |
return None
|
| 481 |
recovery = matches[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
originals = [position for position, event in enumerate(events[:index]) if event == previous]
|
| 483 |
if len(originals) != 1 or events.index(recovery) <= originals[0]:
|
| 484 |
return None
|
| 485 |
try:
|
| 486 |
clean(recovery, ("operator_resume_record",))
|
| 487 |
-
|
|
|
|
|
|
|
| 488 |
return None
|
| 489 |
values = (previous["timestamp"], recovery["timestamp"], current["timestamp"], progress["scene_results"][case]["ts"])
|
| 490 |
if any(not isinstance(value, str) for value in values):
|
|
@@ -501,6 +545,12 @@ def _events(events, submission_id=None, *, submission=None, progress=None):
|
|
| 501 |
timestamps, ids, attempts = {}, {}, {}
|
| 502 |
accepted_recoveries = []
|
| 503 |
for index, event in enumerate(events or []):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 504 |
if submission_id and event.get("submission_id") not in {None, submission_id}:
|
| 505 |
fail("event_submission_mismatch")
|
| 506 |
identity = event.get("event_id")
|
|
@@ -544,8 +594,15 @@ def validate(root, manifest):
|
|
| 544 |
terminals, history_ids = set(), set()
|
| 545 |
recovery_checks = defaultdict(dict)
|
| 546 |
case_reset_checks = defaultdict(dict)
|
|
|
|
| 547 |
for record in records(root, manifest):
|
| 548 |
data, kind, selector = record["data"], record["kind"], record["selector"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 549 |
if kind != "registry" or not selector:
|
| 550 |
continue
|
| 551 |
if selector[0] == "submitters":
|
|
@@ -618,7 +675,12 @@ def validate(root, manifest):
|
|
| 618 |
if previous and any(previous[key] != summary[key] for key in previous.keys() & summary.keys()):
|
| 619 |
fail("conflicting_submission_records")
|
| 620 |
submissions[submission_id] = {**(previous or {}), **summary}
|
| 621 |
-
for recovery in _events(data.get("submission_events"), submission_id, submission=data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 622 |
recovery_checks[submission_id][recovery["event_id"]] = recovery
|
| 623 |
primary = data.get("primary_metric") or (data.get("creation_snapshot") or {}).get("primary_metric")
|
| 624 |
if data.get("status") == "done" and primary and primary in (data.get("metrics") or {}) and data.get("score") != data["metrics"][primary]:
|
|
@@ -731,6 +793,9 @@ def validate(root, manifest):
|
|
| 731 |
results = record["data"].get("scene_results") or {}
|
| 732 |
for recovery in recovery_checks[identity].values():
|
| 733 |
retained = recovery["retained_scenes"]
|
|
|
|
|
|
|
|
|
|
| 734 |
if not set(retained) <= results.keys() or _retained_results_digest({case: results[case] for case in retained}) != recovery["retained_results_digest"]:
|
| 735 |
fail("recovery_retained_results_mismatch")
|
| 736 |
checked.add(identity)
|
|
|
|
| 420 |
return digest.hexdigest()
|
| 421 |
|
| 422 |
|
| 423 |
+
def _legacy_retained_results(recovery, results):
|
| 424 |
+
resumed = datetime.fromisoformat(recovery["timestamp"].replace("Z", "+00:00"))
|
| 425 |
+
retained = {}
|
| 426 |
+
for case, result in results.items():
|
| 427 |
+
timestamp = result.get("ts")
|
| 428 |
+
if not isinstance(timestamp, str):
|
| 429 |
+
fail("recovery_retained_results_mismatch")
|
| 430 |
+
try:
|
| 431 |
+
completed = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
| 432 |
+
if completed.tzinfo is None:
|
| 433 |
+
fail("recovery_retained_results_mismatch")
|
| 434 |
+
except ValueError:
|
| 435 |
+
fail("recovery_retained_results_mismatch")
|
| 436 |
+
if completed < resumed:
|
| 437 |
+
retained[case] = result
|
| 438 |
+
if (len(retained) != recovery["retained_scenes"]
|
| 439 |
+
or _retained_results_digest(retained) != recovery["retained_results_digest"]):
|
| 440 |
+
fail("recovery_retained_results_mismatch")
|
| 441 |
+
return retained
|
| 442 |
+
|
| 443 |
+
|
| 444 |
def _recovery_start_reuse(events, index, submission):
|
| 445 |
if not isinstance(submission, dict) or index < 2 or submission.get("status") != "done":
|
| 446 |
return None
|
| 447 |
running, recovery, failure = events[index], events[index - 1], events[index - 2]
|
| 448 |
identity = submission.get("submission_id")
|
| 449 |
+
legacy = recovery.get("event") == "operator_compatible_resume"
|
| 450 |
+
failure_index = index - 2
|
| 451 |
+
if legacy:
|
| 452 |
+
recovery_index = next(position for position, event in enumerate(events[:index]) if event == recovery)
|
| 453 |
+
preceding = [(position, event) for position, event in enumerate(events[:recovery_index])
|
| 454 |
+
if not event.get("case") and not event.get("case_id")]
|
| 455 |
+
if not preceding:
|
| 456 |
+
return None
|
| 457 |
+
failure_index, failure = preceding[-1]
|
| 458 |
if (not identity or running.get("event") != "running" or running.get("case") or running.get("case_id")
|
| 459 |
+
or recovery.get("event") not in {"operator_compatibility_resume", "operator_compatible_resume"}
|
| 460 |
+
or (not legacy and recovery.get("submission_id") != identity)
|
| 461 |
+
or (legacy and not recovery.get("event_id", "").startswith(f"{identity}:operator_compatible_resume:"))
|
| 462 |
+
or failure.get("event") not in ({"failed", "cancelled"} if legacy else {"failed"})
|
| 463 |
+
or failure.get("event_id") != f"{identity}:terminal"
|
| 464 |
or running.get("timestamp") != submission.get("started_at")
|
| 465 |
or recovery.get("timestamp") != submission.get("requeued_at")
|
| 466 |
or type(submission.get("worker_attempt")) is not int
|
|
|
|
| 469 |
or type(submission.get("scenes_total")) is not int or submission["scenes_total"] <= 0
|
| 470 |
or submission.get("scenes_done") != submission.get("scenes_total") or submission.get("scenes_failed") != 0):
|
| 471 |
return None
|
| 472 |
+
originals = [event for event in events[:failure_index] if event.get("event") == "running"
|
| 473 |
and event.get("event_id") == f"{identity}:running:1" and event.get("timestamp") == running["timestamp"]]
|
| 474 |
if len(originals) != 1:
|
| 475 |
return None
|
|
|
|
| 479 |
return None
|
| 480 |
matches = [event for event in proofs
|
| 481 |
if event.get("event_id") == recovery.get("event_id")]
|
| 482 |
+
if ((not legacy and len(matches) != 1) or (legacy and len(matches) > 1)
|
| 483 |
+
or (matches and record_digest(matches[0]) != record_digest(recovery))):
|
| 484 |
return None
|
| 485 |
try:
|
| 486 |
clean(recovery, ("operator_resume_record",))
|
| 487 |
+
retained_count = recovery["retained_scenes"] if legacy else len(recovery["retained_scenes"])
|
| 488 |
+
if retained_count > submission["scenes_total"]:
|
| 489 |
return None
|
| 490 |
start, failed, resumed = [datetime.fromisoformat(event["timestamp"].replace("Z", "+00:00"))
|
| 491 |
for event in (running, failure, recovery)]
|
|
|
|
| 508 |
or case not in (progress.get("scene_results") or {}) or case in (progress.get("scene_failures") or {})
|
| 509 |
or any(previous.get(key) != current.get(key) for key in (previous.keys() | current.keys()) - {"timestamp", "actor_worker_id"})):
|
| 510 |
return None
|
| 511 |
+
matches = [event for event in events[:index]
|
| 512 |
+
if (event.get("event") == "operator_compatibility_resume" and event.get("submission_id") == identity
|
| 513 |
+
and (event.get("reset_incomplete_case_starts") or {}).get(case) == previous.get("timestamp"))
|
| 514 |
+
or (event.get("event") == "operator_compatible_resume"
|
| 515 |
+
and event.get("event_id", "").startswith(f"{identity}:operator_compatible_resume:"))]
|
| 516 |
+
matches = list({record_digest(event): event for event in matches}.values())
|
| 517 |
if len(matches) != 1:
|
| 518 |
return None
|
| 519 |
recovery = matches[0]
|
| 520 |
+
if recovery.get("event") == "operator_compatible_resume":
|
| 521 |
+
recorded = progress.get("submission_events") or []
|
| 522 |
+
if previous not in recorded or current not in recorded or recovery not in recorded:
|
| 523 |
+
return None
|
| 524 |
originals = [position for position, event in enumerate(events[:index]) if event == previous]
|
| 525 |
if len(originals) != 1 or events.index(recovery) <= originals[0]:
|
| 526 |
return None
|
| 527 |
try:
|
| 528 |
clean(recovery, ("operator_resume_record",))
|
| 529 |
+
retained = (_legacy_retained_results(recovery, progress["scene_results"])
|
| 530 |
+
if recovery.get("event") == "operator_compatible_resume" else recovery["retained_scenes"])
|
| 531 |
+
if case in retained:
|
| 532 |
return None
|
| 533 |
values = (previous["timestamp"], recovery["timestamp"], current["timestamp"], progress["scene_results"][case]["ts"])
|
| 534 |
if any(not isinstance(value, str) for value in values):
|
|
|
|
| 545 |
timestamps, ids, attempts = {}, {}, {}
|
| 546 |
accepted_recoveries = []
|
| 547 |
for index, event in enumerate(events or []):
|
| 548 |
+
if event.get("event") == "operator_compatible_resume":
|
| 549 |
+
if (not submission_id or not event.get("event_id", "").startswith(f"{submission_id}:operator_compatible_resume:")
|
| 550 |
+
or event.get("case") or event.get("case_id")):
|
| 551 |
+
fail("invalid_operator_resume_event")
|
| 552 |
+
clean(event, ("operator_resume_record",))
|
| 553 |
+
accepted_recoveries.append(event)
|
| 554 |
if submission_id and event.get("submission_id") not in {None, submission_id}:
|
| 555 |
fail("event_submission_mismatch")
|
| 556 |
identity = event.get("event_id")
|
|
|
|
| 594 |
terminals, history_ids = set(), set()
|
| 595 |
recovery_checks = defaultdict(dict)
|
| 596 |
case_reset_checks = defaultdict(dict)
|
| 597 |
+
legacy_progress = {}
|
| 598 |
for record in records(root, manifest):
|
| 599 |
data, kind, selector = record["data"], record["kind"], record["selector"]
|
| 600 |
+
if kind == "progress" and any(event.get("event") == "operator_compatible_resume"
|
| 601 |
+
for event in data.get("submission_events") or []):
|
| 602 |
+
identity = data.get("submission_id")
|
| 603 |
+
if identity in legacy_progress and record_digest(legacy_progress[identity]) != record_digest(data):
|
| 604 |
+
fail("conflicting_progress_records")
|
| 605 |
+
legacy_progress[identity] = data
|
| 606 |
if kind != "registry" or not selector:
|
| 607 |
continue
|
| 608 |
if selector[0] == "submitters":
|
|
|
|
| 675 |
if previous and any(previous[key] != summary[key] for key in previous.keys() & summary.keys()):
|
| 676 |
fail("conflicting_submission_records")
|
| 677 |
submissions[submission_id] = {**(previous or {}), **summary}
|
| 678 |
+
for recovery in _events(data.get("submission_events"), submission_id, submission=data,
|
| 679 |
+
progress=legacy_progress.get(submission_id) if kind == "history" else None):
|
| 680 |
+
previous = recovery_checks[submission_id].get(recovery["event_id"])
|
| 681 |
+
if (recovery.get("event") == "operator_compatible_resume" and previous is not None
|
| 682 |
+
and record_digest(previous) != record_digest(recovery)):
|
| 683 |
+
fail("case_reset_recovery_evidence_mismatch")
|
| 684 |
recovery_checks[submission_id][recovery["event_id"]] = recovery
|
| 685 |
primary = data.get("primary_metric") or (data.get("creation_snapshot") or {}).get("primary_metric")
|
| 686 |
if data.get("status") == "done" and primary and primary in (data.get("metrics") or {}) and data.get("score") != data["metrics"][primary]:
|
|
|
|
| 793 |
results = record["data"].get("scene_results") or {}
|
| 794 |
for recovery in recovery_checks[identity].values():
|
| 795 |
retained = recovery["retained_scenes"]
|
| 796 |
+
if recovery.get("event") == "operator_compatible_resume":
|
| 797 |
+
_legacy_retained_results(recovery, results)
|
| 798 |
+
continue
|
| 799 |
if not set(retained) <= results.keys() or _retained_results_digest({case: results[case] for case in retained}) != recovery["retained_results_digest"]:
|
| 800 |
fail("recovery_retained_results_mismatch")
|
| 801 |
checked.add(identity)
|
tools/export_schema.py
CHANGED
|
@@ -108,6 +108,9 @@ OPERATOR_RESUME_EVENT = {**EVENT_FIELDS, "backup_directory": "text", "previous_e
|
|
| 108 |
"retained_results_digest": "sha256", "retained_scenes": ("list", "id"),
|
| 109 |
"reset_incomplete_case_starts": ("map", "timestamp"),
|
| 110 |
"verification": {"pmf_tree_at_both_commits": "sha1", "worker_source_change": "text", "full_view_regression": "text"}}
|
|
|
|
|
|
|
|
|
|
| 111 |
PROVENANCE["compatibility_recoveries"] = ("list", ("operator_resume_record",))
|
| 112 |
SUBMISSION = {**fields("submission_id researcher_identity_id hf_username_snapshot team_id owner owner_username task_name track_id worker_id last_renewed_by source_submission_id submitter_orcid orcid_id", "id"),
|
| 113 |
**fields("display_name team_name user_dataset repo_subdir primary_metric status submitted_at started_at reclaimed_at completed_at updated_at cancel_requested_at error_message quota_consumed_at quota_refunded_at quota_refund_reason evaluation_revision"),
|
|
@@ -165,6 +168,17 @@ def clean(value, rule, depth=0):
|
|
| 165 |
return result
|
| 166 |
if isinstance(rule, tuple):
|
| 167 |
if rule[0] in {"submission_event", "operator_resume_record"}:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
if rule[0] == "operator_resume_record" and (not isinstance(value, dict) or value.get("event") != "operator_compatibility_resume"):
|
| 169 |
fail("invalid_operator_resume_event")
|
| 170 |
if isinstance(value, dict) and value.get("event") == "operator_compatibility_resume":
|
|
|
|
| 108 |
"retained_results_digest": "sha256", "retained_scenes": ("list", "id"),
|
| 109 |
"reset_incomplete_case_starts": ("map", "timestamp"),
|
| 110 |
"verification": {"pmf_tree_at_both_commits": "sha1", "worker_source_change": "text", "full_view_regression": "text"}}
|
| 111 |
+
LEGACY_OPERATOR_RESUME_EVENT = {**EVENT_FIELDS, "backup_directory": "text",
|
| 112 |
+
"retained_results_digest": "sha256", "retained_scenes": "int"}
|
| 113 |
+
OPERATOR_PAUSE_EVENT = {**EVENT_FIELDS, "backup_directory": "text"}
|
| 114 |
PROVENANCE["compatibility_recoveries"] = ("list", ("operator_resume_record",))
|
| 115 |
SUBMISSION = {**fields("submission_id researcher_identity_id hf_username_snapshot team_id owner owner_username task_name track_id worker_id last_renewed_by source_submission_id submitter_orcid orcid_id", "id"),
|
| 116 |
**fields("display_name team_name user_dataset repo_subdir primary_metric status submitted_at started_at reclaimed_at completed_at updated_at cancel_requested_at error_message quota_consumed_at quota_refunded_at quota_refund_reason evaluation_revision"),
|
|
|
|
| 168 |
return result
|
| 169 |
if isinstance(rule, tuple):
|
| 170 |
if rule[0] in {"submission_event", "operator_resume_record"}:
|
| 171 |
+
if isinstance(value, dict) and value.get("event") == "operator_compatible_resume":
|
| 172 |
+
required = {"event", "event_id", "timestamp", "reason", "backup_directory", "retained_results_digest", "retained_scenes"}
|
| 173 |
+
if not required <= value.keys():
|
| 174 |
+
fail("incomplete_operator_resume_event")
|
| 175 |
+
result = clean(value, LEGACY_OPERATOR_RESUME_EVENT, depth + 1)
|
| 176 |
+
if any(result.get(key) is None for key in required) or result["retained_scenes"] < 1:
|
| 177 |
+
fail("invalid_operator_resume_event")
|
| 178 |
+
clean(result["timestamp"], "timestamp", depth + 1)
|
| 179 |
+
return result
|
| 180 |
+
if rule[0] == "submission_event" and isinstance(value, dict) and value.get("event") == "operator_priority_pause":
|
| 181 |
+
return clean(value, OPERATOR_PAUSE_EVENT, depth + 1)
|
| 182 |
if rule[0] == "operator_resume_record" and (not isinstance(value, dict) or value.get("event") != "operator_compatibility_resume"):
|
| 183 |
fail("invalid_operator_resume_event")
|
| 184 |
if isinstance(value, dict) and value.get("event") == "operator_compatibility_resume":
|