HAHAHA97 commited on
Commit
c27d6d7
·
1 Parent(s): 690a680

fix git bug

Browse files
BUSINESS_BACKUP.md CHANGED
@@ -2,6 +2,10 @@
2
 
3
  ## Scope and integration
4
 
 
 
 
 
5
  Default-disabled Frontend-only feature. No Worker, evaluation, scoring, Queue producer, quota, identity, publication or GPU lifecycle changes. No terminal/per-Case callbacks were inserted into business functions.
6
 
7
  Changed files:
 
2
 
3
  ## Scope and integration
4
 
5
+ Historical `operator_compatible_resume` records are accepted as backup audit data without rewriting them into the newer `operator_compatibility_resume` protocol. Their integer retained count and digest must match the successful Progress results completed strictly before the resume timestamp. Terminal/History and Progress recovery evidence must agree. Reused Case-start IDs require both original occurrences in Progress, an intervening resume, the unchanged attempt and a matching final start/result timestamp. Identical recovery occurrences in History remain preserved; contradictory occurrences, unknown fields, unsupported count types and missing evidence fail closed. This compatibility does not certify evaluator equivalence, alter scoring or authorize Worker recovery.
6
+
7
+ Deploy `tools/export_schema.py` and `tools/business_export.py` together on the Frontend before reopening an exhausted backup operation. Keep the existing baseline, destination, receipts and confirmed Git history. A local offline validation success does not mean the live GitHub backup has succeeded; confirm a fresh live backup receipt separately. No Worker restart or rebuild is required for these exporter-only changes.
8
+
9
  Default-disabled Frontend-only feature. No Worker, evaluation, scoring, Queue producer, quota, identity, publication or GPU lifecycle changes. No terminal/per-Case callbacks were inserted into business functions.
10
 
11
  Changed files:
business_recovery_export_regression.py CHANGED
@@ -180,6 +180,87 @@ class RecoveryExportChecks(unittest.TestCase):
180
  progress = next(row["data"] for row in rows if row["source"] == f"queue/progress/{self.sid}.json")
181
  self.assertEqual(progress["submission_events"], clean(self.progress["submission_events"], ("list", SCHEMAS["audit"])))
182
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  def test_case_start_identity_conflict_without_reset_remains_rejected(self):
184
  self.add_case_reset()
185
  for event in self.progress["submission_events"]:
 
180
  progress = next(row["data"] for row in rows if row["source"] == f"queue/progress/{self.sid}.json")
181
  self.assertEqual(progress["submission_events"], clean(self.progress["submission_events"], ("list", SCHEMAS["audit"])))
182
 
183
+ def legacy_recovery(self, cancelled=False, provenance=True):
184
+ self.add_case_reset()
185
+ recovery = {key: self.recovery[key] for key in (
186
+ "timestamp", "reason", "backup_directory", "retained_results_digest")}
187
+ recovery.update(event="operator_compatible_resume", retained_scenes=80,
188
+ event_id=f"{self.sid}:operator_compatible_resume:{self.resumed}")
189
+ self.events[2] = copy.deepcopy(recovery)
190
+ if cancelled:
191
+ self.events[1]["event"] = "cancelled"
192
+ self.events.insert(1, dict(event="operator_priority_pause", event_id=f"{self.sid}:pause",
193
+ timestamp=self.failed, reason="Synthetic priority pause",
194
+ backup_directory="C:/synthetic/private-backup"))
195
+ for index, event in enumerate(self.progress["submission_events"]):
196
+ if event.get("event") == "operator_compatibility_resume":
197
+ self.progress["submission_events"][index] = copy.deepcopy(recovery)
198
+ self.record["evaluation_provenance"]["compatibility_recoveries"] = [copy.deepcopy(recovery)] if provenance else []
199
+ self.record["submission_events"] = self.events
200
+ self.save_records()
201
+ position = next(index for index, event in enumerate(self.events)
202
+ if event.get("event") == "operator_compatible_resume")
203
+ case_events = [event for event in self.progress["submission_events"]
204
+ if event.get("event") != "operator_compatible_resume"]
205
+ combined = (self.events[:position] + [event for event in case_events if event["timestamp"] < self.resumed]
206
+ + [copy.deepcopy(recovery)] + self.events[position:]
207
+ + [event for event in case_events if event["timestamp"] >= self.resumed])
208
+ history = self.source / f"results/{TASKS[1]}/_history.2026-10.jsonl"
209
+ history.write_bytes(json.dumps(dict(self.record, submission_events=combined)).encode() + b"\n")
210
+ return recovery
211
+
212
+ def assert_legacy_round_trip(self, cancelled=False, provenance=True):
213
+ self.legacy_recovery(cancelled=cancelled, provenance=provenance)
214
+ export.export(self.source, self.root / "export", attempts=1)
215
+ export.verify(self.root / "export")
216
+ export.reconstruct(self.root / "export", self.root / "reconstructed")
217
+ rows = [json.loads(raw) for raw in (self.root / "reconstructed/business-records.jsonl").read_bytes().splitlines()]
218
+ progress = next(row["data"] for row in rows if row["source"] == f"queue/progress/{self.sid}.json")
219
+ self.assertEqual(progress["submission_events"], clean(self.progress["submission_events"], ("list", SCHEMAS["audit"])))
220
+ self.assertEqual(progress["scene_results"], self.progress["scene_results"])
221
+
222
+ def test_legacy_count_resume_preserves_raw_events_and_results(self):
223
+ self.assert_legacy_round_trip()
224
+
225
+ def test_legacy_priority_pause_without_provenance_preserves_records(self):
226
+ self.assert_legacy_round_trip(cancelled=True, provenance=False)
227
+
228
+ def test_legacy_history_requires_original_progress_reset_evidence(self):
229
+ self.legacy_recovery()
230
+ history = json.loads((self.source / f"results/{TASKS[1]}/_history.2026-10.jsonl").read_bytes())
231
+ history = clean(history, SCHEMAS["history"])
232
+ with self.assertRaisesRegex(ExportError, "conflicting_event_identity"):
233
+ export._events(history["submission_events"], self.sid, submission=history)
234
+
235
+ def test_legacy_count_and_digest_cannot_bypass_retained_validation(self):
236
+ recovery = self.legacy_recovery()
237
+ for change in ({"retained_scenes": 79}, {"retained_results_digest": "a" * 64}):
238
+ with self.subTest(change=change):
239
+ malformed = dict(recovery, **change)
240
+ with self.assertRaisesRegex(ExportError, "recovery_retained_results_mismatch"):
241
+ export._legacy_retained_results(malformed, self.progress["scene_results"])
242
+
243
+ def test_legacy_unknown_fields_and_invalid_count_are_rejected(self):
244
+ recovery = self.legacy_recovery()
245
+ for change, error in (({"unknown": "refuse"}, "unclassified_field"),
246
+ ({"retained_scenes": True}, "invalid_number"),
247
+ ({"retained_scenes": 0}, "invalid_operator_resume_event")):
248
+ with self.subTest(change=change), self.assertRaisesRegex(ExportError, error):
249
+ clean(dict(recovery, **change), SCHEMAS["audit"])
250
+
251
+ def test_legacy_wrong_submission_and_progress_proof_are_rejected(self):
252
+ self.legacy_recovery()
253
+ events = copy.deepcopy(self.events)
254
+ events[2]["event_id"] = "other:operator_compatible_resume:synthetic"
255
+ with self.assertRaisesRegex(ExportError, "invalid_operator_resume_event"):
256
+ self.validate_events(events=events)
257
+ for event in self.progress["submission_events"]:
258
+ if event.get("event") == "operator_compatible_resume":
259
+ event["reason"] = "Contradictory proof"
260
+ self.save_records()
261
+ with self.assertRaisesRegex(ExportError, "case_reset_recovery_evidence_mismatch"):
262
+ export.export(self.source, self.root / "export", attempts=1)
263
+
264
  def test_case_start_identity_conflict_without_reset_remains_rejected(self):
265
  self.add_case_reset()
266
  for event in self.progress["submission_events"]:
tools/business_export.py CHANGED
@@ -420,14 +420,47 @@ def _retained_results_digest(value):
420
  return digest.hexdigest()
421
 
422
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
423
  def _recovery_start_reuse(events, index, submission):
424
  if not isinstance(submission, dict) or index < 2 or submission.get("status") != "done":
425
  return None
426
  running, recovery, failure = events[index], events[index - 1], events[index - 2]
427
  identity = submission.get("submission_id")
 
 
 
 
 
 
 
 
 
428
  if (not identity or running.get("event") != "running" or running.get("case") or running.get("case_id")
429
- or recovery.get("event") != "operator_compatibility_resume" or recovery.get("submission_id") != identity
430
- or failure.get("event") != "failed" or failure.get("event_id") != f"{identity}:terminal"
 
 
 
431
  or running.get("timestamp") != submission.get("started_at")
432
  or recovery.get("timestamp") != submission.get("requeued_at")
433
  or type(submission.get("worker_attempt")) is not int
@@ -436,7 +469,7 @@ def _recovery_start_reuse(events, index, submission):
436
  or type(submission.get("scenes_total")) is not int or submission["scenes_total"] <= 0
437
  or submission.get("scenes_done") != submission.get("scenes_total") or submission.get("scenes_failed") != 0):
438
  return None
439
- originals = [event for event in events[:index - 2] if event.get("event") == "running"
440
  and event.get("event_id") == f"{identity}:running:1" and event.get("timestamp") == running["timestamp"]]
441
  if len(originals) != 1:
442
  return None
@@ -446,11 +479,13 @@ def _recovery_start_reuse(events, index, submission):
446
  return None
447
  matches = [event for event in proofs
448
  if event.get("event_id") == recovery.get("event_id")]
449
- if len(matches) != 1 or record_digest(matches[0]) != record_digest(recovery):
 
450
  return None
451
  try:
452
  clean(recovery, ("operator_resume_record",))
453
- if len(recovery["retained_scenes"]) > submission["scenes_total"]:
 
454
  return None
455
  start, failed, resumed = [datetime.fromisoformat(event["timestamp"].replace("Z", "+00:00"))
456
  for event in (running, failure, recovery)]
@@ -473,18 +508,27 @@ def _reset_case_start(events, index, previous, progress):
473
  or case not in (progress.get("scene_results") or {}) or case in (progress.get("scene_failures") or {})
474
  or any(previous.get(key) != current.get(key) for key in (previous.keys() | current.keys()) - {"timestamp", "actor_worker_id"})):
475
  return None
476
- matches = [event for event in events[:index] if event.get("event") == "operator_compatibility_resume"
477
- and event.get("submission_id") == identity
478
- and (event.get("reset_incomplete_case_starts") or {}).get(case) == previous.get("timestamp")]
 
 
 
479
  if len(matches) != 1:
480
  return None
481
  recovery = matches[0]
 
 
 
 
482
  originals = [position for position, event in enumerate(events[:index]) if event == previous]
483
  if len(originals) != 1 or events.index(recovery) <= originals[0]:
484
  return None
485
  try:
486
  clean(recovery, ("operator_resume_record",))
487
- if case in recovery["retained_scenes"]:
 
 
488
  return None
489
  values = (previous["timestamp"], recovery["timestamp"], current["timestamp"], progress["scene_results"][case]["ts"])
490
  if any(not isinstance(value, str) for value in values):
@@ -501,6 +545,12 @@ def _events(events, submission_id=None, *, submission=None, progress=None):
501
  timestamps, ids, attempts = {}, {}, {}
502
  accepted_recoveries = []
503
  for index, event in enumerate(events or []):
 
 
 
 
 
 
504
  if submission_id and event.get("submission_id") not in {None, submission_id}:
505
  fail("event_submission_mismatch")
506
  identity = event.get("event_id")
@@ -544,8 +594,15 @@ def validate(root, manifest):
544
  terminals, history_ids = set(), set()
545
  recovery_checks = defaultdict(dict)
546
  case_reset_checks = defaultdict(dict)
 
547
  for record in records(root, manifest):
548
  data, kind, selector = record["data"], record["kind"], record["selector"]
 
 
 
 
 
 
549
  if kind != "registry" or not selector:
550
  continue
551
  if selector[0] == "submitters":
@@ -618,7 +675,12 @@ def validate(root, manifest):
618
  if previous and any(previous[key] != summary[key] for key in previous.keys() & summary.keys()):
619
  fail("conflicting_submission_records")
620
  submissions[submission_id] = {**(previous or {}), **summary}
621
- for recovery in _events(data.get("submission_events"), submission_id, submission=data):
 
 
 
 
 
622
  recovery_checks[submission_id][recovery["event_id"]] = recovery
623
  primary = data.get("primary_metric") or (data.get("creation_snapshot") or {}).get("primary_metric")
624
  if data.get("status") == "done" and primary and primary in (data.get("metrics") or {}) and data.get("score") != data["metrics"][primary]:
@@ -731,6 +793,9 @@ def validate(root, manifest):
731
  results = record["data"].get("scene_results") or {}
732
  for recovery in recovery_checks[identity].values():
733
  retained = recovery["retained_scenes"]
 
 
 
734
  if not set(retained) <= results.keys() or _retained_results_digest({case: results[case] for case in retained}) != recovery["retained_results_digest"]:
735
  fail("recovery_retained_results_mismatch")
736
  checked.add(identity)
 
420
  return digest.hexdigest()
421
 
422
 
423
+ def _legacy_retained_results(recovery, results):
424
+ resumed = datetime.fromisoformat(recovery["timestamp"].replace("Z", "+00:00"))
425
+ retained = {}
426
+ for case, result in results.items():
427
+ timestamp = result.get("ts")
428
+ if not isinstance(timestamp, str):
429
+ fail("recovery_retained_results_mismatch")
430
+ try:
431
+ completed = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
432
+ if completed.tzinfo is None:
433
+ fail("recovery_retained_results_mismatch")
434
+ except ValueError:
435
+ fail("recovery_retained_results_mismatch")
436
+ if completed < resumed:
437
+ retained[case] = result
438
+ if (len(retained) != recovery["retained_scenes"]
439
+ or _retained_results_digest(retained) != recovery["retained_results_digest"]):
440
+ fail("recovery_retained_results_mismatch")
441
+ return retained
442
+
443
+
444
  def _recovery_start_reuse(events, index, submission):
445
  if not isinstance(submission, dict) or index < 2 or submission.get("status") != "done":
446
  return None
447
  running, recovery, failure = events[index], events[index - 1], events[index - 2]
448
  identity = submission.get("submission_id")
449
+ legacy = recovery.get("event") == "operator_compatible_resume"
450
+ failure_index = index - 2
451
+ if legacy:
452
+ recovery_index = next(position for position, event in enumerate(events[:index]) if event == recovery)
453
+ preceding = [(position, event) for position, event in enumerate(events[:recovery_index])
454
+ if not event.get("case") and not event.get("case_id")]
455
+ if not preceding:
456
+ return None
457
+ failure_index, failure = preceding[-1]
458
  if (not identity or running.get("event") != "running" or running.get("case") or running.get("case_id")
459
+ or recovery.get("event") not in {"operator_compatibility_resume", "operator_compatible_resume"}
460
+ or (not legacy and recovery.get("submission_id") != identity)
461
+ or (legacy and not recovery.get("event_id", "").startswith(f"{identity}:operator_compatible_resume:"))
462
+ or failure.get("event") not in ({"failed", "cancelled"} if legacy else {"failed"})
463
+ or failure.get("event_id") != f"{identity}:terminal"
464
  or running.get("timestamp") != submission.get("started_at")
465
  or recovery.get("timestamp") != submission.get("requeued_at")
466
  or type(submission.get("worker_attempt")) is not int
 
469
  or type(submission.get("scenes_total")) is not int or submission["scenes_total"] <= 0
470
  or submission.get("scenes_done") != submission.get("scenes_total") or submission.get("scenes_failed") != 0):
471
  return None
472
+ originals = [event for event in events[:failure_index] if event.get("event") == "running"
473
  and event.get("event_id") == f"{identity}:running:1" and event.get("timestamp") == running["timestamp"]]
474
  if len(originals) != 1:
475
  return None
 
479
  return None
480
  matches = [event for event in proofs
481
  if event.get("event_id") == recovery.get("event_id")]
482
+ if ((not legacy and len(matches) != 1) or (legacy and len(matches) > 1)
483
+ or (matches and record_digest(matches[0]) != record_digest(recovery))):
484
  return None
485
  try:
486
  clean(recovery, ("operator_resume_record",))
487
+ retained_count = recovery["retained_scenes"] if legacy else len(recovery["retained_scenes"])
488
+ if retained_count > submission["scenes_total"]:
489
  return None
490
  start, failed, resumed = [datetime.fromisoformat(event["timestamp"].replace("Z", "+00:00"))
491
  for event in (running, failure, recovery)]
 
508
  or case not in (progress.get("scene_results") or {}) or case in (progress.get("scene_failures") or {})
509
  or any(previous.get(key) != current.get(key) for key in (previous.keys() | current.keys()) - {"timestamp", "actor_worker_id"})):
510
  return None
511
+ matches = [event for event in events[:index]
512
+ if (event.get("event") == "operator_compatibility_resume" and event.get("submission_id") == identity
513
+ and (event.get("reset_incomplete_case_starts") or {}).get(case) == previous.get("timestamp"))
514
+ or (event.get("event") == "operator_compatible_resume"
515
+ and event.get("event_id", "").startswith(f"{identity}:operator_compatible_resume:"))]
516
+ matches = list({record_digest(event): event for event in matches}.values())
517
  if len(matches) != 1:
518
  return None
519
  recovery = matches[0]
520
+ if recovery.get("event") == "operator_compatible_resume":
521
+ recorded = progress.get("submission_events") or []
522
+ if previous not in recorded or current not in recorded or recovery not in recorded:
523
+ return None
524
  originals = [position for position, event in enumerate(events[:index]) if event == previous]
525
  if len(originals) != 1 or events.index(recovery) <= originals[0]:
526
  return None
527
  try:
528
  clean(recovery, ("operator_resume_record",))
529
+ retained = (_legacy_retained_results(recovery, progress["scene_results"])
530
+ if recovery.get("event") == "operator_compatible_resume" else recovery["retained_scenes"])
531
+ if case in retained:
532
  return None
533
  values = (previous["timestamp"], recovery["timestamp"], current["timestamp"], progress["scene_results"][case]["ts"])
534
  if any(not isinstance(value, str) for value in values):
 
545
  timestamps, ids, attempts = {}, {}, {}
546
  accepted_recoveries = []
547
  for index, event in enumerate(events or []):
548
+ if event.get("event") == "operator_compatible_resume":
549
+ if (not submission_id or not event.get("event_id", "").startswith(f"{submission_id}:operator_compatible_resume:")
550
+ or event.get("case") or event.get("case_id")):
551
+ fail("invalid_operator_resume_event")
552
+ clean(event, ("operator_resume_record",))
553
+ accepted_recoveries.append(event)
554
  if submission_id and event.get("submission_id") not in {None, submission_id}:
555
  fail("event_submission_mismatch")
556
  identity = event.get("event_id")
 
594
  terminals, history_ids = set(), set()
595
  recovery_checks = defaultdict(dict)
596
  case_reset_checks = defaultdict(dict)
597
+ legacy_progress = {}
598
  for record in records(root, manifest):
599
  data, kind, selector = record["data"], record["kind"], record["selector"]
600
+ if kind == "progress" and any(event.get("event") == "operator_compatible_resume"
601
+ for event in data.get("submission_events") or []):
602
+ identity = data.get("submission_id")
603
+ if identity in legacy_progress and record_digest(legacy_progress[identity]) != record_digest(data):
604
+ fail("conflicting_progress_records")
605
+ legacy_progress[identity] = data
606
  if kind != "registry" or not selector:
607
  continue
608
  if selector[0] == "submitters":
 
675
  if previous and any(previous[key] != summary[key] for key in previous.keys() & summary.keys()):
676
  fail("conflicting_submission_records")
677
  submissions[submission_id] = {**(previous or {}), **summary}
678
+ for recovery in _events(data.get("submission_events"), submission_id, submission=data,
679
+ progress=legacy_progress.get(submission_id) if kind == "history" else None):
680
+ previous = recovery_checks[submission_id].get(recovery["event_id"])
681
+ if (recovery.get("event") == "operator_compatible_resume" and previous is not None
682
+ and record_digest(previous) != record_digest(recovery)):
683
+ fail("case_reset_recovery_evidence_mismatch")
684
  recovery_checks[submission_id][recovery["event_id"]] = recovery
685
  primary = data.get("primary_metric") or (data.get("creation_snapshot") or {}).get("primary_metric")
686
  if data.get("status") == "done" and primary and primary in (data.get("metrics") or {}) and data.get("score") != data["metrics"][primary]:
 
793
  results = record["data"].get("scene_results") or {}
794
  for recovery in recovery_checks[identity].values():
795
  retained = recovery["retained_scenes"]
796
+ if recovery.get("event") == "operator_compatible_resume":
797
+ _legacy_retained_results(recovery, results)
798
+ continue
799
  if not set(retained) <= results.keys() or _retained_results_digest({case: results[case] for case in retained}) != recovery["retained_results_digest"]:
800
  fail("recovery_retained_results_mismatch")
801
  checked.add(identity)
tools/export_schema.py CHANGED
@@ -108,6 +108,9 @@ OPERATOR_RESUME_EVENT = {**EVENT_FIELDS, "backup_directory": "text", "previous_e
108
  "retained_results_digest": "sha256", "retained_scenes": ("list", "id"),
109
  "reset_incomplete_case_starts": ("map", "timestamp"),
110
  "verification": {"pmf_tree_at_both_commits": "sha1", "worker_source_change": "text", "full_view_regression": "text"}}
 
 
 
111
  PROVENANCE["compatibility_recoveries"] = ("list", ("operator_resume_record",))
112
  SUBMISSION = {**fields("submission_id researcher_identity_id hf_username_snapshot team_id owner owner_username task_name track_id worker_id last_renewed_by source_submission_id submitter_orcid orcid_id", "id"),
113
  **fields("display_name team_name user_dataset repo_subdir primary_metric status submitted_at started_at reclaimed_at completed_at updated_at cancel_requested_at error_message quota_consumed_at quota_refunded_at quota_refund_reason evaluation_revision"),
@@ -165,6 +168,17 @@ def clean(value, rule, depth=0):
165
  return result
166
  if isinstance(rule, tuple):
167
  if rule[0] in {"submission_event", "operator_resume_record"}:
 
 
 
 
 
 
 
 
 
 
 
168
  if rule[0] == "operator_resume_record" and (not isinstance(value, dict) or value.get("event") != "operator_compatibility_resume"):
169
  fail("invalid_operator_resume_event")
170
  if isinstance(value, dict) and value.get("event") == "operator_compatibility_resume":
 
108
  "retained_results_digest": "sha256", "retained_scenes": ("list", "id"),
109
  "reset_incomplete_case_starts": ("map", "timestamp"),
110
  "verification": {"pmf_tree_at_both_commits": "sha1", "worker_source_change": "text", "full_view_regression": "text"}}
111
+ LEGACY_OPERATOR_RESUME_EVENT = {**EVENT_FIELDS, "backup_directory": "text",
112
+ "retained_results_digest": "sha256", "retained_scenes": "int"}
113
+ OPERATOR_PAUSE_EVENT = {**EVENT_FIELDS, "backup_directory": "text"}
114
  PROVENANCE["compatibility_recoveries"] = ("list", ("operator_resume_record",))
115
  SUBMISSION = {**fields("submission_id researcher_identity_id hf_username_snapshot team_id owner owner_username task_name track_id worker_id last_renewed_by source_submission_id submitter_orcid orcid_id", "id"),
116
  **fields("display_name team_name user_dataset repo_subdir primary_metric status submitted_at started_at reclaimed_at completed_at updated_at cancel_requested_at error_message quota_consumed_at quota_refunded_at quota_refund_reason evaluation_revision"),
 
168
  return result
169
  if isinstance(rule, tuple):
170
  if rule[0] in {"submission_event", "operator_resume_record"}:
171
+ if isinstance(value, dict) and value.get("event") == "operator_compatible_resume":
172
+ required = {"event", "event_id", "timestamp", "reason", "backup_directory", "retained_results_digest", "retained_scenes"}
173
+ if not required <= value.keys():
174
+ fail("incomplete_operator_resume_event")
175
+ result = clean(value, LEGACY_OPERATOR_RESUME_EVENT, depth + 1)
176
+ if any(result.get(key) is None for key in required) or result["retained_scenes"] < 1:
177
+ fail("invalid_operator_resume_event")
178
+ clean(result["timestamp"], "timestamp", depth + 1)
179
+ return result
180
+ if rule[0] == "submission_event" and isinstance(value, dict) and value.get("event") == "operator_priority_pause":
181
+ return clean(value, OPERATOR_PAUSE_EVENT, depth + 1)
182
  if rule[0] == "operator_resume_record" and (not isinstance(value, dict) or value.get("event") != "operator_compatibility_resume"):
183
  fail("invalid_operator_resume_event")
184
  if isinstance(value, dict) and value.get("event") == "operator_compatibility_resume":