"""Independent fixed oracle; only frozen source files outside the agent workspace are read.""" import json import math from pathlib import Path import shlex import sys ROOT = Path(__file__).resolve().parent EXPECTED = json.loads((ROOT / 'expected.json').read_text()) def unique_object(pairs): result = {} for key, value in pairs: if key in result: raise ValueError('Duplicate JSON key: ' + key) result[key] = value return result def compare(actual, expected, path, failures): if isinstance(expected, dict): if not isinstance(actual, dict): failures.append(path + ': expected an object') return for key, value in expected.items(): if key == 'sources': continue if key not in actual: failures.append(path + '.' + key + ': missing') else: compare(actual[key], value, path + '.' + key, failures) elif isinstance(expected, list): if not isinstance(actual, list) or len(actual) != len(expected): failures.append(path + ': wrong list length') return if path == 'answer.studies': if any(not isinstance(item, dict) or not isinstance(item.get('id'), str) for item in actual): failures.append(path + ': missing study identity') return actual, expected = sorted(actual, key=lambda x: x['id']), sorted(expected, key=lambda x: x['id']) if path.endswith(('requiredBodyKeys', 'missingRequiredResultFields')): if any(not isinstance(value, str) for value in actual): failures.append(path + ': expected string keys') return actual, expected = sorted(actual), sorted(expected) for index, (a, e) in enumerate(zip(actual, expected)): compare(a, e, f'{path}[{index}]', failures) elif isinstance(expected, bool) or expected is None: if actual is not expected: failures.append(path + ': incorrect policy or missing-coverage interpretation') elif isinstance(expected, (int, float)): tolerance = 1 if path.endswith('candidateMinusBaselineMeanMs') else 0.000001 if isinstance(actual, bool) or not isinstance(actual, (int, float)) or not math.isfinite(actual) or abs(actual - expected) > tolerance: failures.append(path + ': incorrect numeric value') elif path.endswith('Command'): try: a, e = shlex.split(actual), shlex.split(expected) if path.endswith('uploadCommand'): # The CLI accepts --project before or after the positional result. if a[:3] != e[:3] or len(a) != 6 or a.count('--project') != 1: raise ValueError('Invalid upload command') option = a.index('--project') if option not in (3, 4) or a[option + 1] != 'PROJECT_ID' or a[:option] + a[option + 2:] != ['evx', 'experiment', 'upload', './results/result.json']: raise ValueError('Wrong project or result') elif a != e: raise ValueError('Wrong command') except (TypeError, ValueError): failures.append(path + ': incorrect CLI workflow') elif actual != expected: failures.append(path + ': incorrect value') def verify(output): try: text = output.strip() if text.startswith('```json\n') and text.endswith('\n```'): text = text[8:-4] answer = json.loads(text, object_pairs_hook=unique_object) failures = [] compare(answer, EXPECTED, 'answer', failures) sources = answer.get('sources', []) if isinstance(answer, dict) else [] if not isinstance(sources, list) or len(sources) != 3: failures.append('sources: expected three supporting quotations') else: seen = set() for source in sources: if not isinstance(source, dict): failures.append('sources: invalid source object') continue claim, file, quote = source.get('claim'), source.get('file'), source.get('quote') if claim not in {'terms', 'immutable', 'scope'} or claim in seen or file != 'production/full.md' or not isinstance(quote, str) or not 12 <= len(quote) <= 1500: failures.append('sources: missing, duplicate or invalid source') continue seen.add(claim) if quote not in (ROOT / 'baseline' / file).read_text(): failures.append('sources.' + claim + ': quotation absent from frozen source') lower = quote.lower() supported = {'terms': 'accept' in lower and any(word in lower for word in ['authoriz', 'authority', 'review']), 'immutable': 'immutable' in lower, 'scope': 'inconclusive' in lower and any(word in lower for word in ['not evidence', 'not establish', 'no observed'])}[claim] if not supported: failures.append('sources.' + claim + ': quotation does not cover the required boundary') return {'passed': not failures, 'reason': 'Complete evidence reconciliation and API/CLI handoff verified.' if not failures else '; '.join(failures)[:3900], 'metrics': {'failedChecks': len(failures)}} except (ValueError, TypeError, AttributeError, KeyError) as error: return {'passed': False, 'reason': 'Invalid answer: ' + str(error)[:3500], 'metrics': {'failedChecks': 1}} if __name__ == '__main__': raw = sys.stdin.buffer.read(2_000_001) if len(raw) > 2_000_000: raise ValueError('Verifier input limit exceeded') envelope = json.loads(raw) print(json.dumps(verify(envelope['output']), allow_nan=False))