import copy import json from pathlib import Path import unittest import verify as frozen import verify_v2 as revised class VerifierRegression(unittest.TestCase): def setUp(self): self.good = json.loads((Path(__file__).parent / 'expected.json').read_text()) def test_original_controls_remain_qualified_under_both_versions(self): manifest = json.loads((Path(__file__).parent / 'manifest.json').read_text()) for control in manifest['verifier']['controls']: with self.subTest(control=control['id']): self.assertEqual(frozen.verify(control['output'])['passed'], control['expectedPass']) self.assertEqual(revised.verify(control['output'])['passed'], control['expectedPass']) def test_equivalent_scope_sentences_are_source_grounded_and_accepted(self): for sentence in revised.EQUIVALENT_SCOPE_STATEMENTS: with self.subTest(sentence=sentence): value = copy.deepcopy(self.good) value['sources'][2]['quote'] = sentence output = json.dumps(value) self.assertFalse(frozen.verify(output)['passed']) self.assertTrue(revised.verify(output)['passed']) def test_repair_does_not_hide_other_errors_or_forged_quotations(self): for change in ['cost', 'forgery', 'unrelated']: with self.subTest(change=change): value = copy.deepcopy(self.good) value['sources'][2]['quote'] = revised.EQUIVALENT_SCOPE_STATEMENTS[0] if change == 'cost': value['studies'][0]['costUsd'] = 0 elif change == 'forgery': value['sources'][2]['quote'] += ' Invented supporting context.' else: value['sources'][2]['quote'] = 'The command is `evx`; this archive\'s package is `evals-ax`.' self.assertFalse(revised.verify(json.dumps(value))['passed']) if __name__ == '__main__': unittest.main()