#!/usr/bin/env vpython3 # Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import io import json import logging import os import shutil import sys import tempfile import unittest from unittest import mock import common_merge_script_tests import standard_gtest_merge # gtest json output for successfully finished shard #0. GOOD_GTEST_JSON_0 = { 'all_tests': [ 'AlignedMemoryTest.DynamicAllocation', 'AlignedMemoryTest.ScopedDynamicAllocation', 'AlignedMemoryTest.StackAlignment', 'AlignedMemoryTest.StaticAlignment', ], 'disabled_tests': [ 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay', 'FileTest.TouchGetInfo', 'MessageLoopTestTypeDefault.EnsureDeletion', ], 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'], 'per_iteration_data': [ { 'AlignedMemoryTest.DynamicAllocation': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.ScopedDynamicAllocation': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], } ], 'test_locations': { 'AlignedMemoryTest.DynamicAllocation': { 'file': 'foo/bar/allocation_test.cc', 'line': 123, }, 'AlignedMemoryTest.ScopedDynamicAllocation': { 'file': 'foo/bar/allocation_test.cc', 'line': 456, }, # This is a test from a different shard, but this happens in practice # and we should not fail if information is repeated. 'AlignedMemoryTest.StaticAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 12, }, }, } # gtest json output for successfully finished shard #1. GOOD_GTEST_JSON_1 = { 'all_tests': [ 'AlignedMemoryTest.DynamicAllocation', 'AlignedMemoryTest.ScopedDynamicAllocation', 'AlignedMemoryTest.StackAlignment', 'AlignedMemoryTest.StaticAlignment', ], 'disabled_tests': [ 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay', 'FileTest.TouchGetInfo', 'MessageLoopTestTypeDefault.EnsureDeletion', ], 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'], 'per_iteration_data': [ { 'AlignedMemoryTest.StackAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.StaticAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], } ], 'test_locations': { 'AlignedMemoryTest.StackAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 789, }, 'AlignedMemoryTest.StaticAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 12, }, }, } TIMED_OUT_GTEST_JSON_1 = { 'disabled_tests': [], 'global_tags': [], 'all_tests': [ 'AlignedMemoryTest.DynamicAllocation', 'AlignedMemoryTest.ScopedDynamicAllocation', 'AlignedMemoryTest.StackAlignment', 'AlignedMemoryTest.StaticAlignment', ], 'per_iteration_data': [ { 'AlignedMemoryTest.StackAlignment': [ { 'elapsed_time_ms': 54000, 'losless_snippet': True, 'output_snippet': 'timed out', 'output_snippet_base64': '', 'status': 'FAILURE', } ], 'AlignedMemoryTest.StaticAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': '', 'output_snippet_base64': '', 'status': 'NOTRUN', } ], } ], 'test_locations': { 'AlignedMemoryTest.StackAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 789, }, 'AlignedMemoryTest.StaticAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 12, }, }, } # GOOD_GTEST_JSON_0 and GOOD_GTEST_JSON_1 merged. GOOD_GTEST_JSON_MERGED = { 'all_tests': [ 'AlignedMemoryTest.DynamicAllocation', 'AlignedMemoryTest.ScopedDynamicAllocation', 'AlignedMemoryTest.StackAlignment', 'AlignedMemoryTest.StaticAlignment', ], 'disabled_tests': [ 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay', 'FileTest.TouchGetInfo', 'MessageLoopTestTypeDefault.EnsureDeletion', ], 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'], 'missing_shards': [], 'per_iteration_data': [ { 'AlignedMemoryTest.DynamicAllocation': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.ScopedDynamicAllocation': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.StackAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.StaticAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], } ], 'swarming_summary': { 'shards': [ { 'state': 'COMPLETED', 'outputs_ref': { 'view_url': 'blah', }, } ], }, 'test_locations': { 'AlignedMemoryTest.StackAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 789, }, 'AlignedMemoryTest.StaticAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 12, }, 'AlignedMemoryTest.DynamicAllocation': { 'file': 'foo/bar/allocation_test.cc', 'line': 123, }, 'AlignedMemoryTest.ScopedDynamicAllocation': { 'file': 'foo/bar/allocation_test.cc', 'line': 456, }, }, } # Only shard #1 finished. UNRELIABLE_RESULTS is set. BAD_GTEST_JSON_ONLY_1_SHARD = { 'all_tests': [ 'AlignedMemoryTest.DynamicAllocation', 'AlignedMemoryTest.ScopedDynamicAllocation', 'AlignedMemoryTest.StackAlignment', 'AlignedMemoryTest.StaticAlignment', ], 'disabled_tests': [ 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay', 'FileTest.TouchGetInfo', 'MessageLoopTestTypeDefault.EnsureDeletion', ], 'global_tags': [ 'CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX', 'UNRELIABLE_RESULTS', ], 'missing_shards': [0], 'per_iteration_data': [ { 'AlignedMemoryTest.StackAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.StaticAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], } ], 'test_locations': { 'AlignedMemoryTest.StackAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 789, }, 'AlignedMemoryTest.StaticAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 12, }, }, } # GOOD_GTEST_JSON_0 and TIMED_OUT_GTEST_JSON_1 merged. TIMED_OUT_GTEST_JSON_MERGED = { 'all_tests': [ 'AlignedMemoryTest.DynamicAllocation', 'AlignedMemoryTest.ScopedDynamicAllocation', 'AlignedMemoryTest.StackAlignment', 'AlignedMemoryTest.StaticAlignment', ], 'disabled_tests': [ 'ConditionVariableTest.TimeoutAcrossSetTimeOfDay', 'FileTest.TouchGetInfo', 'MessageLoopTestTypeDefault.EnsureDeletion', ], 'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'], 'missing_shards': [], 'per_iteration_data': [ { 'AlignedMemoryTest.DynamicAllocation': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.ScopedDynamicAllocation': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': 'blah\\n', 'output_snippet_base64': 'YmxhaAo=', 'status': 'SUCCESS', } ], 'AlignedMemoryTest.StackAlignment': [ { 'elapsed_time_ms': 54000, 'losless_snippet': True, 'output_snippet': 'timed out', 'output_snippet_base64': '', 'status': 'FAILURE', } ], 'AlignedMemoryTest.StaticAlignment': [ { 'elapsed_time_ms': 0, 'losless_snippet': True, 'output_snippet': '', 'output_snippet_base64': '', 'status': 'NOTRUN', } ], } ], 'swarming_summary': { 'shards': [ { 'state': 'COMPLETED', }, { 'state': 'TIMED_OUT', }, ], }, 'test_locations': { 'AlignedMemoryTest.StackAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 789, }, 'AlignedMemoryTest.StaticAlignment': { 'file': 'foo/bar/allocation_test.cc', 'line': 12, }, 'AlignedMemoryTest.DynamicAllocation': { 'file': 'foo/bar/allocation_test.cc', 'line': 123, }, 'AlignedMemoryTest.ScopedDynamicAllocation': { 'file': 'foo/bar/allocation_test.cc', 'line': 456, }, }, } class _StandardGtestMergeTest(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir) def _write_temp_file(self, path, content): abs_path = os.path.join(self.temp_dir, path.replace('/', os.sep)) if not os.path.exists(os.path.dirname(abs_path)): os.makedirs(os.path.dirname(abs_path)) with open(abs_path, 'w') as f: if isinstance(content, dict): json.dump(content, f) else: assert isinstance(content, str) f.write(content) return abs_path class LoadShardJsonTest(_StandardGtestMergeTest): def test_double_digit_jsons(self): jsons_to_merge = [] for i in range(15): json_dir = os.path.join(self.temp_dir, str(i)) json_path = os.path.join(json_dir, 'output.json') if not os.path.exists(json_dir): os.makedirs(json_dir) with open(json_path, 'w') as f: json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f) jsons_to_merge.append(json_path) content, err = standard_gtest_merge.load_shard_json(0, None, jsons_to_merge) self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content) self.assertIsNone(err) content, err = standard_gtest_merge.load_shard_json( 12, None, jsons_to_merge ) self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content) self.assertIsNone(err) def test_double_task_id_jsons(self): jsons_to_merge = [] for i in range(15): json_dir = os.path.join(self.temp_dir, 'deadbeef%d' % i) json_path = os.path.join(json_dir, 'output.json') if not os.path.exists(json_dir): os.makedirs(json_dir) with open(json_path, 'w') as f: json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f) jsons_to_merge.append(json_path) content, err = standard_gtest_merge.load_shard_json( 0, 'deadbeef0', jsons_to_merge ) self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content) self.assertIsNone(err) content, err = standard_gtest_merge.load_shard_json( 12, 'deadbeef12', jsons_to_merge ) self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content) self.assertIsNone(err) class MergeShardResultsTest(_StandardGtestMergeTest): """Tests for merge_shard_results function.""" # pylint: disable=super-with-arguments def setUp(self): super(MergeShardResultsTest, self).setUp() self.summary = None self.test_files = [] # pylint: enable=super-with-arguments def stage(self, summary, files): self.summary = self._write_temp_file('summary.json', summary) for path, content in files.items(): abs_path = self._write_temp_file(path, content) self.test_files.append(abs_path) def call(self): stdout = io.StringIO() with mock.patch('sys.stdout', stdout): merged = standard_gtest_merge.merge_shard_results( self.summary, self.test_files ) return merged, stdout.getvalue().strip() def assertUnicodeEquals(self, expectation, result): def convert_to_unicode(key_or_value): if isinstance(key_or_value, str): return str(key_or_value) if isinstance(key_or_value, dict): return { convert_to_unicode(k): convert_to_unicode(v) for k, v in key_or_value.items() } if isinstance(key_or_value, list): return [convert_to_unicode(x) for x in key_or_value] return key_or_value unicode_expectations = convert_to_unicode(expectation) unicode_result = convert_to_unicode(result) self.assertEqual(unicode_expectations, unicode_result) def test_ok(self): # Two shards, both successfully finished. self.stage( { 'shards': [ { 'state': 'COMPLETED', }, { 'state': 'COMPLETED', }, ], }, { '0/output.json': GOOD_GTEST_JSON_0, '1/output.json': GOOD_GTEST_JSON_1, }, ) merged, stdout = self.call() merged['swarming_summary'] = { 'shards': [ { 'state': 'COMPLETED', 'outputs_ref': { 'view_url': 'blah', }, } ], } self.assertUnicodeEquals(GOOD_GTEST_JSON_MERGED, merged) self.assertEqual('', stdout) def test_timed_out(self): # Two shards, both successfully finished. self.stage( { 'shards': [ { 'state': 'COMPLETED', }, { 'state': 'TIMED_OUT', }, ], }, { '0/output.json': GOOD_GTEST_JSON_0, '1/output.json': TIMED_OUT_GTEST_JSON_1, }, ) merged, stdout = self.call() self.assertUnicodeEquals(TIMED_OUT_GTEST_JSON_MERGED, merged) self.assertIn('Test runtime exceeded allocated time\n', stdout) def test_missing_summary_json(self): # summary.json is missing, should return None and emit warning. self.summary = os.path.join(self.temp_dir, 'summary.json') merged, output = self.call() self.assertEqual(None, merged) self.assertIn('@@@STEP_WARNINGS@@@', output) self.assertIn('summary.json is missing or can not be read', output) def test_unfinished_shards(self): # Only one shard (#1) finished. Shard #0 did not. self.stage( { 'shards': [ None, { 'state': 'COMPLETED', }, ], }, { '1/output.json': GOOD_GTEST_JSON_1, }, ) merged, stdout = self.call() merged.pop('swarming_summary') self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged) self.assertIn( '@@@STEP_WARNINGS@@@\nsome shards did not complete: 0\n', stdout ) self.assertIn( '@@@STEP_LOG_LINE@some shards did not complete: 0@' 'Missing results from the following shard(s): 0@@@\n', stdout, ) def test_missing_output_json(self): # Shard #0 output json is missing. self.stage( { 'shards': [ { 'state': 'COMPLETED', }, { 'state': 'COMPLETED', }, ], }, { '1/output.json': GOOD_GTEST_JSON_1, }, ) merged, stdout = self.call() merged.pop('swarming_summary') self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged) self.assertIn( 'No result was found: shard 0 test output was missing', stdout ) def test_large_output_json(self): # a shard is too large. self.stage( { 'shards': [ { 'state': 'COMPLETED', }, { 'state': 'COMPLETED', }, ], }, { '0/output.json': GOOD_GTEST_JSON_0, '1/output.json': GOOD_GTEST_JSON_1, }, ) old_json_limit = standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT len0 = len(json.dumps(GOOD_GTEST_JSON_0)) len1 = len(json.dumps(GOOD_GTEST_JSON_1)) large_shard = '0' if len0 > len1 else '1' try: # Override max output.json size just for this test. standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = min(len0, len1) merged, stdout = self.call() merged.pop('swarming_summary') self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged) self.assertIn( 'No result was found: ' 'shard %s test output exceeded the size limit' % large_shard, stdout, ) finally: standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = old_json_limit class CommandLineTest(common_merge_script_tests.CommandLineTest): # pylint: disable=super-with-arguments def __init__(self, methodName='runTest'): super(CommandLineTest, self).__init__(methodName, standard_gtest_merge) # pylint: enable=super-with-arguments if __name__ == '__main__': logging.basicConfig( level=logging.DEBUG if '-v' in sys.argv else logging.ERROR ) if '-v' in sys.argv: unittest.TestCase.maxDiff = None unittest.main()