#!/usr/bin/env vpython3 # Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Unit tests for instrumentation_test_instance.""" # pylint: disable=protected-access import collections import tempfile import unittest from pylib.base import base_test_result from pylib.instrumentation import instrumentation_test_instance import mock # pylint: disable=import-error _INSTRUMENTATION_TEST_INSTANCE_PATH = ( 'pylib.instrumentation.instrumentation_test_instance.%s' ) class InstrumentationTestInstanceTest(unittest.TestCase): @staticmethod def createTestInstance(): c = _INSTRUMENTATION_TEST_INSTANCE_PATH % 'InstrumentationTestInstance' # yapf: disable with mock.patch('%s._initializeApkAttributes' % c), ( mock.patch('%s._initializeDataDependencyAttributes' % c)), ( mock.patch('%s._initializeTestFilterAttributes' %c)), ( mock.patch('%s._initializeFlagAttributes' % c)), ( mock.patch('%s._initializeTestControlAttributes' % c)), ( mock.patch('%s._initializeTestCoverageAttributes' % c)), ( mock.patch('%s._initializeSkiaGoldAttributes' % c)): # yapf: enable return instrumentation_test_instance.InstrumentationTestInstance( mock.MagicMock(), mock.MagicMock(), lambda s: None) _FlagAttributesArgs = collections.namedtuple( '_FlagAttributesArgs', [ 'command_line_flags', 'device_flags_file', 'strict_mode', 'use_apk_under_test_flags_file', 'coverage_dir', ], ) def createFlagAttributesArgs( self, command_line_flags=None, device_flags_file=None, strict_mode=None, use_apk_under_test_flags_file=False, coverage_dir=None, ): return self._FlagAttributesArgs( command_line_flags, device_flags_file, strict_mode, use_apk_under_test_flags_file, coverage_dir, ) def test_initializeFlagAttributes_commandLineFlags(self): o = self.createTestInstance() args = self.createFlagAttributesArgs( command_line_flags=['--foo', '--bar'] ) o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) def test_initializeFlagAttributes_deviceFlagsFile(self): o = self.createTestInstance() with tempfile.NamedTemporaryFile(mode='w') as flags_file: flags_file.write('\n'.join(['--foo', '--bar'])) flags_file.flush() args = self.createFlagAttributesArgs( device_flags_file=flags_file.name ) o._initializeFlagAttributes(args) self.assertEqual( o._flags, ['--enable-test-intents', '--foo', '--bar'] ) def test_initializeFlagAttributes_strictModeOn(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(strict_mode='on') o._initializeFlagAttributes(args) self.assertEqual( o._flags, ['--enable-test-intents', '--strict-mode=on'] ) def test_initializeFlagAttributes_strictModeOn_coverageOn(self): o = self.createTestInstance() args = self.createFlagAttributesArgs( strict_mode='on', coverage_dir='/coverage/dir' ) o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents']) def test_initializeFlagAttributes_strictModeOff(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(strict_mode='off') o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents']) def testGetTests_noFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Foo']}, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestPositiveAndNegativeFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', } ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = [ 'org.chromium.test.SampleTest.*' '-org.chromium.test.SampleTest.testMethod2' ] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_multipleGtestPositiveAndNegativeFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', } ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = [ 'org.chromium.test.SampleTest*testMethod1', 'org.chromium.test.SampleTest.*' '-org.chromium.test.SampleTest.testMethod2', ] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestUnqualifiedNameFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = ['SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_parameterizedTestGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'SmallTest': None}, 'method': 'testMethod1__sandboxed_mode', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1__sandboxed_mode', }, ] o._junit4_runner_class = 'J4Runner' o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_wildcardGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._test_filters = ['org.chromium.test.SampleTest2.*'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_negativeGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._test_filters = ['*-org.chromium.test.SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._annotations = [('SmallTest', None)] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_excludedAnnotationFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, ] o._excluded_annotations = [('SmallTest', None)] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_excludedDoNotReviveAnnotation(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'DisabledTest': None, 'DoNotRevive': {'reason': 'sample reason'}, }, 'method': 'testMethod1', }, { 'annotations': { 'FlakyTest': None, }, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'FlakyTest': None, 'DoNotRevive': {'reason': 'sample reason'}, }, 'method': 'testMethod1', }, ], }, { 'annotations': {'Feature': {'value': ['Baz']}}, 'class': 'org.chromium.test.SampleTest3', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'FlakyTest': None, 'Manual': {'message': 'sample message'}, }, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'FlakyTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, ] o._excluded_annotations = [('DoNotRevive', None), ('Manual', None)] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationSimpleValueFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'SmallTest': None, 'TestValue': '1', }, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None, 'TestValue': '2', }, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'SmallTest': None, 'TestValue': '3', }, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, 'TestValue': '1', }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._annotations = [('TestValue', '1')] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationDictValueFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._annotations = [('Feature', 'Bar')] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTestName(self): test = { 'annotations': { 'RunWith': {'value': 'class J4Runner'}, 'SmallTest': {}, 'Test': { 'expected': 'class org.junit.Test$None', 'timeout': '0', }, 'UiThreadTest': {}, }, 'class': 'org.chromium.TestA', 'method': 'testSimple', } unqualified_class_test = { 'class': test['class'].split('.')[-1], 'method': test['method'], } self.assertEqual( instrumentation_test_instance.GetTestName(test, sep='.'), 'org.chromium.TestA.testSimple', ) self.assertEqual( instrumentation_test_instance.GetTestName( unqualified_class_test, sep='.' ), 'TestA.testSimple', ) def testGetUniqueTestName(self): test = { 'annotations': { 'RunWith': {'value': 'class J4Runner'}, 'SmallTest': {}, 'Test': { 'expected': 'class org.junit.Test$None', 'timeout': '0', }, 'UiThreadTest': {}, }, 'class': 'org.chromium.TestA', 'flags': ['enable_features=abc'], 'method': 'testSimple', } self.assertEqual( instrumentation_test_instance.GetUniqueTestName(test, sep='.'), 'org.chromium.TestA.testSimple_with_enable_features=abc', ) def testGetTestNameWithoutParameterSuffix(self): test = { 'annotations': { 'RunWith': {'value': 'class J4Runner'}, 'SmallTest': {}, 'Test': { 'expected': 'class org.junit.Test$None', 'timeout': '0', }, 'UiThreadTest': {}, }, 'class': 'org.chromium.TestA__sandbox_mode', 'flags': 'enable_features=abc', 'method': 'testSimple', } unqualified_class_test = { 'class': test['class'].split('.')[-1], 'method': test['method'], } self.assertEqual( instrumentation_test_instance.GetTestNameWithoutParameterSuffix( test, sep='.' ), 'org.chromium.TestA', ) self.assertEqual( instrumentation_test_instance.GetTestNameWithoutParameterSuffix( unqualified_class_test, sep='.' ), 'TestA', ) def testGetTests_multipleAnnotationValuesRequested(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Baz']}, 'MediumTest': None, }, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], }, ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Baz']}, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._annotations = [('Feature', 'Bar'), ('Feature', 'Baz')] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGenerateTestResults_noStatus(self): results = instrumentation_test_instance.GenerateTestResults( None, None, [], 1000, None, None ) self.assertEqual([], results) def testGenerateTestResults_testPassed(self): statuses = [ ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ( 0, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None ) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) def testGenerateTestResults_testFailed(self): statuses = [ ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ( -2, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None ) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) def testGenerateTestResults_testUnknownException(self): stacktrace = 'long\nstacktrace' statuses = [ ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ( -1, { 'class': 'test.package.TestClass', 'test': 'testMethod', 'stack': stacktrace, }, ), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None ) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) def testGenerateTestResults_testSkipped_true(self): statuses = [ ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ( -3, { 'class': 'test.package.TestClass', 'test': 'testMethod', }, ), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None ) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) def testGenerateTestResults_beforeClassFailure(self): stacktrace = 'long\nstacktrace' statuses = [ ( 1, { 'class': 'test.package.TestClass', 'test': 'null', }, ), ( -2, { 'class': 'test.package.TestClass', 'test': 'null', 'stack': stacktrace, }, ), ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }, ), ( 0, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }, ), ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }, ), ( 0, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }, ), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None ) self.assertEqual(2, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(base_test_result.ResultType.FAIL, results[1].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) self.assertEqual(stacktrace, results[1].GetLog()) def testGenerateTestResults_afterClassFailure(self): stacktrace = 'long\nstacktrace' statuses = [ ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }, ), ( 0, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }, ), ( 1, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }, ), ( -3, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }, ), ( 1, { 'class': 'test.package.TestClass', 'test': 'null', }, ), ( -2, { 'class': 'test.package.TestClass', 'test': 'null', 'stack': stacktrace, }, ), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None ) self.assertEqual(2, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(base_test_result.ResultType.SKIP, results[1].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) def testParameterizedCommandLineFlagsSwitches(self): o = self.createTestInstance() raw_tests = [ { 'annotations': { 'ParameterizedCommandLineFlags$Switches': { 'value': ['enable-features=abc', 'enable-features=def'] } }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=ghi', 'enable-features=jkl', ] }, }, 'method': 'testMethod2', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': [] }, }, 'method': 'testMethod3', }, { 'annotations': { 'MediumTest': None, 'SkipCommandLineParameterization': None, }, 'method': 'testMethod4', }, ], } ] expected_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=abc', '--enable-features=def'], 'method': 'testMethod1', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=ghi', '--enable-features=jkl'], 'method': 'testMethod2', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod3', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod4', }, ] for i in range(4): expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations'] ) o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testParameterizedCommandLineFlags(self): o = self.createTestInstance() raw_tests = [ { 'annotations': { 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc', 'force-fieldtrials=trial/group', ], } }, { 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc2', 'force-fieldtrials=trial/group2', ], } }, ], }, }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': ['enable-features=def'] } } ], }, }, 'method': 'testMethod2', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags': { 'value': [], }, }, 'method': 'testMethod3', }, { 'annotations': { 'MediumTest': None, 'SkipCommandLineParameterization': None, }, 'method': 'testMethod4', }, ], } ] expected_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': [ '--enable-features=abc', '--force-fieldtrials=trial/group', ], 'method': 'testMethod1', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=def'], 'method': 'testMethod2', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod3', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod4', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': [ '--enable-features=abc2', '--force-fieldtrials=trial/group2', ], 'method': 'testMethod1', }, ] for i in range(4): expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations'] ) expected_tests[4]['annotations'].update(raw_tests[0]['annotations']) expected_tests[4]['annotations'].update( raw_tests[0]['methods'][0]['annotations'] ) o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testDifferentCommandLineParameterizations(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None, 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': ['a1', 'a2'], } }, ], }, }, 'method': 'testMethod2', }, { 'annotations': { 'SmallTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': ['b1', 'b2'], }, }, 'method': 'testMethod3', }, ], } ] expected_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--a1', '--a2'], 'method': 'testMethod2', }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--b1', '--b2'], 'method': 'testMethod3', }, ] for i in range(2): expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations'] ) o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testMultipleCommandLineParameterizations_raises(self): o = self.createTestInstance() raw_tests = [ { 'annotations': { 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc', 'force-fieldtrials=trial/group', ], } }, ], }, }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc', 'force-fieldtrials=trial/group', ], }, }, 'method': 'testMethod1', }, ], }, ] o._junit4_runner_class = 'J4Runner' self.assertRaises( instrumentation_test_instance.CommandLineParameterizationException, o.ProcessRawTests, [raw_tests[0]], ) if __name__ == '__main__': unittest.main(verbosity=2)