# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the first page of one benchmark for every module.

Only benchmarks that have a composable measurement are included.
Ideally this test would be comprehensive, however, running one page
of every benchmark would run impractically long.
"""

import os
import sys
import unittest

from chrome_telemetry_build import chromium_config

from core import results_processor
from core import testing

from telemetry import benchmark as benchmark_module
from telemetry import decorators
from telemetry.testing import progress_reporter

from py_utils import discover
from py_utils import tempfile_ext

from benchmarks import jetstream2
from benchmarks import octane
from benchmarks import rasterize_and_record_micro
from benchmarks import speedometer1
from benchmarks import v8_browsing


# We want to prevent benchmarks from accidentally trying to upload too much
# data to the chrome perf dashboard. So the smoke tests below cap the max
# number of values that each story tested would produce when running on the
# waterfall.
MAX_VALUES_PER_TEST_CASE = 1000


def SmokeTestGenerator(benchmark_class, num_pages=1, story_tag_filter=None):
  """Generates a smoke test for the first N pages from a benchmark.

  Args:
    benchmark_class: a benchmark class to smoke test.
    num_pages: only smoke test the first N pages, since smoke testing
      everything would take too long to run.
    story_tag_filter: only smoke test stories matching with tags.
  """

  # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
  #
  # This smoke test dynamically tests all benchmarks. So disabling it for one
  # failing or flaky benchmark would disable a much wider swath of coverage
  # than is usually intended. Instead, if a particular benchmark is failing,
  # disable it in tools/perf/benchmarks/*.
  @decorators.Disabled('android')  # crbug.com/641934
  def BenchmarkSmokeTestFunc(self):
    # Some benchmarks are running multiple iterations
    # which is not needed for a smoke test
    if hasattr(benchmark_class, 'enable_smoke_test_mode'):
      benchmark_class.enable_smoke_test_mode = True

    with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
      options = testing.GetRunOptions(
        output_dir=temp_dir,
        benchmark_cls=benchmark_class,
        overrides={
          'story_shard_end_index': num_pages,
          'story_tag_filter': story_tag_filter,
        },
        environment=chromium_config.GetDefaultChromiumConfig(),
      )
      options.pageset_repeat = 1  # For smoke testing only run the page once.
      options.output_formats = ['histograms']
      options.max_values_per_test_case = MAX_VALUES_PER_TEST_CASE
      results_processor.ProcessOptions(options)

      return_code = benchmark_class().Run(options)
      # TODO(crbug.com/40105219): Make 111 be the exit code that means
      # "no stories were run.".
      if return_code in (-1, 111):
        self.skipTest('The benchmark was not run.')
      self.assertEqual(
        return_code, 0, msg='Benchmark run failed: %s' % benchmark_class.Name()
      )
      return_code = results_processor.ProcessResults(options, is_unittest=True)
      self.assertEqual(
        return_code,
        0,
        msg='Result processing failed: %s' % benchmark_class.Name(),
      )

  # Set real_test_func as benchmark_class to make typ
  # write benchmark_class source filepath to trace instead of
  # path to this file
  BenchmarkSmokeTestFunc.real_test_func = benchmark_class

  return BenchmarkSmokeTestFunc


# The list of benchmark modules to be excluded from our smoke tests.
_BLOCK_LIST_TEST_MODULES = {
  octane,  # Often fails & take long time to timeout on cq bot.
  rasterize_and_record_micro,  # Always fails on cq bot.
  speedometer1,  # Takes 101 seconds.
  jetstream2,  # Causes CQ shard to timeout, crbug.com/992837
  v8_browsing,  # Flaky on Android, crbug.com/628368.
}

# The list of benchmark names to be excluded from our smoke tests.
_BLOCK_LIST_TEST_NAMES = [
  'memory.long_running_idle_gmail_background_tbmv2',
  'UNSCHEDULED_ad_frames.iframe',  # b/342449133
  'UNSCHEDULED_oortonline_tbmv2',
  'webrtc',  # crbug.com/932036
  'v8.runtime_stats.top_25',  # Fails in Windows, crbug.com/1043048
  'wasmpspdfkit',  # Fails in Chrome OS, crbug.com/1191938
  'memory.desktop',  # crbug.com/1277277 and b/286898261
  'desktop_ui' if sys.platform == 'darwin' else None,  # crbug.com/1370958
  'power.desktop' if sys.platform == 'darwin' else None,  # crbug.com/1370958
  'omnibox.aim.perf' if sys.platform == 'darwin' else None,  # b/544452037
  (
    'UNSCHEDULED_dummy_wpr_benchmark.loading_using_wpr'
    if sys.platform == 'darwin'
    else None
  ),  # b/544452037
]


def MergeDecorators(method, method_attribute, benchmark, benchmark_attribute):
  # Do set union of attributes to eliminate duplicates.
  merged_attributes = getattr(method, method_attribute, set()).union(
    getattr(benchmark, benchmark_attribute, set())
  )
  if merged_attributes:
    setattr(method, method_attribute, merged_attributes)


class BenchmarkSmokeTest(unittest.TestCase):
  pass


def load_tests(loader, standard_tests, pattern):
  del loader, standard_tests, pattern  # unused
  suite = progress_reporter.TestSuite()

  benchmarks_dir = os.path.dirname(__file__)
  top_level_dir = os.path.dirname(benchmarks_dir)

  # Using the default of |index_by_class_name=False| means that if a module
  # has multiple benchmarks, only the last one is returned.
  all_benchmarks = discover.DiscoverClasses(
    benchmarks_dir,
    top_level_dir,
    benchmark_module.Benchmark,
    index_by_class_name=False,
  ).values()
  for benchmark in all_benchmarks:
    if sys.modules[benchmark.__module__] in _BLOCK_LIST_TEST_MODULES:
      continue
    if benchmark.Name() in _BLOCK_LIST_TEST_NAMES:
      continue

    if 'desktop_ui' in benchmark.Name():
      # Run tests with a specific smoke_test tag.
      method = SmokeTestGenerator(
        benchmark, num_pages=None, story_tag_filter='smoke_test'
      )
    else:
      method = SmokeTestGenerator(benchmark)

    # Make sure any decorators are propagated from the original declaration.
    # (access to protected members) pylint: disable=protected-access
    # TODO(dpranke): Since we only pick the first test from every class
    # (above), if that test is disabled, we'll end up not running *any*
    # test from the class. We should probably discover all of the tests
    # in a class, and then throw the ones we don't need away instead.

    disabled_benchmark_attr = decorators.DisabledAttributeName(benchmark)
    disabled_method_attr = decorators.DisabledAttributeName(method)
    enabled_benchmark_attr = decorators.EnabledAttributeName(benchmark)
    enabled_method_attr = decorators.EnabledAttributeName(method)

    MergeDecorators(
      method, disabled_method_attr, benchmark, disabled_benchmark_attr
    )
    MergeDecorators(
      method, enabled_method_attr, benchmark, enabled_benchmark_attr
    )

    setattr(BenchmarkSmokeTest, benchmark.Name(), method)

    suite.addTest(BenchmarkSmokeTest(benchmark.Name()))

  return suite