#!/usr/bin/env vpython3
# Copyright 2020 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script for determining which GPU tests are unexpectedly passing.

This script depends on the `bb` tool, which is available as part of depot tools,
and the `bq` tool, which is available as part of the Google Cloud SDK
https://cloud.google.com/sdk/docs/quickstarts.

Example usage:

unexpected_pass_finder.py \
  --project <BigQuery billing project> \
  --suite <test suite to check> \

Concrete example:

unexpected_pass_finder.py \
  --project luci-resultdb-dev \
  --suite pixel

You would typically want to pass in --remove-stale-expectations as well in order
to have the script automatically remove any expectations it determines are no
longer necessary. If a particular expectation proves to be erroneously flagged
and removed (e.g. due to a very low flake rate that doesn't get caught
consistently by the script), expectations can be omitted from automatic removal
using an inline `# finder:disable` comment for a single expectation or a pair of
`# finder:disable`/`# finder:enable` comments for a block of expectations.
General disables can be handled via `finder:disable-general` and
`finder:enable-general`. Disabling removal only if the expectation is found to
be unused can be handled via `finder:disable-unused` and `finder:enable-unused`.
Disabling removal only if the expectation is found to be stale can be handled
via `finder:disable-stale` and `finder:enable-stale`.
"""

import argparse
import datetime
import os

from gpu_path_util import setup_telemetry_paths  # pylint: disable=unused-import
from gpu_path_util import setup_testing_paths  # pylint: disable=unused-import

# Must come after path setup.
# pylint: disable=wrong-import-order
from unexpected_passes_common import argument_parsing
from unexpected_passes_common import builders
from unexpected_passes_common import expectations
from unexpected_passes_common import result_output
# pylint: enable=wrong-import-order

from gpu_tests import gpu_integration_test

from unexpected_passes import gpu_builders
from unexpected_passes import gpu_expectations
from unexpected_passes import gpu_queries


def ParseArgs() -> argparse.Namespace:
  name_mapping = gpu_integration_test.GenerateTestNameMapping()
  test_suites = list(name_mapping.keys())
  test_suites.sort()

  parser = argparse.ArgumentParser(
    description=(
      'Script for finding cases of stale expectations that can '
      'be removed/modified.'
    )
  )
  argument_parsing.AddCommonArguments(parser)

  input_group = parser.add_mutually_exclusive_group()
  input_group.add_argument(
    '--expectation-file',
    help='A path to an expectation file to read from. If not specified and '
    '--test is not used, will automatically determine based off the '
    'provided suite.',
  )
  input_group.add_argument(
    '--test',
    action='append',
    dest='tests',
    default=[],
    help='The name of a test to check for unexpected passes. Can be passed '
    'multiple times to specify multiple tests. Will be treated as if it was '
    'expected to be flaky on all configurations.',
  )
  parser.add_argument(
    '--suite',
    required=True,
    choices=test_suites,
    help='The test suite being checked.',
  )

  args = parser.parse_args()
  argument_parsing.PerformCommonPostParseSetup(args)
  suite_class = name_mapping[args.suite]

  if not (args.tests or args.expectation_file):
    expectation_files = suite_class.ExpectationsFiles()
    if not expectation_files:
      raise RuntimeError(
        f'Suite {args.suite} does not specify an expectation file and is '
        f'thus not compatible with this script.'
      )
    if len(expectation_files) > 1:
      raise RuntimeError(
        f'Suite {suite_class} specifies {len(expectation_files)} expectation '
        f'files when only 1 is supported.'
      )
    args.expectation_file = expectation_files[0]

  if args.remove_stale_expectations and not args.expectation_file:
    parser.error(
      '--remove-stale-expectations can only be used with expectation files'
    )

  # Change to whatever repo the test suite claims the expectation file lives in.
  # This allows the script to work for most suites if run from outside of
  # chromium/src. Similarly, it allows suites such as WebGPU CTS that have
  # expectation files in a different repo to be work when run from chromium/src.
  os.chdir(suite_class.GetExpectationsFilesRepoPath())

  return args


# pylint: disable=too-many-locals
def main() -> None:
  args = ParseArgs()

  builders_instance = gpu_builders.GpuBuilders(
    args.suite, args.include_internal_builders
  )
  builders.RegisterInstance(builders_instance)
  expectations_instance = gpu_expectations.GpuExpectations()
  expectations.RegisterInstance(expectations_instance)

  test_expectation_map = expectations_instance.CreateTestExpectationMap(
    args.expectation_file,
    args.tests,
    datetime.timedelta(days=args.expectation_grace_period),
  )
  ci_builders = builders_instance.GetCiBuilders()

  querier = gpu_queries.GpuBigQueryQuerier(
    args.suite, args.project, args.num_samples, args.keep_unmatched_results
  )
  # Unmatched results are mainly useful for script maintainers, as they don't
  # provide any additional information for the purposes of finding unexpectedly
  # passing tests or unused expectations.
  unmatched = querier.FillExpectationMapForBuilders(
    test_expectation_map, ci_builders
  )
  try_builders = builders_instance.GetTryBuilders(ci_builders)
  unmatched.update(
    querier.FillExpectationMapForBuilders(test_expectation_map, try_builders)
  )
  unused_expectations = test_expectation_map.FilterOutUnusedExpectations()
  stale, semi_stale, active = test_expectation_map.SplitByStaleness()
  if args.result_output_file:
    with open(args.result_output_file, 'w', encoding='utf-8') as outfile:
      result_output.OutputResults(
        stale,
        semi_stale,
        active,
        unmatched,
        unused_expectations,
        args.output_format,
        outfile,
      )
  else:
    result_output.OutputResults(
      stale,
      semi_stale,
      active,
      unmatched,
      unused_expectations,
      args.output_format,
    )

  affected_urls = set()
  stale_message = ''
  if args.remove_stale_expectations:
    for expectation_file, expectation_map in stale.items():
      affected_urls |= expectations_instance.RemoveExpectationsFromFile(
        expectation_map.keys(), expectation_file, expectations.RemovalType.STALE
      )
      stale_message += (
        f'Stale expectations removed from {expectation_file}. '
        f'Stale comments, etc. may still need to be removed.\n'
      )
    for expectation_file, unused_list in unused_expectations.items():
      affected_urls |= expectations_instance.RemoveExpectationsFromFile(
        unused_list, expectation_file, expectations.RemovalType.UNUSED
      )
      stale_message += (
        f'Unused expectations removed from {expectation_file}. '
        f'Stale comments, etc. may still need to be removed.\n'
      )

  if args.narrow_semi_stale_expectation_scope:
    affected_urls |= expectations_instance.NarrowSemiStaleExpectationScope(
      semi_stale
    )
    stale_message += (
      f'Semi-stale expectations narrowed in '
      f'{args.expectation_file}. Stale comments, etc. may '
      f'still need to be removed.\n'
    )

  if stale_message:
    print(stale_message)
  if affected_urls:
    orphaned_urls = expectations_instance.FindOrphanedBugs(affected_urls)
    if args.bug_output_file:
      with open(args.bug_output_file, 'w', encoding='utf-8') as bug_outfile:
        result_output.OutputAffectedUrls(
          affected_urls,
          orphaned_urls,
          bug_outfile,
          auto_close_bugs=args.auto_close_bugs,
        )
    else:
      result_output.OutputAffectedUrls(
        affected_urls, orphaned_urls, auto_close_bugs=args.auto_close_bugs
      )


# pylint: enable=too-many-locals


if __name__ == '__main__':
  main()