# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from recipe_engine import recipe_api from dataclasses import dataclass, field class Constants: def __init__(self): self.NONTRIVIAL_ROLL_FOOTER = "Recipe-Nontrivial-Roll" self.MANUAL_CHANGE_FOOTER = "Recipe-Manual-Change" self.BYPASS_FOOTER = "Recipe-Tryjob-Bypass-Reason" self.SKIP_RETRY_FOOTER = "Disable-Retries" self.CQ_DEPEND_FOOTER = "Cq-Depend" self.ALL_VALID_FOOTERS = set( [ self.NONTRIVIAL_ROLL_FOOTER, self.MANUAL_CHANGE_FOOTER, self.BYPASS_FOOTER, self.SKIP_RETRY_FOOTER, self.CQ_DEPEND_FOOTER, ] ) constants = Constants() @dataclass(frozen=True) class SubmodulePathsResult: """Holds the results of submodule diff expansion.""" # List of all affected files (including both main repository files # and the expanded files inside any checked-out submodules). affected_files: list[str] = field(default_factory=list) # Submodules that were not checked out by the builder's gclient config. unchecked_out_submodules: list[str] = field(default_factory=list) # Submodules that were deleted by the patch. deleted_submodules: list[str] = field(default_factory=list) # Submodules that were newly added by the patch. new_submodules: list[str] = field(default_factory=list) # Nested submodules that were detected (and thus skipped from recursion). nested_submodules: list[str] = field(default_factory=list) class TryserverApi(recipe_api.RecipeApi): def __init__(self, *args, **kwargs): super(TryserverApi, self).__init__(*args, **kwargs) self._gerrit_change = None # self.m.buildbucket.common_pb2.GerritChange self._gerrit_change_repo_url = None self._gerrit_change_repo_host = None self._gerrit_change_repo_project = None self._gerrit_info_initialized = False self._gerrit_change_target_ref = None self._gerrit_change_fetch_ref = None self._gerrit_change_owner = None self._change_footers = None self._gerrit_commit_message = None def initialize(self): changes = self.m.buildbucket.build.input.gerrit_changes if len(changes) == 1: self.set_change(changes[0]) @property def valid_footers(self): # pragma: nocover return constants.ALL_VALID_FOOTERS @property def constants(self): # pragma: nocover # Nocover to be removed when callers (not within depot_tools) exercise this return constants @property def gerrit_change(self): """Returns current gerrit change, if there is exactly one. Returns a self.m.buildbucket.common_pb2.GerritChange or None. """ return self._gerrit_change @property def gerrit_change_repo_url(self): """Returns canonical URL of the gitiles repo of the current Gerrit CL. Populated iff gerrit_change is populated. """ return self._gerrit_change_repo_url @property def gerrit_change_repo_host(self): """Returns the host of the gitiles repo of the current Gerrit CL. Populated iff gerrit_change is populated. """ return self._gerrit_change_repo_host @property def gerrit_change_repo_project(self): """Returns the project of the gitiles repo of the current Gerrit CL. Populated iff gerrit_change is populated. """ return self._gerrit_change_repo_project @property def gerrit_change_owner(self): """Returns owner of the current Gerrit CL. Populated iff gerrit_change is populated. Is a dictionary with keys like "name". """ self._ensure_gerrit_change_info() return self._gerrit_change_owner @property def gerrit_change_review_url(self): """Returns the review URL for the active patchset.""" # Gerrit redirects to insert the project into the URL. gerrit_change = self._gerrit_change return "https://%s/c/%s/%s" % ( gerrit_change.host, gerrit_change.change, gerrit_change.patchset, ) def _ensure_gerrit_change_info(self): """Initializes extra info about gerrit_change, fetched from Gerrit server. Initializes _gerrit_change_target_ref and _gerrit_change_fetch_ref. May emit a step when called for the first time. """ cl = self.gerrit_change if not cl: # pragma: no cover return if self._gerrit_info_initialized: return td = self._test_data if self._test_data.enabled else {} mock_res = [ { "branch": td.get("gerrit_change_target_ref", "main"), "revisions": { "184ebe53805e102605d11f6b143486d15c23a09c": { "_number": str(cl.patchset), "ref": "refs/changes/%02d/%d/%d" % (cl.change % 100, cl.change, cl.patchset), }, }, "owner": { "name": "John Doe", }, } ] res = self.m.gerrit.get_changes( host="https://" + cl.host, query_params=[("change", cl.change)], # This list must remain static/hardcoded. # If you need extra info, either change it here (hardcoded) or # fetch separately. o_params=["ALL_REVISIONS", "DOWNLOAD_COMMANDS"], limit=1, name="fetch current CL info", timeout=480, step_test_data=lambda: self.m.json.test_api.output(mock_res), )[0] self._gerrit_change_target_ref = res["branch"] if not self._gerrit_change_target_ref.startswith("refs/"): self._gerrit_change_target_ref = ( "refs/heads/" + self._gerrit_change_target_ref ) for rev in res["revisions"].values(): if int(rev["_number"]) == self.gerrit_change.patchset: self._gerrit_change_fetch_ref = rev["ref"] break self._gerrit_change_owner = dict(res["owner"]) self._gerrit_info_initialized = True @property def gerrit_change_fetch_ref(self): """Returns gerrit patch ref, e.g. "refs/heads/45/12345/6, or None. Populated iff gerrit_change is populated. """ self._ensure_gerrit_change_info() return self._gerrit_change_fetch_ref @property def gerrit_change_target_ref(self): """Returns gerrit change destination ref, e.g. "refs/heads/main". Populated iff gerrit_change is populated. """ self._ensure_gerrit_change_info() return self._gerrit_change_target_ref @property def gerrit_change_number(self): """Returns gerrit change patchset, e.g. 12345 for a patch ref of "refs/heads/45/12345/6". Populated iff gerrit_change is populated. Returns None if not populated. """ self._ensure_gerrit_change_info() if not self._gerrit_change: # pragma: nocover return None return int(self._gerrit_change.change) @property def gerrit_patchset_number(self): """Returns gerrit change patchset, e.g. 6 for a patch ref of "refs/heads/45/12345/6". Populated iff gerrit_change is populated Returns None if not populated.. """ self._ensure_gerrit_change_info() if not self._gerrit_change: # pragma: nocover return None return int(self._gerrit_change.patchset) @property def is_tryserver(self): """Returns true iff we have a change to check out.""" return self.is_patch_in_git or self.is_gerrit_issue @property def is_gerrit_issue(self): """Returns true iff the properties exist to match a Gerrit issue.""" if self.gerrit_change: return True # TODO(tandrii): remove this, once nobody is using buildbot Gerrit Poller. return ( "event.patchSet.ref" in self.m.properties and "event.change.url" in self.m.properties and "event.change.id" in self.m.properties ) @property def is_patch_in_git(self): return ( self.m.properties.get("patch_storage") == "git" and self.m.properties.get("patch_repo_url") and self.m.properties.get("patch_ref") ) def require_is_tryserver(self): if self.m.tryserver.is_tryserver: return status = self.m.step.EXCEPTION step_text = "This recipe requires a gerrit CL for the source under test" if self.m.led.launched_by_led: status = self.m.step.FAILURE step_text += ( "\n run 'led edit-cr-cl ' to attach a CL to test" ) self.m.step.empty("not a tryjob", status=status, step_text=step_text) def get_files_affected_by_patch( self, patch_root, report_files_via_property=None, **kwargs ): """Returns list of paths to files affected by the patch. Args: * patch_root: path relative to api.path['root'], usually obtained from api.gclient.get_gerrit_patch_root(). * report_files_via_property: name of the output property to report the list of the files. If None (default), do not report. Returned paths will be relative to to api.path['root']. """ cwd = self.m.context.cwd or self.m.path.start_dir / patch_root with self.m.context(cwd=cwd): step_result = self.m.git( "-c", "core.quotePath=false", "diff", "--cached", "--name-only", name="git diff to analyze patch", stdout=self.m.raw_io.output(), step_test_data=lambda: self.m.raw_io.test_api.stream_output("foo.cc"), **kwargs, ) paths = [ self.m.path.join(patch_root, p.decode("utf-8")) for p in step_result.stdout.splitlines() ] paths.sort() if self.m.platform.is_win: # Looks like "analyze" wants POSIX slashes even on Windows (since git # uses that format even on Windows). paths = [path.replace("\\", "/") for path in paths] step_result.presentation.logs["files"] = paths if report_files_via_property: step_result.presentation.properties[report_files_via_property] = { "total_count": len(paths), # Do not report too many because it might violate build size limits, # and isn't very useful anyway. "first_100": paths[:100], } return paths def get_files_affected_by_patch_with_submodules( self, patch_root, report_files_via_property=None, **kwargs ) -> SubmodulePathsResult: """Returns a SubmodulePathsResult containing affected files and submodule metadata. This is an extended version of get_files_affected_by_patch that also detects and expands submodule diffs. """ try: return self._get_files_affected_by_patch_with_submodules( patch_root, report_files_via_property=report_files_via_property, **kwargs, ) except Exception as ex: # Temporarily prevent any submodule expansion errors from failing the # build while this function is run as an experiment and is only # logging its results. self.m.step.empty( "[Experimental] submodule expansion failed (ignored)", status=self.m.step.WARNING, step_text=str(ex), raise_on_failure=False, ) return SubmodulePathsResult( affected_files=[], unchecked_out_submodules=[], deleted_submodules=[], new_submodules=[], nested_submodules=[], ) def _get_files_affected_by_patch_with_submodules( self, patch_root, report_files_via_property=None, **kwargs ) -> SubmodulePathsResult: """Internal implementation of get_files_affected_by_patch_with_submodules. Args: * patch_root: path relative to api.path['root'], usually obtained from api.gclient.get_gerrit_patch_root(). * report_files_via_property: name of the output property to report the list of the files. If None (default), do not report. """ affected_files = [] unchecked_out_submodules = [] deleted_submodules = [] new_submodules = [] nested_submodules = [] submodule_files = [] submodules_to_process = [] def parse_raw_diff(step_result): """Parses 'git diff --raw' output into (mode, old_sha, new_sha, path).""" NULL_SHA = "0" * 40 for raw_line in step_result.stdout.splitlines(): line = raw_line.decode("utf-8") # each line of a --raw diff is expected to begin with ':' and to have # a tab separating the metadata from the path to the modified file(s) # example: ":100644 100644 bcd1234 0123456 M file0" if line.startswith(":") and "\t" in line: # remove leading ':' and split metadata from path(s) meta, path = line[1:].split("\t", 1) meta_parts = meta.split() if len(meta_parts) != 5: raise ValueError( f"Unexpected git diff --raw metadata (expected 5 fields, got " f"{len(meta_parts)}): {meta!r}" ) old_mode, new_mode, old_sha, new_sha, status = meta_parts if status[0] in ("R", "C"): # In Rename or Copy operation the path contains src_path\tdst_path # example: ":100644 100644 abcd123 1234567 R86 file1 file3" if "\t" not in path: raise ValueError( f"Unexpected git diff --raw path for {status[0]} operation " f"(expected tab-separated src and dst paths): {path!r}" ) src_path, dst_path = path.split("\t", 1) if status[0] == "R": # Yield src_path as deleted for renames only (using its old_mode) yield old_mode, old_sha, NULL_SHA, src_path # Yield dst_path as added for both renames and copies yield new_mode, NULL_SHA, new_sha, dst_path else: # Statuses: A, D, M, T (change in type) and U (file is unmerged) yield new_mode, old_sha, new_sha, path def attach_submodule_log(presentation, files, log_name="submodule_files"): if files: files.sort() if self.m.platform.is_win: files = [path.replace("\\", "/") for path in files] presentation.logs[log_name] = files def finalize_and_log(presentation, files): files.sort() if self.m.platform.is_win: files = [path.replace("\\", "/") for path in files] if report_files_via_property: def format_prop_list(lst): sorted_lst = sorted(lst) if self.m.platform.is_win: sorted_lst = [p.replace("\\", "/") for p in sorted_lst] return { "total_count": len(sorted_lst), "first_100": sorted_lst[:100], } prop_dict = format_prop_list(files) prop_dict.update( { "submodule_files": format_prop_list(submodule_files), "unchecked_out_submodules": format_prop_list( unchecked_out_submodules ), "deleted_submodules": format_prop_list(deleted_submodules), "new_submodules": format_prop_list(new_submodules), "nested_submodules": format_prop_list(nested_submodules), } ) presentation.properties[report_files_via_property] = prop_dict presentation.logs["files"] = files return files cwd = self.m.context.cwd or self.m.path.start_dir / patch_root with self.m.context(cwd=cwd): step_result = self.m.git( "-c", "core.quotePath=false", "diff", "--cached", "--raw", name="[Experimental] git diff --raw to analyze patch", stdout=self.m.raw_io.output(add_output_log=True), step_test_data=lambda: self.m.raw_io.test_api.stream_output(""), **kwargs, ) # 1. Collect all affected paths (including submodule gitlinks) and identify submodules for mode, old_sha, new_sha, rel_path in parse_raw_diff(step_result): affected_files.append(self.m.path.join(patch_root, rel_path)) if mode == "160000": submodules_to_process.append((old_sha, new_sha, rel_path)) # 2. Process submodules if there are any if submodules_to_process: with self.m.context(cwd=cwd): with self.m.step.nest( "[Experimental] git diff submodules" ) as presentation: for old_sha, new_sha, rel_path in submodules_to_process: if new_sha.startswith("0" * 7): # If a submodule is deleted (new_commit starts with 0s), we skip # expanding its files. deleted_submodules.append(rel_path) continue submodule_dir = cwd / rel_path # If the submodule directory does not contain a .git file/directory, # it means the submodule was not checked out by this builder's # gclient/checkout configuration (e.g. platform-specific or # internal-only deps). if not self.m.path.exists(submodule_dir / ".git"): unchecked_out_submodules.append(rel_path) continue if old_sha.startswith("0" * 7): new_submodules.append(rel_path) # 4b825dc642cb6eb9a060e54bf8d69288fbee4904 is Git's empty tree hash. # Used to diff newly added submodules (which have a null old SHA). old_sha = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # this call to git diff is what actually expands the submodule sub_result = self.m.git( "-C", rel_path, "diff", "--raw", old_sha, new_sha, name=rel_path, stdout=self.m.raw_io.output(add_output_log=True), step_test_data=lambda: self.m.raw_io.test_api.stream_output( ":100644 100644 1234567 89abcdef M\tsub_foo.cc" ), ) step_files = [] for sub_mode, _, _, sub_path in parse_raw_diff(sub_result): # Do not recursively expand nested submodules because tryjob checkouts # often perform shallow syncs or only check out the tip commit, meaning older # revisions are missing locally. if sub_mode == "160000": nested_submodules.append("%s/%s" % (rel_path, sub_path)) else: sub_file = self.m.path.join(patch_root, rel_path, sub_path) step_files.append(sub_file) submodule_files.append(sub_file) affected_files.append(sub_file) attach_submodule_log(sub_result.presentation, step_files) attach_submodule_log( presentation, unchecked_out_submodules, "unchecked_out_submodules" ) attach_submodule_log( presentation, deleted_submodules, "deleted_submodules" ) attach_submodule_log(presentation, new_submodules, "new_submodules") attach_submodule_log( presentation, nested_submodules, "nested_submodules" ) affected_files = finalize_and_log(presentation, affected_files) else: affected_files = finalize_and_log( step_result.presentation, affected_files ) return SubmodulePathsResult( affected_files=affected_files, unchecked_out_submodules=unchecked_out_submodules, deleted_submodules=deleted_submodules, new_submodules=new_submodules, nested_submodules=nested_submodules, ) def set_subproject_tag(self, subproject_tag): """Adds a subproject tag to the build. This can be used to distinguish between builds that execute different steps depending on what was patched, e.g. blink vs. pure chromium patches. """ assert self.is_tryserver step_result = self.m.step("TRYJOB SET SUBPROJECT_TAG", cmd=None) step_result.presentation.properties["subproject_tag"] = subproject_tag step_result.presentation.step_text = subproject_tag def _set_failure_type(self, failure_type): if not self.is_tryserver: return # TODO(iannucci): add API to set properties regardless of the current step. step_result = self.m.step("TRYJOB FAILURE", cmd=None) step_result.presentation.properties["failure_type"] = failure_type step_result.presentation.step_text = failure_type step_result.presentation.status = "FAILURE" def set_patch_failure_tryjob_result(self): """Mark the tryjob result as failure to apply the patch.""" self._set_failure_type("PATCH_FAILURE") def set_compile_failure_tryjob_result(self): """Mark the tryjob result as a compile failure.""" self._set_failure_type("COMPILE_FAILURE") def set_test_failure_tryjob_result(self): """Mark the tryjob result as a test failure. This means we started running actual tests (not prerequisite steps like checkout or compile), and some of these tests have failed. """ self._set_failure_type("TEST_FAILURE") def set_invalid_test_results_tryjob_result(self): """Mark the tryjob result as having invalid test results. This means we run some tests, but the results were not valid (e.g. no list of specific test cases that failed, or too many tests failing, etc). """ self._set_failure_type("INVALID_TEST_RESULTS") def set_test_timeout_tryjob_result(self): """Mark the tryjob result as a test timeout. This means tests were scheduled but didn't finish executing within the timeout. """ self._set_failure_type("TEST_TIMEOUT") def set_test_expired_tryjob_result(self): """Mark the tryjob result as a test expiration. This means a test task expired and was never scheduled, most likely due to lack of capacity. """ self._set_failure_type("TEST_EXPIRED") def get_footers(self, patch_text=None): """Retrieves footers from the patch description. footers are machine readable tags embedded in commit messages. See git-footers documentation for more information. """ return self._get_footers(patch_text) def _ensure_gerrit_commit_message(self): """Fetch full commit message for Gerrit change.""" if self._gerrit_commit_message: return self._ensure_gerrit_change_info() self._gerrit_commit_message = self.m.gerrit.get_change_description( "https://%s" % self.gerrit_change.host, self.gerrit_change_number, self.gerrit_patchset_number, timeout=480, ) def _get_footers(self, patch_text=None): if patch_text is not None: return self._get_footer_step(patch_text) if self._change_footers is not None: return self._change_footers if self.gerrit_change: self._ensure_gerrit_commit_message() self._change_footers = self._get_footer_step(self._gerrit_commit_message) return self._change_footers raise Exception( "No patch text or associated changelist, cannot get footers" ) # pragma: nocover def _get_footer_step(self, patch_text): result = self.m.step( "parse description", [ "python3", self.repo_resource("git_footers.py"), "--json", self.m.json.output(), ], stdin=self.m.raw_io.input(data=patch_text), step_test_data=lambda: self.m.json.test_api.output({}), ) return result.json.output def get_footer(self, tag, patch_text=None): """Gets a specific tag from a CL description""" footers = self._get_footers(patch_text) return footers.get(tag, []) def normalize_footer_name(self, footer): return "-".join([word.title() for word in footer.strip().split("-")]) def get_change_description(self): """Gets the CL description.""" self._ensure_gerrit_commit_message() return self._gerrit_commit_message def set_change(self, change): """Set the gerrit change for this module. Args: * change: a self.m.buildbucket.common_pb2.GerritChange. """ self._gerrit_info_initialized = False self._gerrit_change = change gs_suffix = "-review.googlesource.com" host = change.host if host.endswith(gs_suffix): host = "%s.googlesource.com" % host[: -len(gs_suffix)] self._gerrit_change_repo_url = "https://%s/%s" % (host, change.project) self._gerrit_change_repo_host = host self._gerrit_change_repo_project = change.project