| #!/usr/bin/env python3 |
| # |
| # ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==# |
| # |
| # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| # See https://llvm.org/LICENSE.txt for license information. |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| # |
| # ==-------------------------------------------------------------------------==# |
| |
| import argparse |
| from git import Repo # type: ignore |
| import github |
| import os |
| import re |
| import requests |
| import sys |
| import time |
| from typing import List, Optional |
| |
| beginner_comment = """ |
| Hi! |
| |
| This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are: |
| |
| 1) Assign the issue to you. |
| 2) Fix the issue locally. |
| 3) [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. |
| 3.1) Remember that the subdirectories under `test/` create fine-grained testing targets, so you can |
| e.g. use `make check-clang-ast` to only run Clang's AST tests. |
| 4) Create a `git` commit |
| 5) Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. |
| 6) Submit the patch to [Phabricator](https://reviews.llvm.org/). |
| 6.1) Detailed instructions can be found [here](https://llvm.org/docs/Phabricator.html#requesting-a-review-via-the-web-interface) |
| |
| For more instructions on how to submit a patch to LLVM, see our [documentation](https://llvm.org/docs/Contributing.html). |
| |
| If you have any further questions about this issue, don't hesitate to ask via a comment on this Github issue. |
| """ |
| |
| |
| class IssueSubscriber: |
| @property |
| def team_name(self) -> str: |
| return self._team_name |
| |
| def __init__(self, token: str, repo: str, issue_number: int, label_name: str): |
| self.repo = github.Github(token).get_repo(repo) |
| self.org = github.Github(token).get_organization(self.repo.organization.login) |
| self.issue = self.repo.get_issue(issue_number) |
| self._team_name = "issue-subscribers-{}".format(label_name).lower() |
| |
| def run(self) -> bool: |
| for team in self.org.get_teams(): |
| if self.team_name != team.name.lower(): |
| continue |
| |
| comment = "" |
| if team.slug == "issue-subscribers-good-first-issue": |
| comment = "{}\n".format(beginner_comment) |
| |
| comment += "@llvm/{}".format(team.slug) |
| self.issue.create_comment(comment) |
| return True |
| return False |
| |
| |
| def setup_llvmbot_git(git_dir="."): |
| """ |
| Configure the git repo in `git_dir` with the llvmbot account so |
| commits are attributed to llvmbot. |
| """ |
| repo = Repo(git_dir) |
| with repo.config_writer() as config: |
| config.set_value("user", "name", "llvmbot") |
| config.set_value("user", "email", "llvmbot@llvm.org") |
| |
| |
| def phab_api_call(phab_token: str, url: str, args: dict) -> dict: |
| """ |
| Make an API call to the Phabricator web service and return a dictionary |
| containing the json response. |
| """ |
| data = {"api.token": phab_token} |
| data.update(args) |
| response = requests.post(url, data=data) |
| return response.json() |
| |
| |
| def phab_login_to_github_login( |
| phab_token: str, repo: github.Repository.Repository, phab_login: str |
| ) -> Optional[str]: |
| """ |
| Tries to translate a Phabricator login to a github login by |
| finding a commit made in Phabricator's Differential. |
| The commit's SHA1 is then looked up in the github repo and |
| the committer's login associated with that commit is returned. |
| |
| :param str phab_token: The Conduit API token to use for communication with Pabricator |
| :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential |
| :param str phab_login: The Phabricator login to be translated. |
| """ |
| |
| args = { |
| "constraints[authors][0]": phab_login, |
| # PHID for "LLVM Github Monorepo" repository |
| "constraints[repositories][0]": "PHID-REPO-f4scjekhnkmh7qilxlcy", |
| "limit": 1, |
| } |
| # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/ |
| r = phab_api_call( |
| phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args |
| ) |
| data = r["result"]["data"] |
| if len(data) == 0: |
| # Can't find any commits associated with this user |
| return None |
| |
| commit_sha = data[0]["fields"]["identifier"] |
| committer = repo.get_commit(commit_sha).committer |
| if not committer: |
| # This committer had an email address GitHub could not recognize, so |
| # it can't link the user to a GitHub account. |
| print(f"Warning: Can't find github account for {phab_login}") |
| return None |
| return committer.login |
| |
| |
| def phab_get_commit_approvers(phab_token: str, commit: github.Commit.Commit) -> list: |
| args = {"corpus": commit.commit.message} |
| # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/ |
| r = phab_api_call( |
| phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args |
| ) |
| review_id = r["result"]["revisionIDFieldInfo"]["value"] |
| if not review_id: |
| # No Phabricator revision for this commit |
| return [] |
| |
| args = {"constraints[ids][0]": review_id, "attachments[reviewers]": True} |
| # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/ |
| r = phab_api_call( |
| phab_token, "https://reviews.llvm.org/api/differential.revision.search", args |
| ) |
| reviewers = r["result"]["data"][0]["attachments"]["reviewers"]["reviewers"] |
| accepted = [] |
| for reviewer in reviewers: |
| if reviewer["status"] != "accepted": |
| continue |
| phid = reviewer["reviewerPHID"] |
| args = {"constraints[phids][0]": phid} |
| # API documentation: https://reviews.llvm.org/conduit/method/user.search/ |
| r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args) |
| accepted.append(r["result"]["data"][0]["fields"]["username"]) |
| return accepted |
| |
| |
| def extract_commit_hash(arg: str): |
| """ |
| Extract the commit hash from the argument passed to /action github |
| comment actions. We currently only support passing the commit hash |
| directly or use the github URL, such as |
| https://github.com/llvm/llvm-project/commit/2832d7941f4207f1fcf813b27cf08cecc3086959 |
| """ |
| github_prefix = "https://github.com/llvm/llvm-project/commit/" |
| if arg.startswith(github_prefix): |
| return arg[len(github_prefix) :] |
| return arg |
| |
| |
| class ReleaseWorkflow: |
| |
| CHERRY_PICK_FAILED_LABEL = "release:cherry-pick-failed" |
| |
| """ |
| This class implements the sub-commands for the release-workflow command. |
| The current sub-commands are: |
| * create-branch |
| * create-pull-request |
| |
| The execute_command method will automatically choose the correct sub-command |
| based on the text in stdin. |
| """ |
| |
| def __init__( |
| self, |
| token: str, |
| repo: str, |
| issue_number: int, |
| branch_repo_name: str, |
| branch_repo_token: str, |
| llvm_project_dir: str, |
| phab_token: str, |
| ) -> None: |
| self._token = token |
| self._repo_name = repo |
| self._issue_number = issue_number |
| self._branch_repo_name = branch_repo_name |
| if branch_repo_token: |
| self._branch_repo_token = branch_repo_token |
| else: |
| self._branch_repo_token = self.token |
| self._llvm_project_dir = llvm_project_dir |
| self._phab_token = phab_token |
| |
| @property |
| def token(self) -> str: |
| return self._token |
| |
| @property |
| def repo_name(self) -> str: |
| return self._repo_name |
| |
| @property |
| def issue_number(self) -> int: |
| return self._issue_number |
| |
| @property |
| def branch_repo_name(self) -> str: |
| return self._branch_repo_name |
| |
| @property |
| def branch_repo_token(self) -> str: |
| return self._branch_repo_token |
| |
| @property |
| def llvm_project_dir(self) -> str: |
| return self._llvm_project_dir |
| |
| @property |
| def phab_token(self) -> str: |
| return self._phab_token |
| |
| @property |
| def repo(self) -> github.Repository.Repository: |
| return github.Github(self.token).get_repo(self.repo_name) |
| |
| @property |
| def issue(self) -> github.Issue.Issue: |
| return self.repo.get_issue(self.issue_number) |
| |
| @property |
| def push_url(self) -> str: |
| return "https://{}@github.com/{}".format( |
| self.branch_repo_token, self.branch_repo_name |
| ) |
| |
| @property |
| def branch_name(self) -> str: |
| return "issue{}".format(self.issue_number) |
| |
| @property |
| def release_branch_for_issue(self) -> Optional[str]: |
| issue = self.issue |
| milestone = issue.milestone |
| if milestone is None: |
| return None |
| m = re.search("branch: (.+)", milestone.description) |
| if m: |
| return m.group(1) |
| return None |
| |
| def print_release_branch(self) -> None: |
| print(self.release_branch_for_issue) |
| |
| def issue_notify_branch(self) -> None: |
| self.issue.create_comment( |
| "/branch {}/{}".format(self.branch_repo_name, self.branch_name) |
| ) |
| |
| def issue_notify_pull_request(self, pull: github.PullRequest.PullRequest) -> None: |
| self.issue.create_comment( |
| "/pull-request {}#{}".format(self.branch_repo_name, pull.number) |
| ) |
| |
| def make_ignore_comment(self, comment: str) -> str: |
| """ |
| Returns the comment string with a prefix that will cause |
| a Github workflow to skip parsing this comment. |
| |
| :param str comment: The comment to ignore |
| """ |
| return "<!--IGNORE-->\n" + comment |
| |
| def issue_notify_no_milestone(self, comment: List[str]) -> None: |
| message = "{}\n\nError: Command failed due to missing milestone.".format( |
| "".join([">" + line for line in comment]) |
| ) |
| self.issue.create_comment(self.make_ignore_comment(message)) |
| |
| @property |
| def action_url(self) -> str: |
| if os.getenv("CI"): |
| return "https://github.com/{}/actions/runs/{}".format( |
| os.getenv("GITHUB_REPOSITORY"), os.getenv("GITHUB_RUN_ID") |
| ) |
| return "" |
| |
| def issue_notify_cherry_pick_failure( |
| self, commit: str |
| ) -> github.IssueComment.IssueComment: |
| message = self.make_ignore_comment( |
| "Failed to cherry-pick: {}\n\n".format(commit) |
| ) |
| action_url = self.action_url |
| if action_url: |
| message += action_url + "\n\n" |
| message += "Please manually backport the fix and push it to your github fork. Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`" |
| issue = self.issue |
| comment = issue.create_comment(message) |
| issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL) |
| return comment |
| |
| def issue_notify_pull_request_failure( |
| self, branch: str |
| ) -> github.IssueComment.IssueComment: |
| message = "Failed to create pull request for {} ".format(branch) |
| message += self.action_url |
| return self.issue.create_comment(message) |
| |
| def issue_remove_cherry_pick_failed_label(self): |
| if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]: |
| self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL) |
| |
| def pr_request_review(self, pr: github.PullRequest.PullRequest): |
| """ |
| This function will try to find the best reviewers for `commits` and |
| then add a comment requesting review of the backport and assign the |
| pull request to the selected reviewers. |
| |
| The reviewers selected are those users who approved the patch in |
| Phabricator. |
| """ |
| reviewers = [] |
| for commit in pr.get_commits(): |
| approvers = phab_get_commit_approvers(self.phab_token, commit) |
| for a in approvers: |
| login = phab_login_to_github_login(self.phab_token, self.repo, a) |
| if not login: |
| continue |
| reviewers.append(login) |
| if len(reviewers): |
| message = "{} What do you think about merging this PR to the release branch?".format( |
| " ".join(["@" + r for r in reviewers]) |
| ) |
| pr.create_issue_comment(message) |
| pr.add_to_assignees(*reviewers) |
| |
| def create_branch(self, commits: List[str]) -> bool: |
| """ |
| This function attempts to backport `commits` into the branch associated |
| with `self.issue_number`. |
| |
| If this is successful, then the branch is pushed to `self.branch_repo_name`, if not, |
| a comment is added to the issue saying that the cherry-pick failed. |
| |
| :param list commits: List of commits to cherry-pick. |
| |
| """ |
| print("cherry-picking", commits) |
| branch_name = self.branch_name |
| local_repo = Repo(self.llvm_project_dir) |
| local_repo.git.checkout(self.release_branch_for_issue) |
| |
| for c in commits: |
| try: |
| local_repo.git.cherry_pick("-x", c) |
| except Exception as e: |
| self.issue_notify_cherry_pick_failure(c) |
| raise e |
| |
| push_url = self.push_url |
| print("Pushing to {} {}".format(push_url, branch_name)) |
| local_repo.git.push(push_url, "HEAD:{}".format(branch_name), force=True) |
| |
| self.issue_notify_branch() |
| self.issue_remove_cherry_pick_failed_label() |
| return True |
| |
| def check_if_pull_request_exists( |
| self, repo: github.Repository.Repository, head: str |
| ) -> bool: |
| pulls = repo.get_pulls(head=head) |
| return pulls.totalCount != 0 |
| |
| def create_pull_request(self, owner: str, repo_name: str, branch: str) -> bool: |
| """ |
| reate a pull request in `self.branch_repo_name`. The base branch of the |
| pull request will be chosen based on the the milestone attached to |
| the issue represented by `self.issue_number` For example if the milestone |
| is Release 13.0.1, then the base branch will be release/13.x. `branch` |
| will be used as the compare branch. |
| https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch |
| https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch |
| """ |
| repo = github.Github(self.token).get_repo(self.branch_repo_name) |
| issue_ref = "{}#{}".format(self.repo_name, self.issue_number) |
| pull = None |
| release_branch_for_issue = self.release_branch_for_issue |
| if release_branch_for_issue is None: |
| return False |
| head_branch = branch |
| if not repo.fork: |
| # If the target repo is not a fork of llvm-project, we need to copy |
| # the branch into the target repo. GitHub only supports cross-repo pull |
| # requests on forked repos. |
| head_branch = f"{owner}-{branch}" |
| local_repo = Repo(self.llvm_project_dir) |
| push_done = False |
| for _ in range(0, 5): |
| try: |
| local_repo.git.fetch( |
| f"https://github.com/{owner}/{repo_name}", f"{branch}:{branch}" |
| ) |
| local_repo.git.push( |
| self.push_url, f"{branch}:{head_branch}", force=True |
| ) |
| push_done = True |
| break |
| except Exception as e: |
| print(e) |
| time.sleep(30) |
| continue |
| if not push_done: |
| raise Exception("Failed to mirror branch into {}".format(self.push_url)) |
| owner = repo.owner.login |
| |
| head = f"{owner}:{head_branch}" |
| if self.check_if_pull_request_exists(repo, head): |
| print("PR already exists...") |
| return True |
| try: |
| pull = repo.create_pull( |
| title=f"PR for {issue_ref}", |
| body="resolves {}".format(issue_ref), |
| base=release_branch_for_issue, |
| head=head, |
| maintainer_can_modify=False, |
| ) |
| |
| try: |
| if self.phab_token: |
| self.pr_request_review(pull) |
| except Exception as e: |
| print("error: Failed while searching for reviewers", e) |
| |
| except Exception as e: |
| self.issue_notify_pull_request_failure(branch) |
| raise e |
| |
| if pull is None: |
| return False |
| |
| self.issue_notify_pull_request(pull) |
| self.issue_remove_cherry_pick_failed_label() |
| |
| # TODO(tstellar): Do you really want to always return True? |
| return True |
| |
| def execute_command(self) -> bool: |
| """ |
| This function reads lines from STDIN and executes the first command |
| that it finds. The 2 supported commands are: |
| /cherry-pick commit0 <commit1> <commit2> <...> |
| /branch <owner>/<repo>/<branch> |
| """ |
| for line in sys.stdin: |
| line.rstrip() |
| m = re.search(r"/([a-z-]+)\s(.+)", line) |
| if not m: |
| continue |
| command = m.group(1) |
| args = m.group(2) |
| |
| if command == "cherry-pick": |
| arg_list = args.split() |
| commits = list(map(lambda a: extract_commit_hash(a), arg_list)) |
| return self.create_branch(commits) |
| |
| if command == "branch": |
| m = re.match("([^/]+)/([^/]+)/(.+)", args) |
| if m: |
| owner = m.group(1) |
| repo = m.group(2) |
| branch = m.group(3) |
| return self.create_pull_request(owner, repo, branch) |
| |
| print("Do not understand input:") |
| print(sys.stdin.readlines()) |
| return False |
| |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "--token", type=str, required=True, help="GitHub authentiation token" |
| ) |
| parser.add_argument( |
| "--repo", |
| type=str, |
| default=os.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"), |
| help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)", |
| ) |
| subparsers = parser.add_subparsers(dest="command") |
| |
| issue_subscriber_parser = subparsers.add_parser("issue-subscriber") |
| issue_subscriber_parser.add_argument("--label-name", type=str, required=True) |
| issue_subscriber_parser.add_argument("--issue-number", type=int, required=True) |
| |
| release_workflow_parser = subparsers.add_parser("release-workflow") |
| release_workflow_parser.add_argument( |
| "--llvm-project-dir", |
| type=str, |
| default=".", |
| help="directory containing the llvm-project checout", |
| ) |
| release_workflow_parser.add_argument( |
| "--issue-number", type=int, required=True, help="The issue number to update" |
| ) |
| release_workflow_parser.add_argument( |
| "--phab-token", |
| type=str, |
| help="Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/", |
| ) |
| release_workflow_parser.add_argument( |
| "--branch-repo-token", |
| type=str, |
| help="GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.", |
| ) |
| release_workflow_parser.add_argument( |
| "--branch-repo", |
| type=str, |
| default="llvm/llvm-project-release-prs", |
| help="The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)", |
| ) |
| release_workflow_parser.add_argument( |
| "sub_command", |
| type=str, |
| choices=["print-release-branch", "auto"], |
| help="Print to stdout the name of the release branch ISSUE_NUMBER should be backported to", |
| ) |
| |
| llvmbot_git_config_parser = subparsers.add_parser( |
| "setup-llvmbot-git", |
| help="Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot", |
| ) |
| |
| args = parser.parse_args() |
| |
| if args.command == "issue-subscriber": |
| issue_subscriber = IssueSubscriber( |
| args.token, args.repo, args.issue_number, args.label_name |
| ) |
| issue_subscriber.run() |
| elif args.command == "release-workflow": |
| release_workflow = ReleaseWorkflow( |
| args.token, |
| args.repo, |
| args.issue_number, |
| args.branch_repo, |
| args.branch_repo_token, |
| args.llvm_project_dir, |
| args.phab_token, |
| ) |
| if not release_workflow.release_branch_for_issue: |
| release_workflow.issue_notify_no_milestone(sys.stdin.readlines()) |
| sys.exit(1) |
| if args.sub_command == "print-release-branch": |
| release_workflow.print_release_branch() |
| else: |
| if not release_workflow.execute_command(): |
| sys.exit(1) |
| elif args.command == "setup-llvmbot-git": |
| setup_llvmbot_git() |
