utils/ci/phabricator-report

#!/usr/bin/env python3
#===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===----------------------------------------------------------------------===##

import argparse
import io
import os
import phabricator
import re
import socket
import subprocess
import sys
import time

LLVM_REVIEWS_API = "https://reviews.llvm.org/api/"

def exponentialBackoffRetry(f, exception, maxAttempts=3):
    """Tries calling a function, but retry with exponential backoff if the
       function fails with the specified exception.
    """
    waitTime = 1
    attempts = 0
    while True:
        try:
            f()
            break
        except exception as e:
            attempts += 1
            if attempts == maxAttempts:
                raise e
            else:
                time.sleep(waitTime)
                waitTime *= 2

def buildPassed(log):
    """
    Tries to guess whether a build has passed or not based on the logs
    produced by it.

    This is really hacky -- it would be better to use the status of the
    script that runs the tests, however that script is being piped into
    this script, so we can't know its exit status. What we do here is
    basically look for abnormal CMake or Lit output, but that is tightly
    coupled to the specific CI we're running.
    """
    # Lit reporting failures
    matches = re.findall(r"^\s*Failed\s*:\s*(\d+)$", log, flags=re.MULTILINE)
    if matches and any(int(match) > 0 for match in matches):
        return False

    # Error while running CMake
    if 'CMake Error' in log or 'Configuring incomplete, errors occurred!' in log:
        return False

    # Ninja failed to build some target
    if 'FAILED:' in log:
        return False

    return True

def main(argv):
    parser = argparse.ArgumentParser(
        description="""
This script gathers information about a Buildkite build and updates the
Phabricator review associated to the HEAD commit with those results.

The intended usage of this script is to pipe the output of a command defined
in a Buildkite pipeline into it. The script will echo everything to stdout,
like tee, but will also update the Phabricator review associated to HEAD
with the results of the build.

The script is assumed to be running inside a Buildkite agent, and as such,
it assumes the existence of several environment variables that are specific
to Buildkite.

It also assumes that it is running in a context where the HEAD commit contains
the Phabricator ID of the review to update. If the commit does not contain the
Phabricator ID, this script is basically a no-op. This allows running the CI
on commits that are not triggered by a Phabricator review.
""")
    args = parser.parse_args(argv)

    for var in ('BUILDKITE_LABEL', 'BUILDKITE_JOB_ID', 'BUILDKITE_BUILD_URL', 'CONDUIT_TOKEN'):
        if var not in os.environ:
            raise RuntimeError(
                'The {} environment variable must exist -- are you running '
                'this script from a Buildkite agent?'.format(var))

    # First, read all the log input and write it line-by-line to stdout.
    # This is important so that we can follow progress in the Buildkite
    # console. Since we're being piped into in real time, it's also the
    # moment to time the duration of the job.
    start = time.time()
    log = io.StringIO()
    while True:
        line = sys.stdin.readline()
        if line == '':
            break
        sys.stdout.write(line)
        sys.stdout.flush() # flush every line to avoid buffering
        log.write(line)
    end = time.time()

    # Then, extract information from the environment and post-process the logs.
    log.seek(0)
    log = log.read()
    result = 'pass' if buildPassed(log) else 'fail'
    resultObject = {
        'name': '{BUILDKITE_LABEL} ({BUILDKITE_BUILD_URL}#{BUILDKITE_JOB_ID})'.format(**os.environ),
        'result': result,
        'duration': end - start,
        'details': log
    }

    commitMessage = subprocess.check_output(['git', 'log', '--format=%B' , '-n', '1']).decode()
    phabricatorID = re.search(r'^Phabricator-ID:\s+(.+)$', commitMessage, flags=re.MULTILINE)

    # If there's a Phabricator ID in the commit, then the build was triggered
    # by a Phabricator review -- update the results back. Otherwise, don't
    # do anything.
    if phabricatorID:
        phabricatorID = phabricatorID.group(1)
        token = os.environ['CONDUIT_TOKEN']
        phab = phabricator.Phabricator(token=token, host=LLVM_REVIEWS_API)
        exponentialBackoffRetry(
            lambda: phab.harbormaster.sendmessage(buildTargetPHID=phabricatorID, type=result, unit=[resultObject]),
            exception=socket.timeout
        )
    else:
        print('The HEAD commit does not appear to be tied to a Phabricator review -- '
              'not uploading the results to any review.')

if __name__ == '__main__':
    main(sys.argv[1:])