Ported LLVMPoller.
diff --git a/zorg/buildbot/changes/llvmgitpoller.py b/zorg/buildbot/changes/llvmgitpoller.py
index cc1cf42..012d0ef 100644
--- a/zorg/buildbot/changes/llvmgitpoller.py
+++ b/zorg/buildbot/changes/llvmgitpoller.py
@@ -1,284 +1,41 @@
# LLVM buildbot needs to watch multiple projects within a single repository.
-
-# Based on the buildbot.changes.gitpoller.GitPoller source code.
-# For buildbot v0.8.5
-
-import time
-import tempfile
-import os
+# TODO: Handle both author and committer for a commit to build a correct blame list later.
import re
-import itertools
from twisted.python import log
-from twisted.internet import defer, utils
+from datetime import datetime
-from buildbot.util import deferredLocked
-from buildbot.changes import base
-from buildbot.util import epoch2datetime
+from twisted.internet import defer
+from buildbot.util import bytes2unicode
+from buildbot.plugins import changes
-class LLVMPoller(base.PollingChangeSource):
+class LLVMPoller(changes.GitPoller):
"""
Poll LLVM repository for changes and submit them to the change master.
- Following Multiple Projects.
+ Following Multiple LLVM Projects.
This source will poll a remote LLVM git _monorepo_ for changes and submit
them to the change master."""
_repourl = "https://github.com/llvm/llvm-project"
_branch = "master"
- _categories = {
- # Project: Category:
- 'llvm' : 'llvm',
- 'cfe' : 'clang',
- 'polly' : 'polly',
- 'compiler-rt' : 'compiler-rt',
- 'flang' : 'flang',
- 'libc' : 'libc',
- 'libcxx' : 'libcxx',
- 'libcxxabi' : 'libcxxabi',
- 'libunwind' : 'libunwind',
- 'lld' : 'lld',
- 'lldb' : 'lldb',
- 'mlir' : 'mlir',
- 'llgo' : 'llgo',
- 'openmp' : 'openmp',
- }
compare_attrs = ["repourl", "branch", "workdir",
"pollInterval", "gitbin", "usetimestamps",
"category", "project",
"projects"]
- projects = None # Projects and branches to watch.
-
- def __init__(self, repourl=_repourl, branch=_branch,
- workdir=None, pollInterval=10*60,
- gitbin='git', usetimestamps=True,
- category=None, project=None,
- pollinterval=-2, fetch_refspec=None,
- encoding='utf-8', projects=None):
+ def __init__(self,
+ repourl=_repourl, branch=_branch,
+ **kwargs):
self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build", re.IGNORECASE + re.MULTILINE)
self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)")
- # projects is a list of projects to watch or None to watch all.
- if projects:
- if isinstance(projects, str) or isinstance(projects, tuple):
- projects = [projects]
- assert isinstance(projects, list)
- assert len(projects) > 0
+ # TODO: Add support for an optional list of projects.
+ # For now we always watch all the projects.
- # Each project to watch is a string (project name) or a tuple
- # (project name, branch) like ('llvm', 'branches/release_30').
- # But we want it always to be a tuple, so we convert a project
- # name string to a tuple (project, 'master').
- self.projects = set()
- for project in projects:
- if isinstance(project, str):
- project = (project, branch)
-
- assert isinstance(project, tuple)
- self.projects.add(project)
-
- # for backward compatibility; the parameter used to be spelled with 'i'
- if pollinterval != -2:
- pollInterval = pollinterval
- if project is None: project = ''
-
- self.repourl = repourl
- self.branch = branch
- self.pollInterval = pollInterval
- self.fetch_refspec = fetch_refspec
- self.encoding = encoding
- self.lastChange = time.time()
- self.lastPoll = time.time()
- self.gitbin = gitbin
- self.workdir = workdir
- self.usetimestamps = usetimestamps
- self.category = category
- self.project = project
- self.changeCount = 0
- self.commitInfo = {}
- self.initLock = defer.DeferredLock()
-
- if self.workdir == None:
- self.workdir = tempfile.gettempdir() + '/gitpoller_work'
- log.msg("WARNING: LLVMGitPoller using deprecated temporary workdir " +
- "'%s'; consider setting workdir=" % self.workdir)
-
- def startService(self):
- # make our workdir absolute, relative to the master's basedir
- if not os.path.isabs(self.workdir):
- self.workdir = os.path.join(self.master.basedir, self.workdir)
- log.msg("LLVMGitPoller: using workdir '%s'" % self.workdir)
-
- # initialize the repository we'll use to get changes; note that
- # startService is not an event-driven method, so this method will
- # instead acquire self.initLock immediately when it is called.
- if not os.path.exists(self.workdir + r'/.git'):
- d = self.initRepository()
- d.addErrback(log.err, 'while initializing LLVMGitPoller repository')
- else:
- log.msg("LLVMGitPoller repository already exists")
-
- # call this *after* initRepository, so that the initLock is locked first
- base.PollingChangeSource.startService(self)
-
- @deferredLocked('initLock')
- def initRepository(self):
- d = defer.succeed(None)
- def make_dir(_):
- dirpath = os.path.dirname(self.workdir.rstrip(os.sep))
- if not os.path.exists(dirpath):
- log.msg('LLVMGitPoller: creating parent directories for workdir')
- os.makedirs(dirpath)
- d.addCallback(make_dir)
-
- def git_init(_):
- log.msg('LLVMGitPoller: initializing working dir from %s' % self.repourl)
- d = utils.getProcessOutputAndValue(self.gitbin,
- ['init', self.workdir], env=dict(PATH=os.environ['PATH']))
- d.addCallback(self._convert_nonzero_to_failure)
- d.addErrback(self._stop_on_failure)
- return d
- d.addCallback(git_init)
-
- def git_remote_add(_):
- d = utils.getProcessOutputAndValue(self.gitbin,
- ['remote', 'add', 'origin', self.repourl],
- path=self.workdir, env=dict(PATH=os.environ['PATH']))
- d.addCallback(self._convert_nonzero_to_failure)
- d.addErrback(self._stop_on_failure)
- return d
- d.addCallback(git_remote_add)
-
- def git_fetch_origin(_):
- args = ['fetch', 'origin']
- self._extend_with_fetch_refspec(args)
- d = utils.getProcessOutputAndValue(self.gitbin, args,
- path=self.workdir, env=dict(PATH=os.environ['PATH']))
- d.addCallback(self._convert_nonzero_to_failure)
- d.addErrback(self._stop_on_failure)
- return d
- d.addCallback(git_fetch_origin)
-
- def set_master(_):
- log.msg('LLVMGitPoller: checking out %s' % self.branch)
- if self.branch == 'master': # repo is already on branch 'master', so reset
- d = utils.getProcessOutputAndValue(self.gitbin,
- ['reset', '--hard', 'origin/%s' % self.branch],
- path=self.workdir, env=dict(PATH=os.environ['PATH']))
- else:
- d = utils.getProcessOutputAndValue(self.gitbin,
- ['checkout', '-b', self.branch, 'origin/%s' % self.branch],
- path=self.workdir, env=dict(PATH=os.environ['PATH']))
- d.addCallback(self._convert_nonzero_to_failure)
- d.addErrback(self._stop_on_failure)
- return d
- d.addCallback(set_master)
- def get_rev(_):
- d = utils.getProcessOutputAndValue(self.gitbin,
- ['rev-parse', self.branch],
- path=self.workdir, env={})
- d.addCallback(self._convert_nonzero_to_failure)
- d.addErrback(self._stop_on_failure)
- d.addCallback(lambda (out, err, code) : out.strip())
- return d
- d.addCallback(get_rev)
- def print_rev(rev):
- log.msg("LLVMGitPoller: finished initializing working dir from %s at rev %s"
- % (self.repourl, rev))
- d.addCallback(print_rev)
- return d
-
- def describe(self):
- status = ""
- if not self.master:
- status = "[STOPPED - check log]"
- str = 'LLVMGitPoller watching the remote git repository %s, branch: %s %s' \
- % (self.repourl, self.branch, status)
- return str
-
- @deferredLocked('initLock')
- def poll(self):
- d = self._get_changes()
- d.addCallback(self._process_changes)
- d.addErrback(self._process_changes_failure)
- d.addCallback(self._catch_up)
- d.addErrback(self._catch_up_failure)
- return d
-
- def _get_commit_comments(self, rev):
- args = ['log', rev, '--no-walk', r'--format=%s%n%b']
- d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
- def process(git_output):
- stripped_output = git_output.strip().decode(self.encoding)
- if len(stripped_output) == 0:
- raise EnvironmentError('could not get commit comment for rev')
- #log.msg("LLVMGitPoller: _get_commit_comments: '%s'" % stripped_output)
- return stripped_output
- d.addCallback(process)
- return d
-
- def _get_commit_timestamp(self, rev):
- # unix timestamp
- args = ['log', rev, '--no-walk', r'--format=%ct']
- d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
- def process(git_output):
- stripped_output = git_output.strip()
- if self.usetimestamps:
- try:
- stamp = float(stripped_output)
- #log.msg("LLVMGitPoller: _get_commit_timestamp: \'%s\'" % stamp)
- except Exception, e:
- log.msg('LLVMGitPoller: caught exception converting output \'%s\' to timestamp' % stripped_output)
- raise e
- return stamp
- else:
- return None
- d.addCallback(process)
- return d
-
- def _get_commit_files(self, rev):
- args = ['log', rev, '--name-only', '--no-walk', r'--format=%n']
- d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
- def process(git_output):
- fileList = git_output.split()
- #log.msg("LLVMGitPoller: _get_commit_files: \'%s\'" % fileList)
- return fileList
- d.addCallback(process)
- return d
-
- def _get_commit_name(self, rev):
- args = ['log', rev, '--no-walk', r'--format=%aN <%aE>']
- d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
- def process(git_output):
- stripped_output = git_output.strip().decode(self.encoding)
- if len(stripped_output) == 0:
- raise EnvironmentError('could not get commit name for rev')
- #log.msg("LLVMGitPoller: _get_commit_name: \'%s\'" % stripped_output)
- return stripped_output
- d.addCallback(process)
- return d
-
- def _get_changes(self):
- log.msg('LLVMGitPoller: polling git repo at %s' % self.repourl)
-
- self.lastPoll = time.time()
-
- # get a deferred object that performs the fetch
- args = ['fetch', 'origin']
- self._extend_with_fetch_refspec(args)
-
- # This command always produces data on stderr, but we actually do not care
- # about the stderr or stdout from this command. We set errortoo=True to
- # avoid an errback from the deferred. The callback which will be added to this
- # deferred will not use the response.
- d = utils.getProcessOutput(self.gitbin, args,
- path=self.workdir,
- env=dict(PATH=os.environ['PATH']), errortoo=True )
-
- return d
+ super().__init__(repourl=repourl, branch=branch, **kwargs)
def _transform_path(self, fileList):
"""
@@ -288,220 +45,139 @@
NOTE: we don't change result path, just extract a project name.
"""
- #log.msg("LLVMGitPoller: _transform_path: got a file list: %s" % fileList)
-
- if fileList is None or len(fileList) == 0:
- return None
+ #log.msg("LLVMPoller: _transform_path: got a file list: %s" % fileList)
result = {}
+ # It is possible that this commit does not change anything.
+ if not fileList:
+ return result
+
# turn libcxxabi/include/__cxxabi_config.h into
# ("libcxxabi", "libcxxabi/include/__cxxabi_config.h")
# and filter projects we are not watching.
for path in fileList:
+ if not path:
+ continue
+
pieces = path.split('/')
- project = pieces.pop(0)
- #NOTE:TODO: a dirty hack for backward compatibility.
- if project == "clang":
- project = "cfe"
+ project = pieces[0] if len(pieces) > 1 else None
- #log.msg("LLVMGitPoller: _transform_path: processing path %s: project: %s" % (path, project))
- if self.projects:
- #NOTE: multibranch is not supported.
- #log.msg("LLVMGitPoller: _transform_path: (%s, %s) in projects: %s" % (project, self.branch, (project, self.branch) in self.projects))
- if (project, self.branch) in self.projects:
- # Collect file path for each detected projects.
- if project in result:
- result[project].append(path)
- else:
- result[project] = [path]
+ #log.msg("LLVMPoller: _transform_path: processing path %s: project: %s" % (path, project))
+ # Collect file path for each detected projects.
+ if project in result:
+ result[project].append(path)
+ else:
+ result[project] = [path]
- #log.msg("LLVMGitPoller: _transform_path: result: %s" % result)
+ log.msg("LLVMPoller: _transform_path: result: %s" % result)
return [(k, result[k]) for k in result]
- @defer.deferredGenerator
- def _process_changes(self, unused_output):
+ @defer.inlineCallbacks
+ def _process_changes(self, newRev, branch):
+ """
+ Read changes since last change.
+
+ - Read list of commit hashes.
+ - Extract details from each commit.
+ - Add changes to database.
+ """
+
+ # initial run, don't parse all history
+ if not self.lastRev:
+ return
+
# get the change list
- revListArgs = ['log', '%s..origin/%s' % (self.branch, self.branch), r'--format=%H']
+ revListArgs = (['--ignore-missing'] +
+ ['--format=%H', '{}'.format(newRev)] +
+ ['^' + rev
+ for rev in sorted(self.lastRev.values())] +
+ ['--'])
self.changeCount = 0
- d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir,
- env=dict(PATH=os.environ['PATH']), errortoo=False )
- wfd = defer.waitForDeferred(d)
- yield wfd
- results = wfd.getResult()
+ results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
- if not revList:
- return
-
revList.reverse()
- self.changeCount = len(revList)
- log.msg('LLVMGitPoller: processing %d changes: %s in "%s"'
- % (self.changeCount, revList, self.workdir) )
+ if self.buildPushesWithNoCommits and not revList:
+ existingRev = self.lastRev.get(branch)
+ if existingRev != newRev:
+ revList = [newRev]
+ if existingRev is None:
+ # This branch was completely unknown, rebuild
+ log.msg('LLVMPoller: rebuilding {} for new branch "{}"'.format(
+ newRev, branch))
+ else:
+ # This branch is known, but it now points to a different
+ # commit than last time we saw it, rebuild.
+ log.msg('LLVMPoller: rebuilding {} for updated branch "{}"'.format(
+ newRev, branch))
+
+ self.changeCount = len(revList)
+ self.lastRev[branch] = newRev
+
+ if self.changeCount:
+ log.msg('LLVMPoller: processing {} changes: {} from "{}" branch "{}"'.format(
+ self.changeCount, revList, self.repourl, branch))
for rev in revList:
- #log.msg('LLVMGitPoller: waiting defer for revision: %s' % rev)
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
- self._get_commit_name(rev),
+ self._get_commit_author(rev),
+ self._get_commit_committer(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
], consumeErrors=True)
- wfd = defer.waitForDeferred(dl)
- yield wfd
- results = wfd.getResult()
- #log.msg('LLVMGitPoller: got defer results: %s' % results)
+ results = yield dl
# check for failures
- failures = [ r[1] for r in results if not r[0] ]
+ failures = [r[1] for r in results if not r[0]]
if failures:
+ for failure in failures:
+ log.err(
+ failure, "while processing changes for {} {}".format(newRev, branch))
# just fail on the first error; they're probably all related!
- raise failures[0]
+ failures[0].raiseException()
- #log.msg('LLVMGitPoller: begin change adding cycle for revision: %s' % rev)
+ log.msg('>>> LLVMPoller: begin change adding cycle for revision: %s' % rev)
- timestamp, name, files, comments = [ r[1] for r in results ]
+ timestamp, author, committer, files, comments = [r[1] for r in results]
+
where = self._transform_path(files)
- #log.msg('LLVMGitPoller: walking over transformed path/projects: %s' % where)
+
+ projects = list()
+ properties = dict()
+
+ #log.msg('LLVMPoller: walking over transformed path/projects: %s' % where)
for wh in where:
where_project, where_project_files = wh
- #log.msg('LLVMGitPoller: processing transformed pair: %s, files:' % where_project, where_project_files)
+ #log.msg('LLVMPoller: processing transformed pair: %s, files:' % where_project, where_project_files)
+ projects += [where_project]
- properties = dict()
if self.cleanRe.search(comments) or \
any([m for f in where_project_files for m in [self.cleanCfg.search(f)] if m]):
- log.msg("LLVMGitPoller: creating a change with the 'clean' property for r%s" % rev)
+ log.msg("LLVMPoller: creating a change with the 'clean_obj' property for r%s" % rev)
properties['clean_obj'] = (True, "change")
- log.msg("LLVMGitPoller: creating a change rev=%s" % rev)
- d = self.master.addChange(
- author=name,
- revision=rev,
- files=where_project_files,
+ log.msg("LLVMPoller: creating a change rev=%s" % rev)
+ log.msg(" >>> revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \
+ (bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer,
+ projects, files, comments, properties))
+
+ yield self.master.data.updates.addChange(
+ author=author,
+ committer=committer if committer != author else None,
+ revision=bytes2unicode(rev, encoding=self.encoding),
+ files=files,
comments=comments,
- when_timestamp=epoch2datetime(timestamp),
- branch=self.branch,
- category=self._categories.get(where_project, self.category),
- project=where_project,
+ when_timestamp=timestamp,
+ branch=bytes2unicode(self._removeHeads(branch)),
+ category=self.category, ## TODO: Figure out if we could support tags here
+ project=",".join(projects),
# Always promote an external github url of the LLVM project with the changes.
repository=self._repourl,
- src='git',
+ src='git', # Must be one of the buildbot.process.users.srcs
properties=properties)
- wfd = defer.waitForDeferred(d)
- yield wfd
- results = wfd.getResult()
-
- def _process_changes_failure(self, f):
- log.msg('LLVMGitPoller: repo poll failed')
- log.err(f)
- # eat the failure to continue along the defered chain - we still want to catch up
- return None
-
- def _catch_up(self, res):
- if self.changeCount == 0:
- log.msg('LLVMGitPoller: no changes, no catch_up')
- return
- log.msg('LLVMGitPoller: catching up tracking branch')
- args = ['reset', '--hard', 'origin/%s' % (self.branch,)]
- d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']))
- d.addCallback(self._convert_nonzero_to_failure)
- return d
-
- def _catch_up_failure(self, f):
- log.err(f)
- log.msg('LLVMGitPoller: please resolve issues in local repo: %s' % self.workdir)
- # this used to stop the service, but this is (a) unfriendly to tests and (b)
- # likely to leave the error message lost in a sea of other log messages
-
- def _convert_nonzero_to_failure(self, res):
- "utility method to handle the result of getProcessOutputAndValue"
- (stdout, stderr, code) = res
- if code != 0:
- raise EnvironmentError('command failed with exit code %d: %s' % (code, stderr))
- return (stdout, stderr, code)
-
- def _stop_on_failure(self, f):
- "utility method to stop the service when a failure occurs"
- if self.running:
- d = defer.maybeDeferred(lambda : self.stopService())
- d.addErrback(log.err, 'while stopping broken GitPoller service')
- return f
-
- def _extend_with_fetch_refspec(self, args):
- if self.fetch_refspec:
- if type(self.fetch_refspec) in (list,set):
- args.extend(self.fetch_refspec)
- else:
- args.append(self.fetch_refspec)
-
-
-# Run: python -m zorg.buildbot.changes.llvmgitpoller
-if __name__ == '__main__':
- print "Testing Git LLVMPoller..."
- poller = LLVMPoller(projects = [
- "llvm",
- "cfe",
- "clang-tests-external",
- "clang-tools-extra",
- "polly",
- "compiler-rt",
- "libcxx",
- "libcxxabi",
- "libunwind",
- "lld",
- "lldb",
- "openmp",
- "lnt",
- "test-suite"
- ],
- workdir = os.getcwd()
- )
-
- # Test _transform_path method.
- fileList = [
- "clang-tools-extra/clang-doc/Generators.cpp",
- "clang-tools-extra/clang-doc/Generators.h",
- "clang-tools-extra/clang-doc/HTMLGenerator.cpp",
- "clang-tools-extra/clang-doc/MDGenerator.cpp",
- "clang-tools-extra/clang-doc/Representation.cpp",
- "clang-tools-extra/clang-doc/Representation.h",
- "clang-tools-extra/clang-doc/YAMLGenerator.cpp",
- "clang-tools-extra/clang-doc/assets/clang-doc-default-stylesheet.css",
- "clang-tools-extra/clang-doc/assets/index.js",
- "clang-tools-extra/clang-doc/stylesheets/clang-doc-default-stylesheet.css",
- "clang-tools-extra/clang-doc/tool/CMakeLists.txt",
- "clang-tools-extra/clang-doc/tool/ClangDocMain.cpp",
- "clang-tools-extra/unittests/clang-doc/CMakeLists.txt",
- "clang-tools-extra/unittests/clang-doc/ClangDocTest.cpp",
- "clang-tools-extra/unittests/clang-doc/ClangDocTest.h",
- "clang-tools-extra/unittests/clang-doc/GeneratorTest.cpp",
- "clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp",
-
- "llvm/docs/BugpointRedesign.md",
- "llvm/test/Reduce/Inputs/remove-funcs.sh",
- "llvm/test/Reduce/remove-funcs.ll",
- "llvm/tools/LLVMBuild.txt",
- "llvm/tools/llvm-reduce/CMakeLists.txt",
- "llvm/tools/llvm-reduce/DeltaManager.h",
- "llvm/tools/llvm-reduce/LLVMBuild.txt",
- "llvm/tools/llvm-reduce/TestRunner.cpp",
- "llvm/tools/llvm-reduce/TestRunner.h",
- "llvm/tools/llvm-reduce/deltas/Delta.h",
- "llvm/tools/llvm-reduce/deltas/RemoveFunctions.cpp",
- "llvm/tools/llvm-reduce/deltas/RemoveFunctions.h",
- "llvm/tools/llvm-reduce/llvm-reduce.cpp",
-
- "openmp/libomptarget/test/mapping/declare_mapper_api.cpp",
-
- "unknown/lib/unknonw.cpp"
- ]
-
- where = poller._transform_path(fileList)
- for wh in where:
- where_project, where_project_files = wh
- print "category: %s" % poller._categories.get(where_project, poller.category)
- print "project: %s, files(%s): %s\n" % (where_project, len(where_project_files), where_project_files)
