[154434] contrib/buildbot-test

larryv at macports.org larryv at macports.org
Sat Oct 29 07:00:03 CEST 2016


Revision: 154434
          https://trac.macports.org/changeset/154434
Author:   larryv at macports.org
Date:     2016-10-29 07:00:03 +0200 (Sat, 29 Oct 2016)
Log Message:
-----------
buildbot: Switch to GitHub

The production setup will now retrieve all source code from GitHub.

- Tweak `port_from_path` to account for loss of the "dports" prefix.

- Add new configuration keys for the GitHub webhook secret and the URLs
  of each Git repository. The URLs must end with ".git", for now.

- Receive webhook payloads by enabling the GitHub change hook on the
  WebStatus status target. The hook requires both basic authentication
  and a secret. The credentials must be stored in plaintext in the
  htpasswd file; the secret, in config.json.

- Now that each MacPorts component exists in its own repository, filter
  changes based on their "repository" properties instead of vetting the
  leading component of changed files' paths. And only allow changes from
  "master" branches.

- Fetch base, mpbb, the website, and the guide using Git. Allow `mpbb
  checkout` to default to Git by dropping `--svn-url`.

- Stop doing link transformations in the web interface: The GitHub
  change hook automatically adds a "revlink" attribute to changes, and
  we'll be using full Trac URLs in commit messages. (We may restore
  transformations in the future to link "#123" to GitHub pull requests.)

Modified Paths:
--------------
    contrib/buildbot-test/config.json.sample
    contrib/buildbot-test/master.cfg

Modified: contrib/buildbot-test/config.json.sample
===================================================================
--- contrib/buildbot-test/config.json.sample	2016-10-29 01:54:38 UTC (rev 154433)
+++ contrib/buildbot-test/config.json.sample	2016-10-29 05:00:03 UTC (rev 154434)
@@ -2,6 +2,7 @@
     "production":   false,
     "privkey":      "/var/keys/macports-privkey.pem",
     "buildboturl":  "http://domain.tld:8010/",
+    "githubsecret": "",
     "slaveprefix":  "/opt/local",
     "toolsprefix":  "/opt/mports",
     "deploy": {

Modified: contrib/buildbot-test/master.cfg
===================================================================
--- contrib/buildbot-test/master.cfg	2016-10-29 01:54:38 UTC (rev 154433)
+++ contrib/buildbot-test/master.cfg	2016-10-29 05:00:03 UTC (rev 154434)
@@ -32,9 +32,8 @@
 def port_from_path(path, sep='/'):
     components = path.split(sep)
     try:
-        if (components[0] == 'dports' and components[1] != '_resources'
-                and components[3] in ('Portfile', 'files')):
-            return components[2]
+        if (components[0] != '_resources' and components[2] in ('Portfile', 'files')):
+            return components[1]
     except IndexError:
         pass
     # Might be better to throw a custom exception here?
@@ -58,9 +57,13 @@
     'workersfile': _path('slaves.json'),
     'htpasswdfile': _path('htpasswd'),
 
-    # Sources
-    'mpbbsvnurl': 'https://svn.macports.org/repository/macports/contrib/mp-buildbot',
-    'svnurl': 'https://svn.macports.org/repository/macports/trunk',
+    # GitHub. Repository URLs must have the ".git" suffix.
+    'githubsecret': '',
+    'baseurl': 'https://github.com/macports/macports-base.git',
+    'mpbburl': 'https://github.com/macports/mpbb.git',
+    'portsurl': 'https://github.com/macports/macports-ports.git',
+    'wwwurl': 'https://github.com/macports/macports-www.git',
+    'guideurl': 'https://github.com/macports/macports-guide.git',
 
     # Tooling
     'slaveprefix': '/opt/local',
@@ -126,49 +129,53 @@
 c['slavePortnum'] = config['slaveport']
 
 
-####### CHANGESOURCES #######
+####### WEB STATUS AND CHANGE HOOKS #######
 
-# the 'change_source' setting tells the buildmaster how it should find out
-# about source code changes.
+# WebStatus [1] runs a web server that serves the web interface. It can
+# also accept HTTP requests at a hook endpoint [2] and translate them
+# into Changes. We're using the GitHub hook [3] to accept webhook
+# payloads [4].
+#
+# [1]: http://docs.buildbot.net/0.8.12/manual/cfg-statustargets.html#webstatus
+# [2]: http://docs.buildbot.net/0.8.12/manual/cfg-statustargets.html#change-hooks
+# [3]: http://docs.buildbot.net/0.8.12/manual/cfg-statustargets.html#github-hook
+# [4]: https://developer.github.com/webhooks
 
-# poller is used for local testing but PBChangeSource (which relies on
-# notifications from a post-commit script) should be used in production
-
 if config['production']:
-    # TODO
-    sourcedata = []
-#    with open(_path('source.json')) as f:
-#        sourcedata = json.load(f)
-#    c['change_source'] = changes.PBChangeSource(user=sourcedata[0], passwd=sourcedata[1], port=sourcedata[2])
+    # The username doesn't correspond to any real user and can be set to
+    # an arbitrary value. The password must be stored in plaintext and
+    # can be added to the password file with "htpasswd -p".
+    change_hook_kwargs = {
+        'change_hook_auth': ['file:' + config['htpasswdfile']],
+        'change_hook_dialects': {
+            'github': {
+                'secret': config['githubsecret'].encode('utf-8'),
+                'strict': True
+                }
+            }
+        }
 else:
-    c['change_source'] = changes.SVNPoller(
-        svnurl='https://svn.macports.org/repository/macports/trunk',
-        #svnbin='/opt/local/bin/svn',
-        pollinterval=300,
-        category='macports',
-        project='ports')
+    # TODO Add alternate change source, probably a GitPoller.
+    change_hook_kwargs = {}
 
+c['status'] = [
+    WebStatus(
+        http_port=config['httpport'],
+        authz=util.Authz(
+            auth=util.HTPasswdAprAuth(config['htpasswdfile']),
+            gracefulShutdown='auth',
+            forceBuild='auth',
+            forceAllBuilds='auth',
+            pingBuilder='auth',
+            stopBuild='auth',
+            stopAllBuilds='auth',
+            cancelPendingBuild='auth'),
+        **change_hook_kwargs)
+    ]
 
+
 ####### SCHEDULERS #######
 
-def change_has_base(change):
-    for f in change.files:
-        if f.startswith('base'):
-            return True
-    return False
-
-def change_has_guide(change):
-    for f in change.files:
-        if f.startswith('doc-new'):
-            return True
-    return False
-
-def change_has_www(change):
-    for f in change.files:
-        if f.startswith('www'):
-            return True
-    return False
-
 base_platforms = [plat for plat in build_platforms if 'legacy' not in plat and '10.6_i386' not in plat]
 port_platforms = [plat for plat in build_platforms if 'linux' not in plat and '10.5_ppc' != plat]
 
@@ -177,17 +184,21 @@
 portbuilder_buildernames = map('ports-{}-builder'.format, port_platforms)
 portbuilder_triggerables = map('ports-{}-trigger'.format, port_platforms)
 
+# The ChangeFilters assume that Git URLs end with ".git".
 c['schedulers'] = [
     schedulers.SingleBranchScheduler(
         name='base',
         treeStableTimer=None,
         change_filter=util.ChangeFilter(
-            filter_fn=change_has_base),
+            repository=config['baseurl'][:-4],
+            branch='master'),
         builderNames=base_buildernames),
     schedulers.SingleBranchScheduler(
         name='ports',
         treeStableTimer=None,
         change_filter=util.ChangeFilter(
+            repository=config['portsurl'][:-4],
+            branch='master',
             # Should actually skip changes to files/ only, but only if
             # we know the last build of the port succeeded.
             filter_fn=lambda change: any(port_from_path(f) for f in change.files)),
@@ -222,7 +233,8 @@
             name='www',
             treeStableTimer=300,
             change_filter=util.ChangeFilter(
-                filter_fn=change_has_www),
+                repository=config['wwwurl'][:-4],
+                branch='master'),
             builderNames=['docs-www']),
         schedulers.ForceScheduler(
             name='www_force',
@@ -235,7 +247,8 @@
             name='guide',
             treeStableTimer=300,
             change_filter=util.ChangeFilter(
-                filter_fn=change_has_guide),
+                repository=config['guideurl'][:-4],
+                branch='master'),
             builderNames=['docs-guide']),
         schedulers.ForceScheduler(
             name='guide_force',
@@ -259,10 +272,11 @@
 base_factory = util.BuildFactory()
 base_factory.workdir = '../build'
 
-#base_factory.addStep(steps.SVN(
-#   repourl=util.Interpolate('https://svn.macports.org/repository/macports/%(src::branch:-trunk)s/base'),
-base_factory.addStep(steps.SVN(
-    repourl='https://svn.macports.org/repository/macports/trunk/base',
+# Set progress=True on Git steps to prevent timeouts on slow fetches.
+base_factory.addStep(steps.Git(
+    repourl=config['baseurl'],
+    progress=True,
+    mode='full',
     method='copy',
     env={'PATH': path_ports}))
 base_factory.addStep(steps.Configure(command=util.WithProperties("""
@@ -297,7 +311,7 @@
         # support forced build properties
         ports = set(self.getProperty('portlist', default='').split())
 
-        # paths should be dports/category/portdir(/...)
+        # paths should be category/portdir(/...)
         ports.update(ifilter(None, imap(port_from_path, self.build.allFiles())))
 
         self.setProperty('fullportlist', ' '.join(ports))
@@ -350,13 +364,12 @@
     portwatcher_factory.useProgress = False
     portwatcher_factory.workdir = '../build'
 
-    # get mp-buildbot; we'll do the checkout of base and dports via these scripts
-    portwatcher_factory.addStep(steps.SVN(
-        repourl=config['mpbbsvnurl'],
+    # get mpbb; we'll do the checkout of base and dports via these scripts
+    portwatcher_factory.addStep(steps.Git(
+        repourl=config['mpbburl'],
+        progress=True,
         env={'PATH': path_ports},
         alwaysUseLatest=True,
-        preferLastChangedRev=True,
-        mode='incremental',
         workdir=os.path.join(portwatcher_factory.workdir, 'mpbb'),
         haltOnFailure=True))
 
@@ -374,7 +387,7 @@
         haltOnFailure=True))
 
     portwatcher_factory.addStep(steps.ShellCommand(
-        command=['./mpbb/mpbb', '--prefix', util.WithProperties(prefix), 'checkout', '--svn-url', config['svnurl']],
+        command=['./mpbb/mpbb', '--prefix', util.WithProperties(prefix), 'checkout', '--ports-url', config['portsurl']],
         timeout=3600,
         name='checkout',
         description=['syncing', 'ports'],
@@ -502,8 +515,9 @@
 if 'www' in config['deploy']:
     docs_www_factory = util.BuildFactory()
     # TODO: incremental mode with cleanup?
-    docs_www_factory.addStep(steps.SVN(
-        repourl='https://svn.macports.org/repository/macports/trunk/www',
+    docs_www_factory.addStep(steps.Git(
+        repourl=config['wwwurl'],
+        progress=True,
         mode='full',
         method='copy',
         workdir='www'))
@@ -520,8 +534,9 @@
 if 'guide' in config['deploy']:
     docs_guide_factory = util.BuildFactory()
     # TODO: incremental mode with cleanup?
-    docs_guide_factory.addStep(steps.SVN(
-        repourl='https://svn.macports.org/repository/macports/trunk/doc-new',
+    docs_guide_factory.addStep(steps.Git(
+        repourl=config['guideurl'],
+        progress=True,
         mode='full',
         method='copy',
         workdir='guide'))
@@ -608,26 +623,8 @@
             env=merge_dicts(env_buildinfo, {'PATH': path_ports})))
 
 
-####### STATUS TARGETS #######
+####### MAIL NOTIFIERS #######
 
-# 'status' is a list of Status Targets. The results of each build will be
-# pushed to these targets. buildbot/status/*.py has a variety to choose from,
-# including web pages, email senders, and IRC bots.
-
-c['status'] = []
-
-htauth = util.HTPasswdAprAuth(config['htpasswdfile'])
-
-authz_cfg = util.Authz(
-    auth=htauth,
-    gracefulShutdown='auth',
-    forceBuild='auth',
-    forceAllBuilds='auth',
-    pingBuilder='auth',
-    stopBuild='auth',
-    stopAllBuilds='auth',
-    cancelPendingBuild='auth')
-
 # TODO: This is the old mail notifier;
 # - useful functionality could be copied
 # - then the code should be removed
@@ -805,15 +802,8 @@
 c['title'] = config['title']
 c['titleURL'] = config['titleurl']
 c['buildbotURL'] = config['buildboturl']
-c['status'].append(WebStatus(
-    http_port=config['httpport'],
-    authz=authz_cfg,
-    changecommentlink=(r'#(\d+)', r'https://trac.macports.org/ticket/\1', r'Ticket \g<0>')))
 
-c['revlink'] = util.RevlinkMatch([r'https://svn.macports.org/repository/macports/(.*)'],
-                                  r'https://trac.macports.org/changeset/%s')
 
-
 ####### DATABASE #######
 
 # This specifies what database buildbot uses to store its state. You can
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macports.org/pipermail/macports-changes/attachments/20161029/b4623524/attachment-0002.html>


More information about the macports-changes mailing list