[146776] contrib
mojca at macports.org
mojca at macports.org
Thu Mar 17 02:43:06 PDT 2016
Revision: 146776
https://trac.macports.org/changeset/146776
Author: mojca at macports.org
Date: 2016-03-17 02:43:06 -0700 (Thu, 17 Mar 2016)
Log Message:
-----------
buildbot improvements
Modified Paths:
--------------
contrib/buildbot-test/master.cfg
Added Paths:
-----------
contrib/buildbot-test/
contrib/buildbot-test/config.json.sample
contrib/buildbot-test/slaves.json.sample
Added: contrib/buildbot-test/config.json.sample
===================================================================
--- contrib/buildbot-test/config.json.sample (rev 0)
+++ contrib/buildbot-test/config.json.sample 2016-03-17 09:43:06 UTC (rev 146776)
@@ -0,0 +1,9 @@
+{
+ "production": 0,
+ "privkey": "/var/keys/macports-privkey.pem",
+ "slaveport": 9989,
+ "httpport": 8010,
+ "buildboturl": "http://localhost:8010/",
+ "htpasswdfile": "htpasswd",
+ "mpbbsvnurl": "https://svn.macports.org/repository/macports/contrib/mp-buildbot"
+}
Modified: contrib/buildbot-test/master.cfg
===================================================================
--- contrib/buildbot/master.cfg 2016-03-17 04:17:40 UTC (rev 146766)
+++ contrib/buildbot-test/master.cfg 2016-03-17 09:43:06 UTC (rev 146776)
@@ -1,5 +1,6 @@
# -*- python -*-
# ex: set syntax=python:
+# vim:ft=python
import json
import os
@@ -21,18 +22,27 @@
production = False
privkey = ''
+slaveport = 9989
+httpport = 8010
+buildboturl = "http://localhost:8010/"
+htpasswdfile = "htpasswd"
+mpbbsvnurl = "https://svn.macports.org/repository/macports/contrib/mp-buildbot"
+
if os.path.exists(_path('config.json')):
with open(_path('config.json')) as f:
configdata = json.load(f)
- production = configdata['production']
- privkey = configdata['privkey']
+ locals = locals()
+ for key in ["production", "privkey", "slaveport", "httpport", "buildboturl", "htpasswdfile", "mpbbsvnurl"]:
+ if configdata[key]:
+ locals[key] = configdata[key]
# platforms we are building for
if production:
# this list should get longer in future
- build_platforms = ["snowleopard-x86_64", "lion-x86_64", "mtln-x86_64"]
+ #build_platforms = ["snowleopard-x86_64", "lion-x86_64", "mtln-x86_64", "mavericks-x86_64", "yosemite-x86_64", "elcapitan-x86_64", "oelinux-6.3-x86_64"]
+ build_platforms = []
else:
- build_platforms = ["snowleopard-x86_64"]
+ build_platforms = ["10.6-x86_64","10.9-x86_64"]
# Allow spaces and tabs in property values
import re
@@ -49,19 +59,17 @@
c['slaves'] = []
slavedata = {}
-if production:
- with open(_path('slaves.json')) as f:
- slavedata = json.load(f)
- for slave, pwd in slavedata.items():
- c['slaves'].append(BuildSlave(slave, pwd))
-else:
- c['slaves'] = [BuildSlave("snowleopard-x86_64", "pass")]
+with open(_path('slaves.json')) as f:
+ slavedata = json.load(f)
+for slave, pwd in slavedata.items():
+ c['slaves'].append(BuildSlave(slave, pwd))
+
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
-c['slavePortnum'] = 17000
+c['slavePortnum'] = slaveport
####### CHANGESOURCES
@@ -74,16 +82,20 @@
if production:
from buildbot.changes.pb import PBChangeSource
sourcedata = []
- with open(_path('source.json')) as f:
- sourcedata = json.load(f)
- c['change_source'] = PBChangeSource(user=sourcedata[0], passwd=sourcedata[1], port=sourcedata[2])
+# with open(_path('source.json')) as f:
+# sourcedata = json.load(f)
+# c['change_source'] = PBChangeSource(user=sourcedata[0], passwd=sourcedata[1], port=sourcedata[2])
else:
from buildbot.changes.svnpoller import SVNPoller
- c['change_source'] = [SVNPoller(
- 'https://svn.macports.org/repository/macports/trunk',
- svnbin='/opt/local/bin/svn',
- pollinterval=300),
- ]
+ c['change_source'] = (SVNPoller(
+ svnurl='https://svn.macports.org/repository/macports/trunk',
+ #'https://svn.macports.org/repository/macports/trunk',
+ #svnbin='/opt/local/bin/svn',
+ pollinterval=300,
+ category='macports',
+ project='ports',
+ ),
+ )
####### SCHEDULERS
@@ -107,11 +119,15 @@
portsfilter = ChangeFilter(filter_fn=change_has_ports)
basefilter = ChangeFilter(filter_fn=change_has_base)
-base_buildernames = ["buildbase-"+plat for plat in build_platforms]
-ports_buildernames = ["buildports-"+plat for plat in build_platforms if 'linux' not in plat]
+base_buildernames = ["base-"+plat for plat in build_platforms]
+portwatcher_buildernames = ["portwatcher-"+plat for plat in build_platforms if 'linux' not in plat]
+portbuilder_buildernames = ["portbuilder-"+plat for plat in build_platforms if 'linux' not in plat]
+portbuilder_triggerables = ["triggerable-"+plat for plat in portbuilder_buildernames]
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.schedulers.triggerable import Triggerable
+
c['schedulers'] = [SingleBranchScheduler(
name="base",
treeStableTimer=None,
@@ -121,148 +137,186 @@
name="ports",
treeStableTimer=None,
change_filter = portsfilter,
- builderNames=ports_buildernames),
+ builderNames=portwatcher_buildernames),
ForceScheduler(
name="force",
- builderNames=ports_buildernames+base_buildernames)
+ builderNames=portwatcher_buildernames+portbuilder_buildernames+base_buildernames)
]
+for plat in portbuilder_buildernames:
+ c['schedulers'].append(Triggerable( name="triggerable-"+plat, builderNames=[plat] ))
+
####### BUILDERS
-#c['mergeRequests'] = True
+# WARNING: mergeRequests has to be False or Triggerable builds will not be scheduled correctly!
+c['mergeRequests'] = False
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
from buildbot.process.factory import BuildFactory, GNUAutoconf
-from buildbot.process.properties import WithProperties
-from buildbot.steps.source import SVN
-from buildbot.steps.shell import ShellCommand, Compile, Configure
+from buildbot.process.properties import WithProperties, Interpolate
+from buildbot.steps.source.svn import SVN
+from buildbot.steps.shell import ShellCommand, Compile, Configure, SetPropertyFromCommand
base_factory = BuildFactory()
-base_factory.addStep(SVN(baseURL='https://svn.macports.org/repository/macports/%%BRANCH%%/base',
- mode="copy", defaultBranch="trunk"))
-maybe_sudo="""
-if [[ `id -u` -eq 0 ]]; then
- [[ -n "$SUDO_USER" ]] || SUDO_USER=buildbot
- CMD_PREFIX="sudo -u $SUDO_USER"
- INSTALL_USER=$SUDO_USER
-else
- INSTALL_USER=$USER
-fi
-"""
-base_factory.addStep(Configure(command=WithProperties(maybe_sudo+"""
-$CMD_PREFIX env PATH=/usr/bin:/bin:/usr/sbin:/sbin ./configure --enable-readline \
+base_factory.addStep(SVN(repourl='https://svn.macports.org/repository/macports/trunk/base',
+#base_factory.addStep(SVN(repourl=Interpolate('https://svn.macports.org/repository/macports/%(src::branch:-trunk)s/base'),
+ method="copy"))
+base_factory.addStep(Configure(command=WithProperties("""
+env PATH=/usr/bin:/bin:/usr/sbin:/sbin ./configure --enable-readline \
--prefix=%(workdir)s/opt/local \
--with-applications-dir=%(workdir)s/opt/local/Applications \
- --with-install-user=$INSTALL_USER --with-install-group=`id -gn $INSTALL_USER`
-""")))
-base_factory.addStep(Compile(command=maybe_sudo+"""
-$CMD_PREFIX make -j`sysctl -n hw.activecpu`
-"""))
-base_factory.addStep(ShellCommand(command=maybe_sudo+"""
-$CMD_PREFIX make install
-""", name="install", description="install"))
-base_factory.addStep(ShellCommand(command=maybe_sudo+"""
-$CMD_PREFIX make test
-""", name="test", description="test"))
+ --with-install-user=`id -un` \
+ --with-install-group=`id -gn` \
+"""),logfiles={"config.log": "config.log"}))
+base_factory.addStep(Compile(command="make -j`sysctl -n hw.activecpu`"))
+base_factory.addStep(ShellCommand(command="make install", name="install", description="install"))
+base_factory.addStep(ShellCommand(command="make test", name="test", description="test"))
base_factory.addStep(ShellCommand(command=WithProperties("make distclean; rm -rf %(workdir)s/opt/local"),
name="clean", description="clean"))
-
# custom class to make the file list available on the slave...
-class ShellCommandWithPortList(ShellCommand):
- name = 'write portlist file'
- description = 'write portlist file'
+class SetPropertyFromCommandWithPortlist(SetPropertyFromCommand):
+ name = 'generate portlist'
+ description = 'generate portlist'
def setBuild(self, build):
- ShellCommand.setBuild(self, build)
+ SetPropertyFromCommand.setBuild(self, build)
+ portset = set()
# support forced build properties
- try:
- portlist = self.getProperty('portlist').strip()
- except:
- portlist = ''
+ if self.getProperty('portlist'):
+ for v in self.getProperty('portlist').strip().split():
+ portset.add(v)
- portset = set()
# paths should be dports/category/portdir(/...)
for f in self.build.allFiles():
comps = f.split('/')
if len(comps) >= 3 and comps[0] == 'dports' and comps[1] != '_resources':
portset.add(comps[2])
- portlist += ' ' + ' '.join(portset)
- self.setProperty('portlist', portlist)
+ self.setProperty('portlist', ' '.join(portset))
+
# can't run with PREFIX/SRC_PREFIX inside the workdir in production,
# because archives must be built with prefix=/opt/local
if production:
prefix='/opt/local'
src_prefix='/opt/mports'
- dlhost='packages at packages.macports.org'
+ dlhost='packages at packages-origin.macports.org'
dlpath='/var/www/html/packages'
else:
- prefix='%(workdir)s/opt/local'
- src_prefix='%(workdir)s/opt/mports'
+ prefix='/opt/local'
+ src_prefix='/opt/dports'
+ tools_prefix='/opt/mports'
+
+ #prefix='%(workdir)s/opt/local'
+ #src_prefix='%(workdir)s/opt/mports'
+
dlpath='./deployed_archives'
dlhost=''
ulpath='archive_staging'
ulpath_unique=ulpath+'-%(buildername)s'
+from buildbot.steps.transfer import DirectoryUpload
+from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.trigger import Trigger
-ports_factory = BuildFactory()
-# get MPAB itself; we'll do the checkout of base and dports via MPAB's script
-ports_factory.addStep(SVN(svnurl='https://svn.macports.org/repository/macports/contrib/mpab',
- mode="update"))
-ports_factory.addStep(ShellCommand(command=["./mpsync.sh"],
- name="sync",
- description="sync",
- env={'PREFIX': WithProperties(prefix),
- 'SRC_PREFIX': WithProperties(src_prefix),
- 'BASE_UPDATE': 'selfupdate'}))
-ports_factory.addStep(ShellCommandWithPortList(command=WithProperties('rm -f portlist; for portname in %(portlist)s; do for subport in `./subports.tcl $portname`; do echo $subport >> portlist; done; done')))
-# run MPAB on the port list
-ports_factory.addStep(Compile(command=["./mpab", "buildports", "portlist"],
- env={'PREFIX': WithProperties(prefix),
- 'SRC_PREFIX': WithProperties(src_prefix)},
- logfiles={"progress": "progress.log"},
- haltOnFailure=False))
+class TriggerWithPortlist(Trigger):
+ def setTriggerTuple(self,t):
+ self.triggertuple = t
-ports_factory.addStep(ShellCommand(command=["./gather_archives.sh"],
- name="gather archives",
- description="gather distributable archives",
- env={'PREFIX': WithProperties(prefix),
- 'ULPATH': ulpath}))
+ def getSchedulersAndProperties(self):
+ sp = []
+ for scheduler in self.schedulerNames:
+ for port in self.build.getProperty("subportlist").split():
+ sp.append([scheduler,{"portlist": port}])
+ return sp
+
+
+# -- Port Watcher --
+
+def make_portwatcher_factory(triggerable):
+ portwatcher_factory = BuildFactory()
+ # get mp-buildbot; we'll do the checkout of base and dports via these scripts
+ portbuilder_factory.addStep(SVN(repourl=mpbbsvnurl,alwaysUseLatest=True,preferLastChangedRev=True,mode="incremental",workdir="build/mpbb",haltOnFailure=True))
+
+ portwatcher_factory.addStep(SetPropertyFromCommandWithPortlist(command=WithProperties('for portname in %(portlist)s; do for subport in `./mpbb/tools/subports.tcl $portname`; do echo $subport; done; done' ),
+ property="subportlist"))
+
+ portwatcher_factory.addStep(TriggerWithPortlist(schedulerNames=[triggerable],
+ waitForFinish=True,
+ updateSourceStamp=True))
+
+ # make a logfile summarising the success/failure status for each port
+ #portwatcher_factory.addStep(ShellCommand(command=["./do_status.sh"],
+ # name="status",
+ # description="status",
+ # env={'PREFIX': WithProperties(prefix)
+ # },
+ # logfiles={"portstatus": "portstatus.log"}))
+
+ return portwatcher_factory
+
+from buildbot.process.buildstep import BuildStep
+
+class InfoStep(BuildStep):
+ def __init__(self, title='', **kwargs):
+ BuildStep.__init__(self,**kwargs)
+ self.name = title
+ self.description = title
+ self.descriptionDone = title
+ self.doStepIf = False
+
+# -- Port Builder --
+
+portbuilder_factory = BuildFactory()
+
+# XXX: use InfoStep instead of running a dummy command
+portbuilder_factory.addStep(ShellCommand(command=["/usr/bin/true"],descriptionDone=WithProperties("Port %(portlist)s")))
+#portbuilder_factory.addStep(InfoStep(title=WithProperties("Port %(portlist)s")))
+
+portbuilder_factory.addStep(SVN(repourl=mpbbsvnurl,alwaysUseLatest=True,preferLastChangedRev=True,mode="incremental",workdir="build/mpbb",haltOnFailure=True))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'selfupdate', '--prefix', WithProperties(prefix)],haltOnFailure=True,
+ name="selfupdate", description="selfupdate"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'checkout', '--prefix', WithProperties(prefix), '--svn-url', 'http://linux.mpm16/svn-test/trunk'],
+ timeout=3600,haltOnFailure=True,
+ name="checkout", description="checkout"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'install-dependencies', '--prefix', WithProperties(prefix), '--port', WithProperties('%(portlist)s')],haltOnFailure=True,
+ name="install-dependencies", description="install-dependencies"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'install-port', '--prefix', WithProperties(prefix), '--port', WithProperties('%(portlist)s')],haltOnFailure=True,
+ name="install-port", description="install-port"))
+portbuilder_factory.addStep(ShellCommand(command=['./mpbb/mpbb', 'gather-archives', '--prefix', WithProperties(prefix), '--port', WithProperties('%(portlist)s'), '--archive-site', 'http://linux.mpm16/packages', '--staging-dir', ulpath],haltOnFailure=True,
+ name="gather-archives", description="gather-archives"))
+
# upload archives from build slave to master
-from buildbot.steps.transfer import DirectoryUpload
-ports_factory.addStep(DirectoryUpload(slavesrc=ulpath, masterdest=WithProperties(ulpath_unique)))
+portbuilder_factory.addStep(DirectoryUpload(slavesrc=ulpath, masterdest=WithProperties(ulpath_unique)))
+
+# XXX: move deploy_archives.sh functionality to mp-buildbot
# sign generated binaries and sync to download server (if distributable)
-from buildbot.steps.master import MasterShellCommand
-ports_factory.addStep(MasterShellCommand(command=["./deploy_archives.sh", WithProperties(ulpath_unique)],
- name="deploy archives",
- description="deploy archives",
- env={'PRIVKEY': privkey,
- 'DLHOST': dlhost,
- 'DLPATH': dlpath}))
+#if production:
+ #portbuilder_factory.addStep(MasterShellCommand(command=["./deploy_archives.sh", WithProperties(ulpath_unique)],
+ # name="deploy archives",
+ # description="deploy archives",
+ # env={'PRIVKEY': privkey,
+ # 'DLHOST': dlhost,
+ # 'DLPATH': dlpath}))
-# make a logfile summarising the success/failure status for each port
-ports_factory.addStep(ShellCommand(command=["./do_status.sh"],
- name="status",
- description="status",
- env={'PREFIX': WithProperties(prefix)
- },
- logfiles={"portstatus": "portstatus.log"}))
# TODO: do we want to upload the individual logs so maintainers can review them?
-ports_factory.addStep(ShellCommand(command="./cleanup_old.sh",
- name="cleanup",
- description="cleanup",
- env={'PREFIX': WithProperties(prefix),
- 'ULPATH': ulpath}))
+#ports_factory.addStep(ShellCommand(command="./cleanup_old.sh",
+# name="cleanup",
+# description="cleanup",
+# env={'PREFIX': WithProperties(prefix),
+# 'ULPATH': ulpath}))
+# === BUILDER CONFIGURATION ===
+
from buildbot.config import BuilderConfig
+# XXX: slavenames assignment should be automatic and more generic
portsslaves = {}
baseslaves = {}
if production:
@@ -270,20 +324,27 @@
for plat in build_platforms:
baseslaves[plat] = filter(lambda x: x.endswith(plat+"-base"), slavenames)
portsslaves[plat] = filter(lambda x: x.endswith(plat+"-ports"), slavenames)
-else:
- slavenames = ["snowleopard-x86_64"]
- portsslaves = {build_platforms[0]:slavenames[0]}
- baseslaves = {build_platforms[0]:slavenames[0]}
+#else:
+ #slavenames = ["snowleopard-x86_64"]
+ #slavenames = ["base-10.6-x86_64","10.6-x86_64","10.9-x86_64"]
+ #portsslaves = {build_platforms[0]:slavenames[0],build_platforms[1]:slavenames[1]}
+ #baseslaves = {build_platforms[0]:slavenames[0],build_platforms[1]:slavenames[1]}
c['builders']=[]
for plat in build_platforms:
- c['builders']+= [BuilderConfig(name="buildbase-"+plat,
- slavenames=baseslaves[plat],
- factory=base_factory)]
- if 'linux' not in plat:
- c['builders']+= [BuilderConfig(name="buildports-"+plat,
- slavenames=portsslaves[plat],
- factory=ports_factory)]
+ c['builders']+= [BuilderConfig(name="base-"+plat,
+ slavenames=["base-"+plat],
+ factory=base_factory,
+ env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin"})]
+ if 'linux' not in plat:
+ c['builders']+= [BuilderConfig(name="portwatcher-"+plat,
+ slavenames=[plat],
+ factory=make_portwatcher_factory("triggerable-portbuilder-"+plat),
+ env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin"})]
+ c['builders']+= [BuilderConfig(name="portbuilder-"+plat,
+ slavenames=[plat],
+ factory=portbuilder_factory,
+ env={"PATH": tools_prefix+"/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin"})]
####### STATUS TARGETS
@@ -295,30 +356,24 @@
from buildbot.status import html
from buildbot.status.web import auth, authz
-if production:
- from buildbot.status.web.auth import HTPasswdAuth
- htauth = HTPasswdAuth('htpasswd')
- authz_cfg=authz.Authz(
- auth=htauth,
- gracefulShutdown = 'auth',
- forceBuild = 'auth',
- forceAllBuilds = 'auth',
- pingBuilder = 'auth',
- stopBuild = 'auth',
- stopAllBuilds = 'auth',
- cancelPendingBuild = 'auth',
- )
-else:
- authz_cfg=authz.Authz(
- gracefulShutdown = True,
- forceBuild = True,
- forceAllBuilds = True,
- pingBuilder = True,
- stopBuild = True,
- stopAllBuilds = True,
- cancelPendingBuild = True
- )
+from buildbot.status.web.auth import HTPasswdAuth
+
+# add new users with crypt, not md5/sha1:
+# htpasswd -d -n $USER
+htauth = HTPasswdAuth(htpasswdfile)
+
+authz_cfg=authz.Authz(
+ auth=htauth,
+ gracefulShutdown = 'auth',
+ forceBuild = 'auth',
+ forceAllBuilds = 'auth',
+ pingBuilder = 'auth',
+ stopBuild = 'auth',
+ stopAllBuilds = 'auth',
+ cancelPendingBuild = 'auth',
+)
+
if production:
# send mail about base failures to users on the blamelist
from buildbot.status.mail import MailNotifier
@@ -337,18 +392,18 @@
failedPorts = set()
interestedUsers = set()
+ # XXX: needs to be rewritten for the new steps of mp-buildbot
statusStep = [x for x in build.getSteps() if x.getName() == "status"][0]
statusLog = [x for x in statusStep.getLogs() if x.getName() == "portstatus"][0]
for line in statusLog.getText().splitlines():
halves = line.split()
if halves[0] == "[FAIL]":
failedPorts.add(halves[1])
-
- fakeAddresses = {'nomaintainer', 'nomaintainer at macports.org', 'openmaintainer', 'openmaintainer at macports.org'}
+
for p in failedPorts:
output = subprocess.Popen(['/opt/local/bin/port', 'info', '--index', '--maintainers', '--line', p], stdout=subprocess.PIPE).communicate()[0].strip()
for m in output.split(','):
- if m not in fakeAddresses:
+ if m != "nomaintainer at macports.org" and m != "openmaintainer at macports.org":
interestedUsers.add(m)
ss = build.getSourceStamp()
@@ -382,20 +437,24 @@
c['title'] = "MacPorts"
c['titleURL'] = "http://www.macports.org/"
-if production:
- c['buildbotURL'] = "http://build.macports.org/"
- c['status'].append(html.WebStatus(http_port=8710, authz=authz_cfg))
-else:
- c['buildbotURL'] = "http://localhost:8010/"
- c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+c['buildbotURL'] = buildboturl
+c['status'].append(html.WebStatus(http_port=httpport, authz=authz_cfg))
+
####### DB URL
# This specifies what database buildbot uses to store change and scheduler
# state. You can leave this at its default for all but the largest
# installations.
-c['db_url'] = "sqlite:///state.sqlite"
+#c['db_url'] = "sqlite:///state.sqlite"
+c['db'] = {
+ # This specifies what database buildbot uses to store its state. You can leave
+ # this at its default for all but the largest installations.
+ 'db_url' : "sqlite:///state.sqlite",
+}
+
+
######## Cache settings
c['buildHorizon'] = 10000
c['logHorizon'] = 5000
Added: contrib/buildbot-test/slaves.json.sample
===================================================================
--- contrib/buildbot-test/slaves.json.sample (rev 0)
+++ contrib/buildbot-test/slaves.json.sample 2016-03-17 09:43:06 UTC (rev 146776)
@@ -0,0 +1,4 @@
+{
+ "base-10.6-x86_64": "***",
+ "10.6-x86_64": "***",
+}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/macports-changes/attachments/20160317/93c434a4/attachment-0001.html>
More information about the macports-changes
mailing list