[146768] contrib
cal at macports.org
cal at macports.org
Thu Mar 17 01:06:27 PDT 2016
Revision: 146768
https://trac.macports.org/changeset/146768
Author: cal at macports.org
Date: 2016-03-17 01:06:27 -0700 (Thu, 17 Mar 2016)
Log Message:
-----------
contrib: Add new buildbot tooling
MacPortsAutoBuild is complicated and hasn't been refactored in a while. This is
a re-implementation of the necessary functionality that has been written by
petr@ and cal@ during MacPorts Meeting 2016 at Kranjska Gora, Slovenia.
Added Paths:
-----------
contrib/mp-buildbot/
contrib/mp-buildbot/README
contrib/mp-buildbot/mpbb
contrib/mp-buildbot/mpbb-checkout
contrib/mp-buildbot/mpbb-cleanup
contrib/mp-buildbot/mpbb-gather-archives
contrib/mp-buildbot/mpbb-install-dependencies
contrib/mp-buildbot/mpbb-install-port
contrib/mp-buildbot/mpbb-selfupdate
contrib/mp-buildbot/tools/
contrib/mp-buildbot/tools/uninstall-old-ports.tcl
Added: contrib/mp-buildbot/README
===================================================================
--- contrib/mp-buildbot/README (rev 0)
+++ contrib/mp-buildbot/README 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,58 @@
+# MacPorts Buildbot Scripts
+
+This folder contains a collection of scripts that will be run by the MacPorts
+buildbot buildslaves for continuous integration and package precompilation.
+
+## General structure
+
+There is a driver script mpbb (Macports buildbot) would call the scripts for
+the separated steps, using mpbb <step>. The single steps are provided by
+separate scripts using the following scheme: mpbb-<step>. The steps that are
+needed to be executed to build are the following ones:
+
+ 1. selfupdate --nosync
+ mpbb selfupdate --prefix /opt/local
+ 2. checkout ports tree (at specific rev, should update portindex)
+ mpbb checkout --prefix /opt/local --workdir "$workdir" --svn-url "$svnurl" --svn-revision "$svnrev"
+ 3. install all dependencies of a given port
+ mpbb install-dependencies --prefix /opt/local --port "$port"
+ 4. install a given port
+ mpbb install-port --prefix /opt/local --port "$port"
+ 5. gather archives
+ mpbb gather-archives --prefix /opt/local --port "$port" --workdir "$workdir" --archive-site "$archive_site" --staging-dir "$(pwd)/archive-staging"
+ 6. upload
+ Must be implemented in the buildmaster
+ 7. deploy
+ Must be implemented in the buildmaster
+ 8. cleanup
+ You do not have to run this, because install-dependencies runs it on
+ error and install-port always runs it. You can run it manually to clean
+ up any problems that might have occurred on the buildslave.
+ mpbb cleanup --prefix /opt/local --port "$port"
+
+## Step Implementation API
+
+Step provider scripts are sourced and should provide a number of functions:
+
+ - help: Should print a help message to stderr. Does not need to deal with
+ ending the execution.
+ - $command: Run the actual command.
+
+Some shell variables are available for usage in your subcommand:
+
+ - $command is the name of the subcommand
+ - $option_archive_site is the URL to the packages archive that will be used
+ to check for existing uploaded packages.
+ - $option_staging_dir is the folder where archives that are distributable
+ and should be upload must be put.
+ - $option_port is the port that should be installed in the run of mpbb.
+ - $option_prefix is the path to the MacPorts installation to use, as passed
+ with --prefix.
+ - $option_svn is the path to the svn binary to use.
+ - $option_svn_revision is the revision number to checkout in the given
+ Subversion repository.
+ - $option_svn_url is a URL pointing to a Subversion repository that
+ contains the dports and base subdirectories.
+ - $option_workdir is a path to a directory that can be used to store
+ temporary data. This data is retained between builds. You can, for
+ example, store a Subversion checkout there.
Added: contrib/mp-buildbot/mpbb
===================================================================
--- contrib/mp-buildbot/mpbb (rev 0)
+++ contrib/mp-buildbot/mpbb 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,173 @@
+#!/bin/bash
+# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4
+
+# Abort on error and undefined variables. Enable debug output
+set -eu
+
+## Options and their defaults
+option_default_archive_site=http://packages.macports.org
+option_default_help=0
+option_default_port=
+option_default_prefix=/opt/local
+option_default_staging_dir=${PWD:-/tmp/mp-buildbot}/archive-staging
+option_default_svn=$(which svn)
+option_default_svn_revision=HEAD
+option_default_svn_url=https://svn.macports.org/repository/macports/trunk
+option_default_workdir=${PWD:-/tmp/mp-buildbot}
+option_archive_site=$option_default_archive_site
+option_help=$option_default_help
+option_port=$option_default_port
+option_prefix=$option_default_prefix
+option_staging_dir=$option_default_staging_dir
+option_svn=$option_default_svn
+option_svn_revision=$option_default_svn_revision
+option_svn_url=$option_default_svn_url
+option_workdir=$option_default_workdir
+
+errmsg() {
+ printf >&2 "Error: %s\n" "$1"
+}
+
+usage() {
+ cols=$(tput cols) || cols=80
+
+ fmt -w $(( cols - 8 )) >&2 <<-EOF
+Usage: $0 COMMAND [OPTION...]
+
+mpbb, a script to help build MacPorts ports in a continous integration
+environment.
+
+OPTIONS
+
+ --archive-site URL
+ Base URL of the packages archive to check whether an archive was not
+ published yet. Default is $option_default_archive_site.
+
+ --help
+ Print this usage message.
+
+ --port PORT
+ Name of the port to build in MacPorts. This is required for the steps that
+ build a port.
+
+ --prefix PREFIX
+ The prefix of the MacPorts installation that will build the ports. Defaults
+ to $option_default_prefix.
+
+ --svn BINARY
+ Absolute path to the svn binary that you want to use for SVN operations. The
+ default is to find svn in your path, which is currently $option_default_svn.
+
+ --svn-url SVNURL
+ URL to a Subversion repository in a format accepted by Subversion. The
+ referenced folder must contain a dports and a base folder. The default is
+ $option_default_svn_url.
+
+ --svn-revision REVISION
+ Revision number in the specified Subversion repository to checkout. Defaults
+ to $option_default_svn_revision.
+
+ --staging-dir DIR
+ Directory where new distributable archives should be copied for deployment
+ on the archive server. Defaults to the 'archive-staging' subfolder in the
+ current directory.
+
+ --workdir WORKDIR
+ A scratch area that mpbb will use to put temporary files, ideally kept
+ between builds. Your MacPorts installation in --prefix needs to be able to
+ access this location. Defaults to your current directory, or
+ /tmp/mp-buildbot if \$PWD isn't set.
+EOF
+
+ printf >&2 "\nCOMMANDS\n"
+ for command in "${commands[@]}"; do
+ printf >&2 " %s\n" "$command"
+ printf " %s\n\n" "$(${command}-help | tr '\n' ' ')" | fmt -w $(( cols - 8 )) >&2
+ done
+
+ exit 2
+}
+
+## Load the mpbb-$command scripts and source them to load the actual implementations and help messages of available commands
+commands=()
+thisdir=$(cd "$(dirname "$0")" && pwd)
+for cmdfile in "$thisdir/mpbb-"*; do
+ . "$cmdfile"
+ commands+=(${cmdfile#$thisdir/mpbb-})
+done
+
+## The first argument should be the subcommand
+if [[ $# -lt 1 ]]; then
+ errmsg "No command specified"
+ usage
+fi
+
+command=$1
+shift
+if [ ! -f "$thisdir/mpbb-$command" ]; then
+ if [ "$command" != "--help" ]; then
+ errmsg "Unknown command $command"
+ fi
+ usage
+fi
+
+## Flag Parsing
+while [[ $# -gt 0 ]]; do
+ key=$1
+
+ case "$key" in
+ --*)
+ case "$key" in
+ --archive-site)
+ option_archive_site=$2
+ shift
+ ;;
+ --help)
+ option_help=1
+ ;;
+ --port)
+ option_port=$2
+ shift
+ ;;
+ --prefix)
+ option_prefix=$2
+ shift
+ ;;
+ --staging-dir)
+ option_staging_dir=$2
+ shift
+ ;;
+ --svn-url)
+ option_svn_url=$2
+ shift
+ ;;
+ --svn-revision)
+ option_svn_revision=$2
+ shift
+ ;;
+ --workdir)
+ option_workdir=$2
+ shift
+ ;;
+ --)
+ break
+ ;;
+ esac
+
+ shift
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+## If subcommand help is requested, print that
+if [[ $option_help -eq 1 ]]; then
+ usage
+fi
+## Otherwise, run the command and deal with errors
+if ! $command; then
+ errmsg "Failed to run $command"
+ exit 1
+fi
Property changes on: contrib/mp-buildbot/mpbb
___________________________________________________________________
Added: svn:executable
+ *
Added: contrib/mp-buildbot/mpbb-checkout
===================================================================
--- contrib/mp-buildbot/mpbb-checkout (rev 0)
+++ contrib/mp-buildbot/mpbb-checkout 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Note:
+# This script is sourced by the its wrapper script.
+# Do not execute this directly!
+
+
+checkout-help() {
+ cat <<EOF
+Update or checkout a working copy of the ports tree from --svn-url to specific
+SVN revision (--svn-revision), update or create the PortIndex and configure
+the MacPorts installation to use this ports tree.
+EOF
+}
+
+checkout() {
+ dports_svn=${option_workdir}/dports
+ tools_svn=${option_workdir}/tools
+ svn=${option_svn}
+ svn_url=${option_svn_url}
+ svn_rev=${option_svn_revision}
+
+ if [[ -d "${tools_svn}/.svn" ]] ; then
+ echo "Update macports tools from svn..."
+ if [[ -e "${tools_svn}/.svn/lock" ]]; then
+ "$svn" --non-interactive cleanup "${tools_svn}"
+ fi
+ "$svn" update --non-interactive \
+ -r HEAD \
+ "${tools_svn}"
+ else
+ echo "Checking out macports tools from svn..."
+ mkdir -p "${option_workdir}"
+ "$svn" checkout --non-interactive \
+ -r HEAD "${svn_url}/base/portmgr/jobs" \
+ "${tools_svn}"
+ fi
+
+ if [[ -d "${dports_svn}/.svn" ]] ; then
+ echo "Update macports from svn..."
+ # TODO: add switching of SVN server
+ if [[ -e "${dports_svn}/.svn/lock" ]]; then
+ "$svn" --non-interactive cleanup "${dports_svn}"
+ fi
+ "$svn" update --non-interactive \
+ -r "${svn_rev}" \
+ "${dports_svn}"
+ else
+ echo "Checking out macports from svn..."
+ mkdir -p "${option_workdir}"
+ "$svn" checkout --non-interactive \
+ -r "${svn_rev}" "${svn_url}/dports" \
+ "${dports_svn}"
+ fi
+
+ (cd "${dports_svn}" && "${option_prefix}/bin/portindex")
+
+ cat > "${option_prefix}/etc/macports/sources.conf" <<EOF
+# Automatically overwritten by mpbb-checkout
+# Do not edit !!!
+#
+file://${dports_svn} [default]
+EOF
+
+}
Added: contrib/mp-buildbot/mpbb-cleanup
===================================================================
--- contrib/mp-buildbot/mpbb-cleanup (rev 0)
+++ contrib/mp-buildbot/mpbb-cleanup 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Note:
+# This script is sourced by the its wrapper script.
+# Do not execute this directly!
+
+cleanup-help() {
+ echo "Clean up build remnants and leftovers caused by the build of port --port."
+ echo "Note that you should not have to run this explicitly, as it is automatically run after install-dependencies on error and in any case after install-port."
+}
+
+cleanup() {
+ if [ -z "${option_port}" ]; then
+ errmsg "--port is required"
+ return 1
+ fi
+
+ echo "Running cleanup..."
+ # Deactivate all ports
+ "${option_prefix}/bin/port" -f deactivate active
+ # Clean the ports we built and their dependencies
+ "${option_prefix}/bin/port" -f clean --dist --logs --work "${option_port}" rdepof:"${option_port}"
+ # Uninstall obsolete ports
+ "${option_prefix}/bin/port-tclsh" "${thisdir}/tools/uninstall-old-ports.tcl"
+}
Added: contrib/mp-buildbot/mpbb-gather-archives
===================================================================
--- contrib/mp-buildbot/mpbb-gather-archives (rev 0)
+++ contrib/mp-buildbot/mpbb-gather-archives 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Note:
+# This script is sourced by the its wrapper script.
+# Do not execute this directly!
+
+
+gather-archives-help() {
+ echo "Gather unpublished and distributable archives of the built port."
+}
+
+gather-archives() {
+ curl=/usr/bin/curl
+ tclsh=${option_prefix}/bin/port-tclsh
+ mkdir -p "${option_staging_dir}"
+
+ archive_path=$("${option_prefix}/bin/port" -q location "${option_port}")
+ archive_basename=$(basename "${archive_path}")
+
+ if ! ${curl} -fIsL "${option_archive_site}/${option_port}/${archive_basename}" > /dev/null; then
+
+ # check for distributability
+ if "${tclsh}" "${option_workdir}/tools/port_binary_distributable.tcl" -v "${option_port}"; then
+ echo "Staging archive for upload: ${archive_basename}"
+ mkdir -p "${option_staging_dir}/${option_port}"
+ cp "${archive_path}" "${option_staging_dir}/${option_port}/${archive_basename}":w
+ fi
+ fi
+}
Added: contrib/mp-buildbot/mpbb-install-dependencies
===================================================================
--- contrib/mp-buildbot/mpbb-install-dependencies (rev 0)
+++ contrib/mp-buildbot/mpbb-install-dependencies 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Note:
+# This script is sourced by the its wrapper script.
+# Do not execute this directly!
+
+
+install-dependencies-help() {
+ echo "Builds and install all dependencies of port --port."
+}
+
+install-dependencies() {
+ if [ -z "${option_port}" ]; then
+ errmsg "--port is required"
+ return 1
+ fi
+
+ if ! "${option_prefix}/bin/port" -d install --unrequested depof:"${option_port}"; then
+
+ echo "Build failed, cleaning up..."
+ cleanup
+ return 1
+ fi
+}
Added: contrib/mp-buildbot/mpbb-install-port
===================================================================
--- contrib/mp-buildbot/mpbb-install-port (rev 0)
+++ contrib/mp-buildbot/mpbb-install-port 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Note:
+# This script is sourced by the its wrapper script.
+# Do not execute this directly!
+
+
+install-port-help() {
+ echo "Builds and install the port --port itself."
+}
+
+install-port() {
+ if [ -z "${option_port}" ]; then
+ errmsg "--port is required"
+ return 1
+ fi
+
+ status=0
+ "${option_prefix}/bin/port" -d install "${option_port}" || status=$?
+
+ cleanup
+ return $status
+}
Added: contrib/mp-buildbot/mpbb-selfupdate
===================================================================
--- contrib/mp-buildbot/mpbb-selfupdate (rev 0)
+++ contrib/mp-buildbot/mpbb-selfupdate 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Note:
+# This script is sourced by the its wrapper script.
+# Do not execute this directly!
+
+selfupdate-help() {
+ echo "Run selfupdate --nosync in the MacPorts installation specified by the --prefix flag"
+}
+
+selfupdate() {
+ "${option_prefix}/bin/port" -d selfupdate --nosync
+}
Added: contrib/mp-buildbot/tools/uninstall-old-ports.tcl
===================================================================
--- contrib/mp-buildbot/tools/uninstall-old-ports.tcl (rev 0)
+++ contrib/mp-buildbot/tools/uninstall-old-ports.tcl 2016-03-17 08:06:27 UTC (rev 146768)
@@ -0,0 +1,103 @@
+#!/usr/bin/env port-tclsh
+
+proc printUsage {} {
+ puts "Usage: $::argv0 \[-hV\]"
+ puts " -h This help"
+ puts " -V show version and MacPorts version being used"
+}
+
+set MY_VERSION 0.2
+
+set showVersion 0
+
+set origArgv $::argv
+while {[string index [lindex $::argv 0] 0] == "-" } {
+ switch [string range [lindex $::argv 0] 1 end] {
+ h {
+ printUsage
+ exit 0
+ }
+ V {
+ set showVersion 1
+ }
+ default {
+ puts "Unknown option [lindex $::argv 0]"
+ printUsage
+ exit 2
+ }
+ }
+ set ::argv [lrange $::argv 1 end]
+}
+
+package require macports
+mportinit
+
+if {$showVersion} {
+ puts "uninstall-old-ports.tcl version $MY_VERSION"
+ puts "MacPorts version [macports::version]"
+ exit 0
+}
+
+##
+# Compare two versions in the form of (epoch, version, revision).
+#
+# @param versionA Tcl array of the first version with the keys epoch, version, and revision.
+# @param versionB Tcl array of the second version in the same format as versionA.
+# @return An integer < 0 if the first version is smaller than the second. 0, if
+# both versions are equal. An integer > 0 if the second version is
+# larger than the first.
+proc compare_version_tuple {versionA versionB} {
+ upvar $versionA vA
+ upvar $versionB vB
+
+ set epochCompare [vercmp $vA(epoch) $vB(epoch)]
+ set versionCompare [vercmp $vA(version) $vB(version)]
+ set revisionCompare [vercmp $vA(revision) $vB(revision)]
+
+ if {$epochCompare != 0} {
+ return $epochCompare
+ }
+ if {$versionCompare != 0} {
+ return $versionCompare
+ }
+ return $revisionCompare
+}
+
+foreach port [registry::installed] {
+ # Set to yes if a port is obsolete
+ set old no
+
+ set installed_name [lindex $port 0]
+ set installed_version [lindex $port 1]
+ set installed_revision [lindex $port 2]
+ set installed_variants [lindex $port 3]
+ set installed_epoch [lindex $port 5]
+
+ array set installed_version_tuple {}
+ set installed_version_tuple(epoch) $installed_epoch
+ set installed_version_tuple(version) $installed_version
+ set installed_version_tuple(revision) $installed_revision
+
+ set portindex_match [mportlookup $installed_name]
+ if {[llength $portindex_match] < 2} {
+ # Not found in index, classify as old
+ ui_msg "Removing $installed_name$installed_variants $installed_epoch@$installed_version-$installed_revision because it is no longer in the PortIndex"
+ set old yes
+ } else {
+ array unset portinfo
+ array set portinfo [lindex $portindex_match 1]
+
+ set result [compare_version_tuple portinfo installed_version_tuple]
+ if {$result > 0} {
+ # Port is outdated because the version in the index is newer than
+ # the installed one
+ ui_msg "Removing $installed_name$installed_variants $installed_epoch@$installed_version-$installed_revision because there is a newer version in the PortIndex"
+ set old yes
+ }
+ # If the version we have is newer than the one in the PortIndex, we are
+ # probably building agaist an old version of the ports tree.
+ }
+ if {$old} {
+ registry::uninstall $installed_name $installed_version $installed_revision $installed_variants [list ports_force 1]
+ }
+}
Property changes on: contrib/mp-buildbot/tools/uninstall-old-ports.tcl
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/macports-changes/attachments/20160317/1b7c84ef/attachment.html>
More information about the macports-changes
mailing list