[152581] contrib/mp-buildbot
cal at macports.org
cal at macports.org
Mon Sep 12 12:14:24 PDT 2016
Revision: 152581
https://trac.macports.org/changeset/152581
Author: cal at macports.org
Date: 2016-09-12 12:14:24 -0700 (Mon, 12 Sep 2016)
Log Message:
-----------
mp-buildbot: Don't re-run previously failed builds
If a port's build has previously failed, we do not want to waste cycles
to re-attempt it (e.g. as dependencies of other ports), unless the
failed port's definition has changed.
To achieve that, create a file with a reference to the last failed build
for every failing port, and fail builds that depend on failed ports
early.
Modified Paths:
--------------
contrib/mp-buildbot/README.md
contrib/mp-buildbot/mpbb
contrib/mp-buildbot/mpbb-install-dependencies
contrib/mp-buildbot/mpbb-install-port
Added Paths:
-----------
contrib/mp-buildbot/functions
contrib/mp-buildbot/tools/canonical_variants.tcl
Modified: contrib/mp-buildbot/README.md
===================================================================
--- contrib/mp-buildbot/README.md 2016-09-12 19:13:35 UTC (rev 152580)
+++ contrib/mp-buildbot/README.md 2016-09-12 19:14:24 UTC (rev 152581)
@@ -91,3 +91,6 @@
persist for the duration of an `mpbb` run, so it may be used to
share ancillary files (e.g., a Subversion checkout of the ports
tree) between builds of different ports.
+- `$option_failcache_dir`:
+ A directory for storing information about previously failed builds which
+ saves time because builds that are known to fail will not be attempted.
Added: contrib/mp-buildbot/functions
===================================================================
--- contrib/mp-buildbot/functions (rev 0)
+++ contrib/mp-buildbot/functions 2016-09-12 19:14:24 UTC (rev 152581)
@@ -0,0 +1,134 @@
+#!/bin/bash
+# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4
+
+# Helper functions for mp-buildbot
+
+# Print $0 and arguments to standard error.
+# Unset IFS to ensure "$*" uses spaces as separators.
+msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
+err() { msg error: "$@"; }
+warn() { msg warning: "$@"; }
+
+## Compute a failcache hash for the given port
+#
+# Computes and prints a hash uniquely identifying a specific state of a port's
+# definition files, including the Portfile's hash as well as the port's
+# patchfiles. To build the hash, this function executes the following
+# algorithm:
+# - For the Portfile, and each file in files/ (if any), calculate a SHA256
+# hash
+# - Sort the hash values alphabetically
+# - Hash the result using SHA256
+# This means a failcache entry will not match if a patchfile changes. A common
+# case where this is the desired behavior is a port committed without
+# a required patchfile.
+#
+# Valid arguments are all arguments accepted by "port dir".
+compute_failcache_hash() {
+ local portdir
+ local -a filelist
+
+ portdir=$("${option_prefix}/bin/port" dir "$@")
+ if [ $? -ne 0 ] || [ -z "$portdir" ]; then
+ err "Could not compute failcache hash: port dir" "$@" "failed"
+ return 1
+ fi
+
+ if [ ! -d "$portdir" ]; then
+ err "Port directory $portdir does not exist"
+ return 2
+ fi
+
+ filelist=("$portdir/Portfile")
+ if [ -d "$portdir/files" ]; then
+ filelist+=("$portdir/files")
+ fi
+
+ find "${filelist[@]}" -type f -exec openssl dgst -sha256 {} \; |\
+ cut -d' ' -f2 |\
+ sort |\
+ openssl dgst -sha256 |\
+ cut -d' ' -f2
+}
+
+## Compute a key that uniquely identifies a (port, variants, portfile-hash) tuple
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Invokes "port dir" to find the Portfile and patchfiles and
+# computes a checksum of these files that will become part of the hash.
+failcache_key() {
+ local port=$1
+ if [ -z "$port" ]; then
+ err "failcache_key expects a port argument, but none was given."
+ return 1
+ fi
+
+ local checksum
+ checksum=$(compute_failcache_hash "$port")
+ if [ $? -ne 0 ]; then
+ err "compute_failcache_hash $port failed"
+ return 2
+ fi
+
+ local canonical_variants
+ canonical_variants=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/canonical_variants.tcl" "$@")
+ if [ $? -ne 0 ]; then
+ err "tools/canonical_variants.tcl" "$@" "failed"
+ return 4
+ fi
+
+ echo "$port $canonical_variants $checksum"
+}
+
+## Test whether a given port with variants has previously failed.
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Succeeds if the port did not previsouly fail to build,
+# fails if the port is known to fail.
+failcache_test() {
+ local key
+ key=$(failcache_key "$@")
+ if [ $? -ne 0 ]; then
+ err "Could not determine failcache key for" "$@"
+ return 1
+ fi
+
+ if [ -f "${option_failcache_dir}/${key}" ]; then
+ printf "port %s previously failed in build %s\n" "${key}" "$(<"${option_failcache_dir}/${key}")"
+ return 1
+ else
+ return 0
+ fi
+}
+
+## Mark a build of a given port with variants as successful.
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Removes any database entries that marked a port as failed.
+failcache_success() {
+ local key
+ key=$(failcache_key "$@")
+ if [ $? -ne 0 ]; then
+ err "Could not determine failcache key for" "$@"
+ return 1
+ fi
+
+ rm -f "${option_failcache_dir}/${key}"
+}
+
+## Mark a build of a given port with variants as failed.
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Creates or updates the timestamp of a database entry that
+# marks a port as failed.
+failcache_failure() {
+ local key
+ key=$(failcache_key "$@")
+ if [ $? -ne 0 ]; then
+ err "Could not determine failcache key for" "$@"
+ return 1
+ fi
+
+ mkdir -p "${option_failcache_dir}"
+ echo "${BUILDBOT_BUILDURL:-unknown}" > "${option_failcache_dir}/${key}"
+}
Modified: contrib/mp-buildbot/mpbb
===================================================================
--- contrib/mp-buildbot/mpbb 2016-09-12 19:13:35 UTC (rev 152580)
+++ contrib/mp-buildbot/mpbb 2016-09-12 19:14:24 UTC (rev 152581)
@@ -8,11 +8,10 @@
# Don't inherit any option variables from the calling environment.
unset "${!option_@}"
-# Print $0 and arguments to standard error.
-# Unset IFS to ensure "$*" uses spaces as separators.
-msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
-err() { msg error: "$@"; }
-warn() { msg warning: "$@"; }
+# Load function library
+thisdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=functions
+. "$thisdir/functions"
mpbb-usage() {
# "prog" is defined in mpbb-help.
@@ -126,6 +125,7 @@
option_jobs_dir=${option_work_dir}/infrastructure/jobs
option_log_dir=${option_work_dir}/logs
}
+option_failcache_dir=${option_work_dir}/failcache
# Inform the user if old repositories are still present.
if [[ -d ${option_work_dir}/tools/.svn ]]; then
@@ -146,7 +146,6 @@
# must define functions "FOO" and "FOO-usage".
cmds=()
usages=(mpbb-usage)
-thisdir=$(cd "$(dirname "$0")" && pwd)
for cmdfile in "$thisdir/mpbb-"*; do
# Unfortunately ShellCheck does not currently support following multiple
# files, so we'll just disable the warning.
Modified: contrib/mp-buildbot/mpbb-install-dependencies
===================================================================
--- contrib/mp-buildbot/mpbb-install-dependencies 2016-09-12 19:13:35 UTC (rev 152580)
+++ contrib/mp-buildbot/mpbb-install-dependencies 2016-09-12 19:14:24 UTC (rev 152581)
@@ -41,6 +41,9 @@
local dependencies
local dependencies_count
local dependencies_counter
+ local depname
+ local depvariants
+ local failcachecounter
# $option_log_dir is set in mpbb
# shellcheck disable=SC2154
local log_status_dependencies="${option_log_dir}/dependencies-progress.txt"
@@ -53,7 +56,7 @@
# calculate list of dependencies in-order
# $option_prefix and $thisdir are set in mpbb
# shellcheck disable=SC2154
- dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port")
+ dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@")
if [ $? -ne 0 ]; then
echo "Calculating dependencies for '$port' failed, aborting." >&2
echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
@@ -72,26 +75,67 @@
echo "$dependencies" | sed -E 's/^/ - /' | tee -a "$log_status_dependencies"
echo >> "$log_status_dependencies"
- echo "$dependencies" | while read -r dependency; do
+ # Check whether any of the dependencies have previously failed
+ failcachecounter=0
+ while read -r dependency; do
# Split portname +variant1+variant2 into portname and variants, where
# the variants are optional.
depname=${dependency%% *}
depvariants=${dependency:${#depname}+1}
+ # $depvariants isn't quoted on purpose
+ # shellcheck disable=SC2086
+ if ! failcache_test "$depname" $depvariants; then
+ text="Dependency '${depname}' with variants '${depvariants}' has previously failed and is required."
+ echo "$text" >&2
+ echo "$text" >> "$log_status_dependencies"
+ echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
+ failcachecounter=$((failcachecounter + 1))
+ fi
+ done <<<"$dependencies"
+
+ if [ $failcachecounter -gt 0 ]; then
+ echo "Aborting build because $failcachecounter dependencies are known to fail." >&2
+ return 1
+ fi
+
+ while read -r dependency; do
+ # Split portname +variant1+variant2 into portname and variants, where
+ # the variants are optional.
+ depname=${dependency%% *}
+ depvariants=${dependency:${#depname}+1}
+
text="Installing dependency ($dependencies_counter of $dependencies_count) '${depname}' with variants '${depvariants}'"
echo "----> ${text}"
echo -n "${text} ... " >> "$log_status_dependencies"
# $depvariants isn't quoted on purpose
# shellcheck disable=SC2086
if ! "${option_prefix}/bin/port" -d install --unrequested "$depname" $depvariants; then
- echo "Build of dependency '${depname}' failed, aborting." >&2
+ echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2
echo "[FAIL]" >> "$log_status_dependencies"
echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
+
+ # Update failcache
+ # $depvariants isn't quoted on purpose
+ # shellcheck disable=SC2086
+ failcache_failure "$depname" $depvariants
+ if [ $? -ne 0 ]; then
+ err "failcache_failure $depname $depvariants failed."
+ return 1
+ fi
return 1
else
echo "[OK]" >> "$log_status_dependencies"
+ # Remove failcache if it exists
+ # $depvariants isn't quoted on purpose
+ # shellcheck disable=SC2086
+ failcache_success "$depname" $depvariants
+ if [ $? -ne 0 ]; then
+ err "failcache_success $depname $depvariants failed."
+ return 1
+ fi
dependencies_counter=$((dependencies_counter + 1))
fi
- done
+ done <<<"$dependencies"
}
Modified: contrib/mp-buildbot/mpbb-install-port
===================================================================
--- contrib/mp-buildbot/mpbb-install-port 2016-09-12 19:13:35 UTC (rev 152580)
+++ contrib/mp-buildbot/mpbb-install-port 2016-09-12 19:14:24 UTC (rev 152581)
@@ -55,10 +55,23 @@
time_start=$(date +%s)
# $option_prefix is set in mpbb
# shellcheck disable=SC2154
- if ! "${option_prefix}/bin/port" -dk install "$port"; then
+ if "${option_prefix}/bin/port" -dk install "$@"; then
+ # Remove failcache if it exists
+ failcache_success "$@"
+ if [ $? -ne 0 ]; then
+ err "failcache_success" "$@" "failed."
+ return 1
+ fi
+ else
echo "Build of '$port' failed."
# log: summary for the portwatcher
echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
+ # update failcache
+ failcache_failure "$@"
+ if [ $? -ne 0 ]; then
+ err "failcache_failure" "$@" "failed."
+ return 1
+ fi
return 1
fi
time_stop=$(date +%s)
Added: contrib/mp-buildbot/tools/canonical_variants.tcl
===================================================================
--- contrib/mp-buildbot/tools/canonical_variants.tcl (rev 0)
+++ contrib/mp-buildbot/tools/canonical_variants.tcl 2016-09-12 19:14:24 UTC (rev 152581)
@@ -0,0 +1,97 @@
+#!/usr/bin/env port-tclsh
+# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4
+# Writes the canonical list of variants as it appears in binary archive names
+# to stdout.
+#
+# Copyright (c) 2016 The MacPorts Project.
+# Copyright (c) 2016 Clemens Lang <cal at macports.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# 3. Neither the name of the MacPorts project, nor the names of any contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
+# AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package require macports
+
+if {[llength $::argv] == 0} {
+ puts stderr "Usage: $argv0 <portname> \[(+|-)variant...\]"
+ exit 1
+}
+
+# initialize macports
+if {[catch {mportinit "" "" ""} result]} {
+ ui_error "$errorInfo"
+ ui_error "Failed to initialize ports sytem: $result"
+ exit 1
+}
+
+# look up the path of the Portfile for the given port
+set portname [lindex $::argv 0]
+#try -pass_signal {...}
+try {
+ set result [mportlookup $portname]
+ if {[llength $result] < 2} {
+ ui_error "No such port: $portname"
+ exit 1
+ }
+} catch {{*} eCode eMessage} {
+ ui_error "mportlookup $portname failed: $eMessage"
+ exit 1
+}
+
+# parse the given variants from the command line
+array set variants {}
+foreach item [lrange $::argv 1 end] {
+ foreach {_ sign variant} [regexp -all -inline -- {([-+])([[:alpha:]_]+[\w\.]*)} $item] {
+ set variants($variant) $sign
+ }
+}
+
+# open the port to get its active variants
+array set portinfo [lindex $result 1]
+#try -pass_signal {...}
+try {
+ set mport [mportopen $portinfo(porturl) [list subport $portname] [array get variants]]
+} catch {{*} eCode eMessage} {
+ ui_error "mportopen ${portinfo(porturl)} failed: $eMessage"
+ exit 1
+}
+
+array set info [mportinfo $mport]
+puts $info(canonical_active_variants)
+
+#try -pass_signal {...}
+try {
+ mportclose $mport
+} catch {{*} eCode eMessage} {
+ ui_warn "mportclose $portname failed: $eMessage"
+}
+
+# shut down MacPorts
+#try -pass_signal {...}
+try {
+ mportshutdown
+} catch {{*} eCode eMessage} {
+ ui_error "mportshutdown failed: $eMessage"
+ exit 1
+}
Property changes on: contrib/mp-buildbot/tools/canonical_variants.tcl
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/macports-changes/attachments/20160912/ac9d43ea/attachment-0001.html>
More information about the macports-changes
mailing list