RFC: mp-buildbot failcache
Clemens Lang
cal at macports.org
Mon Sep 12 05:41:37 PDT 2016
Hi,
here's a patch against mp-buildbot that would add a fail cache for use
on the buildbots. Unfortunately I don't have a test setup yet, so I'm
hesitating to commit this immediately. I'd welcome feedback, a code
review, or testing by somebody with a working builbot setup.
Basically, I'm keeping failcache per tuple of (portname, canonical
variants, checksum) where
- I need one additional call to a custom port client to dump the
canonical variants
- calculate the checksum as
sha256(sort(sha256(Portfile), sha256(files/*)))
which will make the buildbots re-try a dependency if, for example, a
maintainer forgot to commit a patch file
Index: functions
===================================================================
--- functions (nonexistent)
+++ functions (working copy)
@@ -0,0 +1,128 @@
+#!/bin/bash
+# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4
+
+# Helper functions for mp-buildbot
+
+# Print $0 and arguments to standard error.
+# Unset IFS to ensure "$*" uses spaces as separators.
+msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
+err() { msg error: "$@"; }
+warn() { msg warning: "$@"; }
+
+## Compute a failcache hash for the given port
+#
+# Computes and prints a hash uniquely identifying a specific state of a port's
+# definition files, including the Portfile's hash as well as the port's
+# patchfiles. To build the hash, this function executes the following
+# algorithm:
+# - For the Portfile, and each file in files/ (if any), calculate a SHA256
+# hash
+# - Sort the hash values alphabetically
+# - Hash the result using SHA256
+# This means a failcache entry will not match if a patchfile changes. A common
+# case where this is the desired behavior is a port committed without
+# a required patchfile.
+#
+# Valid arguments are all arguments accepted by "port dir".
+compute_failcache_hash() {
+ local portdir
+ local -a filelist
+
+ portdir=$("${option_prefix}/bin/port" dir "$@")
+ if [ $? -ne 0 ] || [ -z "$portdir" ]; then
+ err "Could not compute failcache hash: port dir" "$@" "failed"
+ return 1
+ fi
+
+ if [ ! -d "$portdir" ]; then
+ err "Port directory $portdir does not exist"
+ return 2
+ fi
+
+ filelist=("$portdir/Portfile")
+ if [ -d "$portdir/files" ]; then
+ filelist+=("$portdir/files")
+ fi
+
+ find "${filelist[@]}" -type f -exec openssl dgst -sha256 {} \; |\
+ cut -d' ' -f2 |\
+ sort |\
+ openssl dgst -sha256 |\
+ cut -d' ' -f2
+}
+
+## Compute a key that uniquely identifies a (port, variants, portfile-hash) tuple
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Invokes "port dir" to find the Portfile and patchfiles and
+# computes a checksum of these files that will become part of the hash.
+failcache_key() {
+ local port=$1
+ if [ -z "$port" ]; then
+ err "failcache_key expects a port argument, but none was given."
+ return 1
+ fi
+
+ local checksum
+ checksum=$(compute_failcache_hash "$port")
+ if [ $? -ne 0 ]; then
+ err "compute_failcache_hash $port failed"
+ return 2
+ fi
+
+ local canonical_variants
+ canonical_variants=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/canonical_variants.tcl" "$@")
+ if [ $? -ne 0 ]; then
+ err "tools/canonical_variants.tcl" "$@" "failed"
+ return 4
+ fi
+
+ echo "$port $canonical_variants $checksum"
+}
+
+## Test whether a given port with variants has previously failed.
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Succeeds if the port did not previsouly fail to build, fails
+# if the port is known to fail.
+failcache_test() {
+ local key
+ key=$(failcache_key "$@")
+ if [ $? -ne 0 ]; then
+ err "Could not determine failcache key for" "$@"
+ return 1
+ fi
+
+ test -f "${option_failcache_dir}/${key}"
+}
+
+## Mark a build of a given port with variants as successful.
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Removes any database entries that marked a port as failed.
+failcache_success() {
+ local key
+ key=$(failcache_key "$@")
+ if [ $? -ne 0 ]; then
+ err "Could not determine failcache key for" "$@"
+ return 1
+ fi
+
+ rm -f "${option_failcache_dir}/${key}"
+}
+
+## Mark a build of a given port with variants as failed.
+#
+# Valid arguments are a port name, optionally followed by a variant
+# specification. Creates or updates the timestamp of a database entry that
+# marks a port as failed.
+failcache_failure() {
+ local key
+ key=$(failcache_key "$@")
+ if [ $? -ne 0 ]; then
+ err "Could not determine failcache key for" "$@"
+ return 1
+ fi
+
+ touch "${option_failcache_dir}/${key}"
+}
Index: mpbb
===================================================================
--- mpbb (revision 152544)
+++ mpbb (working copy)
@@ -8,11 +8,9 @@
# Don't inherit any option variables from the calling environment.
unset "${!option_@}"
-# Print $0 and arguments to standard error.
-# Unset IFS to ensure "$*" uses spaces as separators.
-msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
-err() { msg error: "$@"; }
-warn() { msg warning: "$@"; }
+# Load function library
+thisdir=$(cd "$(dirname "$0")" && pwd)
+. "$thisdir/functions"
mpbb-usage() {
# "prog" is defined in mpbb-help.
@@ -121,6 +119,7 @@
# Not really options, but pretend they are because they're global.
option_jobs_dir=${option_work_dir}/infrastructure/jobs
option_log_dir=${option_work_dir}/logs
+option_failcache_dir=${option_work_dir}/failcache
# Inform the user if old repositories are still present.
if [[ -d ${option_work_dir}/tools/.svn ]]; then
@@ -141,7 +140,6 @@
# must define functions "FOO" and "FOO-usage".
cmds=()
usages=(mpbb-usage)
-thisdir=$(cd "$(dirname "$0")" && pwd)
for cmdfile in "$thisdir/mpbb-"*; do
# Unfortunately ShellCheck does not currently support following multiple
# files, so we'll just disable the warning.
Index: mpbb-install-dependencies
===================================================================
--- mpbb-install-dependencies (revision 152544)
+++ mpbb-install-dependencies (working copy)
@@ -41,6 +41,9 @@
local dependencies
local dependencies_count
local dependencies_counter
+ local depname
+ local depvariants
+ local failcache_key
# $option_log_dir is set in mpbb
# shellcheck disable=SC2154
local log_status_dependencies="${option_log_dir}/dependencies-progress.txt"
@@ -53,7 +56,7 @@
# calculate list of dependencies in-order
# $option_prefix and $thisdir are set in mpbb
# shellcheck disable=SC2154
- dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port")
+ dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@")
if [ $? -ne 0 ]; then
echo "Calculating dependencies for '$port' failed, aborting." >&2
echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
@@ -72,6 +75,24 @@
echo "$dependencies" | sed -E 's/^/ - /' | tee -a "$log_status_dependencies"
echo >> "$log_status_dependencies"
+ # Check whether any of the dependencies have previously failed
+ echo "$dependencies" | while read -r dependency; do
+ # Split portname +variant1+variant2 into portname and variants, where
+ # the variants are optional.
+ depname=${dependency%% *}
+ depvariants=${dependency:${#depname}+1}
+
+ # $depvariants isn't quoted on purpose
+ # shellcheck disable=SC2086
+ if ! failcache_test "$depname" $depvariants; then
+ text="Dependency '${depname}' with variants '${depvariants}' has previously failed and is required."
+ echo "$text" >&2
+ echo "$text" >> "$log_status_dependencies"
+ echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
+ return 1
+ fi
+ done
+
echo "$dependencies" | while read -r dependency; do
# Split portname +variant1+variant2 into portname and variants, where
# the variants are optional.
@@ -84,12 +105,21 @@
# $depvariants isn't quoted on purpose
# shellcheck disable=SC2086
if ! "${option_prefix}/bin/port" -d install --unrequested "$depname" $depvariants; then
- echo "Build of dependency '${depname}' failed, aborting." >&2
+ echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2
echo "[FAIL]" >> "$log_status_dependencies"
echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
+
+ # Update failcache
+ # $depvariants isn't quoted on purpose
+ # shellcheck disable=SC2086
+ failcache_failure "$depname" $depvariants
return 1
else
echo "[OK]" >> "$log_status_dependencies"
+ # Remove failcache if it exists
+ # $depvariants isn't quoted on purpose
+ # shellcheck disable=SC2086
+ failcache_success "$depname" $depvariants
dependencies_counter=$((dependencies_counter + 1))
fi
done
Index: mpbb-install-port
===================================================================
--- mpbb-install-port (revision 152544)
+++ mpbb-install-port (working copy)
@@ -55,10 +55,15 @@
time_start=$(date +%s)
# $option_prefix is set in mpbb
# shellcheck disable=SC2154
- if ! "${option_prefix}/bin/port" -dk install "$port"; then
+ if "${option_prefix}/bin/port" -dk install "$@"; then
+ # Remove failcache if it exists
+ failcache_success "$@"
+ else
echo "Build of '$port' failed."
# log: summary for the portwatcher
echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
+ # update failcache
+ failcache_failure "$@"
return 1
fi
time_stop=$(date +%s)
Index: tools/canonical_variants.tcl
===================================================================
--- tools/canonical_variants.tcl (nonexistent)
+++ tools/canonical_variants.tcl (working copy)
@@ -0,0 +1,97 @@
+#!/usr/bin/env port-tclsh
+# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4
+# Writes the canonical list of variants as it appears in binary archive names
+# to stdout.
+#
+# Copyright (c) 2016 The MacPorts Project.
+# Copyright (c) 2016 Clemens Lang <cal at macports.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# 3. Neither the name of the MacPorts project, nor the names of any contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
+# AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package require macports
+
+if {[llength $::argv] == 0} {
+ puts stderr "Usage: $argv0 <portname> \[(+|-)variant...\]"
+ exit 1
+}
+
+# initialize macports
+if {[catch {mportinit "" "" ""} result]} {
+ ui_error "$errorInfo"
+ ui_error "Failed to initialize ports sytem: $result"
+ exit 1
+}
+
+# look up the path of the Portfile for the given port
+set portname [lindex $::argv 0]
+#try -pass_signal {...}
+try {
+ set result [mportlookup $portname]
+ if {[llength $result] < 2} {
+ ui_error "No such port: $portname"
+ exit 1
+ }
+} catch {{*} eCode eMessage} {
+ ui_error "mportlookup $portname failed: $eMessage"
+ exit 1
+}
+
+# parse the given variants from the command line
+array set variants {}
+foreach item [lrange $::argv 1 end] {
+ foreach {_ sign variant} [regexp -all -inline -- {([-+])([[:alpha:]_]+[\w\.]*)} $item] {
+ set variants($variant) $sign
+ }
+}
+
+# open the port to get its active variants
+array set portinfo [lindex $result 1]
+#try -pass_signal {...}
+try {
+ set mport [mportopen $portinfo(porturl) [list subport $portname] [array get variants]]
+} catch {{*} eCode eMessage} {
+ ui_error "mportopen ${portinfo(porturl)} failed: $eMessage"
+ exit 1
+}
+
+array set info [mportinfo $mport]
+puts $info(canonical_active_variants)
+
+#try -pass_signal {...}
+try {
+ mportclose $mport
+} catch {{*} eCode eMessage} {
+ ui_warn "mportclose $portname failed: $eMessage"
+}
+
+# shut down MacPorts
+#try -pass_signal {...}
+try {
+ mportshutdown
+} catch {{*} eCode eMessage} {
+ ui_error "mportshutdown failed: $eMessage"
+ exit 1
+}
Property changes on: tools/canonical_variants.tcl
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Id
\ No newline at end of property
More information about the macports-dev
mailing list