[152581] contrib/mp-buildbot
Ryan Schmidt
ryandesign at macports.org
Mon Sep 12 12:41:55 PDT 2016
On Sep 12, 2016, at 2:14 PM, cal at macports.org wrote:
>
> Revision
> 152581
> Author
> cal at macports.org
> Date
> 2016-09-12 12:14:24 -0700 (Mon, 12 Sep 2016)
> Log Message
>
> mp-buildbot: Don't re-run previously failed builds
>
> If a port's build has previously failed, we do not want to waste cycles
> to re-attempt it (e.g. as dependencies of other ports), unless the
> failed port's definition has changed.
>
> To achieve that, create a file with a reference to the last failed build
> for every failing port, and fail builds that depend on failed ports
> early.
Fantastic! Thank you so much.
> Modified Paths
>
> • contrib/mp-buildbot/README.md
> • contrib/mp-buildbot/mpbb
> • contrib/mp-buildbot/mpbb-install-dependencies
> • contrib/mp-buildbot/mpbb-install-port
> Added Paths
>
> • contrib/mp-buildbot/functions
> • contrib/mp-buildbot/tools/canonical_variants.tcl
> Diff
>
> Modified: contrib/mp-buildbot/README.md (152580 => 152581)
>
> --- contrib/mp-buildbot/README.md 2016-09-12 19:13:35 UTC (rev 152580)
> +++ contrib/mp-buildbot/README.md 2016-09-12 19:14:24 UTC (rev 152581)
>
> @@ -91,3 +91,6 @@
>
> persist for the duration of an `mpbb` run, so it may be used to
>
> share ancillary files (e.g., a Subversion checkout of the ports
>
> tree) between builds of different ports.
>
> +- `$option_failcache_dir`:
> + A directory for storing information about previously failed builds which
> + saves time because builds that are known to fail will not be attempted.
>
> Added: contrib/mp-buildbot/functions (0 => 152581)
>
> --- contrib/mp-buildbot/functions (rev 0)
> +++ contrib/mp-buildbot/functions 2016-09-12 19:14:24 UTC (rev 152581)
>
> @@ -0,0 +1,134 @@
>
> +#!/bin/bash
> +# -*- coding: utf-8; mode: sh; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=sh:et:sw=4:ts=4:sts=4
> +
> +# Helper functions for mp-buildbot
> +
> +# Print $0 and arguments to standard error.
> +# Unset IFS to ensure "$*" uses spaces as separators.
> +msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
> +err() { msg error: "$@"; }
> +warn() { msg warning: "$@"; }
> +
> +## Compute a failcache hash for the given port
> +#
> +# Computes and prints a hash uniquely identifying a specific state of a port's
> +# definition files, including the Portfile's hash as well as the port's
> +# patchfiles. To build the hash, this function executes the following
> +# algorithm:
> +# - For the Portfile, and each file in files/ (if any), calculate a SHA256
> +# hash
> +# - Sort the hash values alphabetically
> +# - Hash the result using SHA256
> +# This means a failcache entry will not match if a patchfile changes. A common
> +# case where this is the desired behavior is a port committed without
> +# a required patchfile.
> +#
> +# Valid arguments are all arguments accepted by "port dir".
> +compute_failcache_hash() {
> + local portdir
> + local -a filelist
> +
> + portdir=$("${option_prefix}/bin/port" dir "$@")
> + if [ $? -ne 0 ] || [ -z "$portdir" ]; then
> + err "Could not compute failcache hash: port dir" "$@" "failed"
> + return 1
> + fi
> +
> + if [ ! -d "$portdir" ]; then
> + err "Port directory $portdir does not exist"
> + return 2
> + fi
> +
> + filelist=("$portdir/Portfile")
> + if [ -d "$portdir/files" ]; then
> + filelist+=("$portdir/files")
> + fi
> +
> + find "${filelist[@]}" -type f -exec openssl dgst -sha256 {} \; |\
> + cut -d' ' -f2 |\
> + sort |\
> + openssl dgst -sha256 |\
> + cut -d' ' -f2
> +}
> +
> +## Compute a key that uniquely identifies a (port, variants, portfile-hash) tuple
> +#
> +# Valid arguments are a port name, optionally followed by a variant
> +# specification. Invokes "port dir" to find the Portfile and patchfiles and
> +# computes a checksum of these files that will become part of the hash.
> +failcache_key() {
> + local port=$1
> + if [ -z "$port" ]; then
> + err "failcache_key expects a port argument, but none was given."
> + return 1
> + fi
> +
> + local checksum
> + checksum=$(compute_failcache_hash "$port")
> + if [ $? -ne 0 ]; then
> + err "compute_failcache_hash $port failed"
> + return 2
> + fi
> +
> + local canonical_variants
> + canonical_variants=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/canonical_variants.tcl" "$@")
> + if [ $? -ne 0 ]; then
> + err "tools/canonical_variants.tcl" "$@" "failed"
> + return 4
> + fi
> +
> + echo "$port $canonical_variants $checksum"
> +}
> +
> +## Test whether a given port with variants has previously failed.
> +#
> +# Valid arguments are a port name, optionally followed by a variant
> +# specification. Succeeds if the port did not previsouly fail to build,
> +# fails if the port is known to fail.
> +failcache_test() {
> + local key
> + key=$(failcache_key "$@")
> + if [ $? -ne 0 ]; then
> + err "Could not determine failcache key for" "$@"
> + return 1
> + fi
> +
> + if [ -f "${option_failcache_dir}/${key}" ]; then
> + printf "port %s previously failed in build %s\n" "${key}" "$(<"${option_failcache_dir}/${key}")"
> + return 1
> + else
> + return 0
> + fi
> +}
> +
> +## Mark a build of a given port with variants as successful.
> +#
> +# Valid arguments are a port name, optionally followed by a variant
> +# specification. Removes any database entries that marked a port as failed.
> +failcache_success() {
> + local key
> + key=$(failcache_key "$@")
> + if [ $? -ne 0 ]; then
> + err "Could not determine failcache key for" "$@"
> + return 1
> + fi
> +
> + rm -f "${option_failcache_dir}/${key}"
> +}
> +
> +## Mark a build of a given port with variants as failed.
> +#
> +# Valid arguments are a port name, optionally followed by a variant
> +# specification. Creates or updates the timestamp of a database entry that
> +# marks a port as failed.
> +failcache_failure() {
> + local key
> + key=$(failcache_key "$@")
> + if [ $? -ne 0 ]; then
> + err "Could not determine failcache key for" "$@"
> + return 1
> + fi
> +
> + mkdir -p "${option_failcache_dir}"
> + echo "${BUILDBOT_BUILDURL:-unknown}" > "${option_failcache_dir}/${key}"
> +}
>
> Modified: contrib/mp-buildbot/mpbb (152580 => 152581)
>
> --- contrib/mp-buildbot/mpbb 2016-09-12 19:13:35 UTC (rev 152580)
> +++ contrib/mp-buildbot/mpbb 2016-09-12 19:14:24 UTC (rev 152581)
>
> @@ -8,11 +8,10 @@
>
> # Don't inherit any option variables from the calling environment.
>
> unset "${!option_@}"
>
>
>
> -# Print $0 and arguments to standard error.
> -# Unset IFS to ensure "$*" uses spaces as separators.
> -msg() (unset IFS; printf >&2 '%s: %s\n' "$0" "$*")
> -err() { msg error: "$@"; }
> -warn() { msg warning: "$@"; }
>
> +# Load function library
> +thisdir=$(cd "$(dirname "$0")" && pwd)
> +# shellcheck source=functions
> +. "$thisdir/functions"
>
>
>
> mpbb-usage() {
>
> # "prog" is defined in mpbb-help.
>
> @@ -126,6 +125,7 @@
>
> option_jobs_dir=${option_work_dir}/infrastructure/jobs
>
> option_log_dir=${option_work_dir}/logs
>
> }
>
> +option_failcache_dir=${option_work_dir}/failcache
>
>
>
> # Inform the user if old repositories are still present.
>
> if [[ -d ${option_work_dir}/tools/.svn ]]; then
>
> @@ -146,7 +146,6 @@
>
> # must define functions "FOO" and "FOO-usage".
>
> cmds=()
>
> usages=(mpbb-usage)
>
> -thisdir=$(cd "$(dirname "$0")" && pwd)
>
> for cmdfile in "$thisdir/mpbb-"*; do
>
> # Unfortunately ShellCheck does not currently support following multiple
>
> # files, so we'll just disable the warning.
>
> Modified: contrib/mp-buildbot/mpbb-install-dependencies (152580 => 152581)
>
> --- contrib/mp-buildbot/mpbb-install-dependencies 2016-09-12 19:13:35 UTC (rev 152580)
> +++ contrib/mp-buildbot/mpbb-install-dependencies 2016-09-12 19:14:24 UTC (rev 152581)
>
> @@ -41,6 +41,9 @@
>
> local dependencies
>
> local dependencies_count
>
> local dependencies_counter
>
> + local depname
> + local depvariants
> + local failcachecounter
>
> # $option_log_dir is set in mpbb
>
> # shellcheck disable=SC2154
>
> local log_status_dependencies="${option_log_dir}/dependencies-progress.txt"
>
> @@ -53,7 +56,7 @@
>
> # calculate list of dependencies in-order
>
> # $option_prefix and $thisdir are set in mpbb
>
> # shellcheck disable=SC2154
>
> - dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port")
>
> + dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@")
>
> if [ $? -ne 0 ]; then
>
> echo "Calculating dependencies for '$port' failed, aborting." >&2
>
> echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
>
> @@ -72,26 +75,67 @@
>
> echo "$dependencies" | sed -E 's/^/ - /' | tee -a "$log_status_dependencies"
>
> echo >> "$log_status_dependencies"
>
>
>
> - echo "$dependencies" | while read -r dependency; do
>
> + # Check whether any of the dependencies have previously failed
> + failcachecounter=0
> + while read -r dependency; do
>
> # Split portname +variant1+variant2 into portname and variants, where
>
> # the variants are optional.
>
> depname=${dependency%% *}
>
> depvariants=${dependency:${#depname}+1}
>
>
>
> + # $depvariants isn't quoted on purpose
> + # shellcheck disable=SC2086
> + if ! failcache_test "$depname" $depvariants; then
> + text="Dependency '${depname}' with variants '${depvariants}' has previously failed and is required."
> + echo "$text" >&2
> + echo "$text" >> "$log_status_dependencies"
> + echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
> + failcachecounter=$((failcachecounter + 1))
> + fi
> + done <<<"$dependencies"
> +
> + if [ $failcachecounter -gt 0 ]; then
> + echo "Aborting build because $failcachecounter dependencies are known to fail." >&2
> + return 1
> + fi
> +
> + while read -r dependency; do
> + # Split portname +variant1+variant2 into portname and variants, where
> + # the variants are optional.
> + depname=${dependency%% *}
> + depvariants=${dependency:${#depname}+1}
> +
>
> text="Installing dependency ($dependencies_counter of $dependencies_count) '${depname}' with variants '${depvariants}'"
>
> echo "----> ${text}"
>
> echo -n "${text} ... " >> "$log_status_dependencies"
>
> # $depvariants isn't quoted on purpose
>
> # shellcheck disable=SC2086
>
> if ! "${option_prefix}/bin/port" -d install --unrequested "$depname" $depvariants; then
>
> - echo "Build of dependency '${depname}' failed, aborting." >&2
>
> + echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2
>
> echo "[FAIL]" >> "$log_status_dependencies"
>
> echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress"
>
> +
> + # Update failcache
> + # $depvariants isn't quoted on purpose
> + # shellcheck disable=SC2086
> + failcache_failure "$depname" $depvariants
> + if [ $? -ne 0 ]; then
> + err "failcache_failure $depname $depvariants failed."
> + return 1
> + fi
>
> return 1
>
> else
>
> echo "[OK]" >> "$log_status_dependencies"
>
> + # Remove failcache if it exists
> + # $depvariants isn't quoted on purpose
> + # shellcheck disable=SC2086
> + failcache_success "$depname" $depvariants
> + if [ $? -ne 0 ]; then
> + err "failcache_success $depname $depvariants failed."
> + return 1
> + fi
>
> dependencies_counter=$((dependencies_counter + 1))
>
> fi
>
> - done
>
> + done <<<"$dependencies"
>
> }
>
>
>
> Modified: contrib/mp-buildbot/mpbb-install-port (152580 => 152581)
>
> --- contrib/mp-buildbot/mpbb-install-port 2016-09-12 19:13:35 UTC (rev 152580)
> +++ contrib/mp-buildbot/mpbb-install-port 2016-09-12 19:14:24 UTC (rev 152581)
>
> @@ -55,10 +55,23 @@
>
> time_start=$(date +%s)
>
> # $option_prefix is set in mpbb
>
> # shellcheck disable=SC2154
>
> - if ! "${option_prefix}/bin/port" -dk install "$port"; then
>
> + if "${option_prefix}/bin/port" -dk install "$@"; then
> + # Remove failcache if it exists
> + failcache_success "$@"
> + if [ $? -ne 0 ]; then
> + err "failcache_success" "$@" "failed."
> + return 1
> + fi
> + else
>
> echo "Build of '$port' failed."
>
> # log: summary for the portwatcher
>
> echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress"
>
> + # update failcache
> + failcache_failure "$@"
> + if [ $? -ne 0 ]; then
> + err "failcache_failure" "$@" "failed."
> + return 1
> + fi
>
> return 1
>
> fi
>
> time_stop=$(date +%s)
>
> Added: contrib/mp-buildbot/tools/canonical_variants.tcl (0 => 152581)
>
> --- contrib/mp-buildbot/tools/canonical_variants.tcl (rev 0)
> +++ contrib/mp-buildbot/tools/canonical_variants.tcl 2016-09-12 19:14:24 UTC (rev 152581)
>
> @@ -0,0 +1,97 @@
>
> +#!/usr/bin/env port-tclsh
> +# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4
> +# Writes the canonical list of variants as it appears in binary archive names
> +# to stdout.
> +#
> +# Copyright (c) 2016 The MacPorts Project.
> +# Copyright (c) 2016 Clemens Lang <
> cal at macports.org
> >
> +#
> +# Redistribution and use in source and binary forms, with or without
> +# modification, are permitted provided that the following conditions
> +# are met:
> +# 1. Redistributions of source code must retain the above copyright
> +# notice, this list of conditions and the following disclaimer.
> +# 2. Redistributions in binary form must reproduce the above copyright
> +# notice, this list of conditions and the following disclaimer in
> +# the documentation and/or other materials provided with the
> +# distribution.
> +# 3. Neither the name of the MacPorts project, nor the names of any contributors
> +# may be used to endorse or promote products derived from this software
> +# without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
> +# AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
> +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
> +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
> +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
> +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
> +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
> +# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
> +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
> +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
> +# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +package require macports
> +
> +if {[llength $::argv] == 0} {
> + puts stderr "Usage: $argv0 <portname> \[(+|-)variant...\]"
> + exit 1
> +}
> +
> +# initialize macports
> +if {[catch {mportinit "" "" ""} result]} {
> + ui_error "$errorInfo"
> + ui_error "Failed to initialize ports sytem: $result"
> + exit 1
> +}
> +
> +# look up the path of the Portfile for the given port
> +set portname [lindex $::argv 0]
> +#try -pass_signal {...}
> +try {
> + set result [mportlookup $portname]
> + if {[llength $result] < 2} {
> + ui_error "No such port: $portname"
> + exit 1
> + }
> +} catch {{*} eCode eMessage} {
> + ui_error "mportlookup $portname failed: $eMessage"
> + exit 1
> +}
> +
> +# parse the given variants from the command line
> +array set variants {}
> +foreach item [lrange $::argv 1 end] {
> + foreach {_ sign variant} [regexp -all -inline -- {([-+])([[:alpha:]_]+[\w\.]*)} $item] {
> + set variants($variant) $sign
> + }
> +}
> +
> +# open the port to get its active variants
> +array set portinfo [lindex $result 1]
> +#try -pass_signal {...}
> +try {
> + set mport [mportopen $portinfo(porturl) [list subport $portname] [array get variants]]
> +} catch {{*} eCode eMessage} {
> + ui_error "mportopen ${portinfo(porturl)} failed: $eMessage"
> + exit 1
> +}
> +
> +array set info [mportinfo $mport]
> +puts $info(canonical_active_variants)
> +
> +#try -pass_signal {...}
> +try {
> + mportclose $mport
> +} catch {{*} eCode eMessage} {
> + ui_warn "mportclose $portname failed: $eMessage"
> +}
> +
> +# shut down MacPorts
> +#try -pass_signal {...}
> +try {
> + mportshutdown
> +} catch {{*} eCode eMessage} {
> + ui_error "mportshutdown failed: $eMessage"
> + exit 1
> +}
>
> Property changes on: contrib/mp-buildbot/tools/canonical_variants.tcl
>
> ___________________________________________________________________
>
> Added: svn:executable
> Added: svn:keywords
> Added: svn:eol-style
> _______________________________________________
> macports-changes mailing list
> macports-changes at lists.macosforge.org
> https://lists.macosforge.org/mailman/listinfo/macports-changes
More information about the macports-dev
mailing list