<pre style='margin:0'>
Joshua Root (jmroot) pushed a commit to branch master
in repository mpbb.
</pre>
<p><a href="https://github.com/macports/mpbb/commit/bdea90b82f3fa0c91da3dc9522140daab275d9ca">https://github.com/macports/mpbb/commit/bdea90b82f3fa0c91da3dc9522140daab275d9ca</a></p>
<pre style="white-space: pre; background: #F8F8F8">The following commit(s) were added to refs/heads/master by this push:
<span style='display:block; white-space:pre;color:#404040;'> new bdea90b mirror-multi.tcl: don't repeatedly mirror distfiles
</span>bdea90b is described below
<span style='display:block; white-space:pre;color:#808000;'>commit bdea90b82f3fa0c91da3dc9522140daab275d9ca
</span>Author: Joshua Root <jmr@macports.org>
AuthorDate: Tue Dec 28 18:49:43 2021 +1100
<span style='display:block; white-space:pre;color:#404040;'> mirror-multi.tcl: don't repeatedly mirror distfiles
</span><span style='display:block; white-space:pre;color:#404040;'>
</span><span style='display:block; white-space:pre;color:#404040;'> Skip mirroring if any of the port's distfiles already had a checksum
</span><span style='display:block; white-space:pre;color:#404040;'> mismatch this run. Also skip if all of the port's distfiles were
</span><span style='display:block; white-space:pre;color:#404040;'> already successfully mirrored, since that information becomes readily
</span><span style='display:block; white-space:pre;color:#404040;'> available in solving the aforementioned problem.
</span><span style='display:block; white-space:pre;color:#404040;'>
</span><span style='display:block; white-space:pre;color:#404040;'> Fixes: https://trac.macports.org/ticket/57343
</span>---
tools/mirror-multi.tcl | 93 ++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 79 insertions(+), 14 deletions(-)
<span style='display:block; white-space:pre;color:#808080;'>diff --git a/tools/mirror-multi.tcl b/tools/mirror-multi.tcl
</span><span style='display:block; white-space:pre;color:#808080;'>index 27d9ceb..f5b8ee6 100755
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>--- a/tools/mirror-multi.tcl
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>+++ b/tools/mirror-multi.tcl
</span><span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -53,8 +53,9 @@ foreach vers {20 21} {
</span> }
set deptypes {depends_fetch depends_extract depends_build depends_lib depends_run depends_test}
<span style='display:block; white-space:pre;background:#ffe0e0;'>-array set processed {}
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>-array set mirror_done {}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+array set processed [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+array set mirror_done [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+array set distfiles_results [list]
</span>
proc check_mirror_done {portname} {
global mirror_done mirrorcache_dir
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -151,19 +152,77 @@ proc get_variants {portinfovar} {
</span> # work around the bug where the mirror target claims to succeed when
# the distfile checksums did not match
proc check_distfiles {mport} {
<span style='display:block; white-space:pre;background:#ffe0e0;'>- set distpath [_mportkey $mport distpath]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global distfiles_results
</span> if {[catch {_mportkey $mport all_dist_files} all_dist_files]} {
# no distfiles, no problem
return 0
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distpath [_mportkey $mport distpath]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set ret 0
</span> foreach distfile $all_dist_files {
<span style='display:block; white-space:pre;background:#ffe0e0;'>- if {![file exists [file join $distpath $distfile]]} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- return 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set filepath [file join $distpath $distfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[file exists $filepath]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfiles_results($filepath) 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } else {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfiles_results($filepath) 0
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set ret 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return $ret
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# Given a distribution file name, return the name without an attached tag
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# Example : getdistname distfile.tar.gz:tag1 returns "distfile.tar.gz"
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# / isn't included in the regexp, thus allowing port specification in URLs.
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc getdistname {name} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ regexp {(.+):[0-9A-Za-z_-]+$} $name match name
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return $name
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# check if mirroring should be skipped due to all distfiles having
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# previously been successfully mirrored, or any distfile previously
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# having a checksum mismatch
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# Returns:
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# 0 - mirror needed
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# 1 - mirror not needed
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# 2 - mirror already failed for at least one distfile
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc skip_mirror {mport identifier} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {([catch {_mportkey $mport distfiles} distfiles] || $distfiles eq "")
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ && ([catch {_mportkey $mport patchfiles} patchfiles] || $patchfiles eq "")} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # no distfiles, no need to mirror
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {![info exists distfiles]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfiles [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {![info exists patchfiles]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set patchfiles [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global distfiles_results
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distpath [_mportkey $mport distpath]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set filespath [_mportkey $mport filespath]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set any_unmirrored 0
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ foreach distfile [concat $distfiles $patchfiles] {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[file exists [file join $filespath $distfile]]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ continue
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfile [getdistname $distfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set filepath [file join $distpath $distfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {![info exists distfiles_results($filepath)]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set any_unmirrored 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } elseif {$distfiles_results($filepath) == 0} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ ui_msg "Skipping ${identifier}: $distfile already failed checksum"
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return 2
</span> }
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$any_unmirrored == 0} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ ui_msg "Skipping ${identifier}: all distfiles already mirrored"
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span> return 0
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span> proc mirror_port {portinfo_list} {
global platforms deptypes processed
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -185,12 +244,16 @@ proc mirror_port {portinfo_list} {
</span> array unset portinfo
array set portinfo [mportinfo $mport]
<span style='display:block; white-space:pre;background:#ffe0e0;'>- if {$do_mirror} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set skip_result [skip_mirror $mport $portname]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$do_mirror && $skip_result == 0} {
</span> incr attempted
mportexec $mport clean
if {[mportexec $mport mirror] == 0 && [check_distfiles $mport] == 0} {
incr succeeded
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ } elseif {$skip_result == 2} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # count as a failure
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ incr attempted
</span> }
mportclose $mport
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -199,7 +262,6 @@ proc mirror_port {portinfo_list} {
</span>
foreach variant $variants {
ui_msg "$portname +${variant}"
<span style='display:block; white-space:pre;background:#ffe0e0;'>- incr attempted
</span> if {[catch {mportopen $porturl [list subport $portname] [list $variant +]} mport]} {
ui_error "mportopen $porturl failed: $mport"
continue
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -207,20 +269,21 @@ proc mirror_port {portinfo_list} {
</span> array unset portinfo
array set portinfo [mportinfo $mport]
lappend deps {*}[get_dep_list portinfo]
<span style='display:block; white-space:pre;background:#ffe0e0;'>- if {$do_mirror} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set skip_result [skip_mirror $mport "$portname +${variant}"]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$do_mirror && $skip_result == 0} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ incr attempted
</span> mportexec $mport clean
if {[mportexec $mport mirror] == 0 && [check_distfiles $mport] == 0} {
incr succeeded
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>- } else {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- incr succeeded
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } elseif {$skip_result == 2} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ incr attempted
</span> }
mportclose $mport
}
foreach {os_major os_arch} $platforms {
ui_msg "$portname with platform 'darwin $os_major $os_arch'"
<span style='display:block; white-space:pre;background:#ffe0e0;'>- incr attempted
</span> if {[catch {mportopen $porturl [list subport $portname os_major $os_major os_arch $os_arch] {}} mport]} {
ui_error "mportopen $porturl failed: $mport"
continue
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -228,13 +291,15 @@ proc mirror_port {portinfo_list} {
</span> array unset portinfo
array set portinfo [mportinfo $mport]
lappend deps {*}[get_dep_list portinfo]
<span style='display:block; white-space:pre;background:#ffe0e0;'>- if {$do_mirror} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set skip_result [skip_mirror $mport "$portname darwin $os_major $os_arch"]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$do_mirror && $skip_result == 0} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ incr attempted
</span> mportexec $mport clean
if {[mportexec $mport mirror] == 0 && [check_distfiles $mport] == 0} {
incr succeeded
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>- } else {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- incr succeeded
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } elseif {$skip_result == 2} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ incr attempted
</span> }
mportclose $mport
}
</pre><pre style='margin:0'>
</pre>