<pre style='margin:0'>
Joshua Root (jmroot) pushed a commit to branch master
in repository macports-base.
</pre>
<p><a href="https://github.com/macports/macports-base/commit/678921f1f76042f8fe354b2cce0c20c0b285f5fc">https://github.com/macports/macports-base/commit/678921f1f76042f8fe354b2cce0c20c0b285f5fc</a></p>
<pre style="white-space: pre; background: #F8F8F8">The following commit(s) were added to refs/heads/master by this push:
<span style='display:block; white-space:pre;color:#404040;'> new 678921f1 port reclaim: persistently cache distfile info
</span>678921f1 is described below
<span style='display:block; white-space:pre;color:#808000;'>commit 678921f1f76042f8fe354b2cce0c20c0b285f5fc
</span>Author: Joshua Root <jmr@macports.org>
AuthorDate: Thu Feb 24 03:32:04 2022 +1100
<span style='display:block; white-space:pre;color:#404040;'> port reclaim: persistently cache distfile info
</span><span style='display:block; white-space:pre;color:#404040;'>
</span><span style='display:block; white-space:pre;color:#404040;'> The cache is simply a Tcl array mapping each port name + variants to a
</span><span style='display:block; white-space:pre;color:#404040;'> dist_subdir and a list of files, plus a fingerprint (consisting of the
</span><span style='display:block; white-space:pre;color:#404040;'> version, revision and Portfile sha256 hash) so the entries can be
</span><span style='display:block; white-space:pre;color:#404040;'> refreshed when the ports change.
</span><span style='display:block; white-space:pre;color:#404040;'>
</span><span style='display:block; white-space:pre;color:#404040;'> The cache is also invalidated if the global variations (i.e. from
</span><span style='display:block; white-space:pre;color:#404040;'> variants.conf) change. It may be possible to avoid that with a bit of
</span><span style='display:block; white-space:pre;color:#404040;'> extra cleverness.
</span>---
src/macports1.0/reclaim.tcl | 121 +++++++++++++++++++++++++++++++++-----------
1 file changed, 91 insertions(+), 30 deletions(-)
<span style='display:block; white-space:pre;color:#808080;'>diff --git a/src/macports1.0/reclaim.tcl b/src/macports1.0/reclaim.tcl
</span><span style='display:block; white-space:pre;color:#808080;'>index fdc21b10..8c96f019 100644
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>--- a/src/macports1.0/reclaim.tcl
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>+++ b/src/macports1.0/reclaim.tcl
</span><span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -238,6 +238,40 @@ namespace eval reclaim {
</span> return [array get varray]
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ proc load_distfile_cache {varname} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ upvar $varname var
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ try -pass_signal {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fd [open |[list $macports::autoconf::gzip_path -d < [file join $macports::portdbpath reclaim distfiles.gz]] r]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set data [gets $fd]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ close $fd
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array set var $data
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$var(:global_variations) ne [array get macports::global_variations]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array unset var
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array set var [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } else {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ unset var(:global_variations)
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } catch {{*} eCode eMessage} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ ui_debug "Failed to load distfiles cache: $eMessage"
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array set var [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ catch {close $fd}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ proc save_distfile_cache {varname} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ upvar $varname var
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ try -pass_signal {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ file mkdir [file join $macports::portdbpath reclaim]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fd [open |[list $macports::autoconf::gzip_path > [file join $macports::portdbpath reclaim distfiles.gz]] w]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set var(:global_variations) [array get macports::global_variations]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ puts $fd [array get var]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ close $fd
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } catch {{*} eCode eMessage} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ ui_debug "Failed to save distfiles cache: $eMessage"
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ catch {close $fd}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span> proc remove_distfiles {} {
# Check for distfiles in both the root, and home directories. If found, delete them.
# Args:
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -266,6 +300,7 @@ namespace eval reclaim {
</span> }
ui_msg "$macports::ui_prefix Building list of distfiles still in use"
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ load_distfile_cache distfile_cache_prev
</span> set installed_ports [registry::entry imaged]
set port_count [llength $installed_ports]
set i 1
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -273,41 +308,68 @@ namespace eval reclaim {
</span>
foreach port $installed_ports {
# skip additional versions installed with the same variants
<span style='display:block; white-space:pre;background:#ffe0e0;'>- if {[info exists seen([$port name],[$port requested_variants])]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set cache_key [$port name],[$port requested_variants]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[info exists distfile_cache_new($cache_key)]} {
</span> continue
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>- set seen([$port name],[$port requested_variants]) 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array unset cacheinfo
</span> array unset portinfo
<span style='display:block; white-space:pre;background:#ffe0e0;'>- # Get mport reference
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- try -pass_signal {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- if {[catch {mportlookup [$port name]} lookup_result] || [llength $lookup_result] < 2} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- ui_warn [msgcat::mc "Port %s not found: %s" [$port name] $lookup_result]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- continue
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- }
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- array set portinfo [lindex $lookup_result 1]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set mport [mportopen $portinfo(porturl) [list subport $portinfo(name)] [get_variations [$port requested_variants]]]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- } catch {{*} eCode eMessage} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- $progress intermission
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- ui_warn [msgcat::mc "Failed to open port %s %s: %s" [$port name] [$port requested_variants] $eMessage]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- #registry::entry close $port
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[catch {mportlookup [$port name]} lookup_result] || [llength $lookup_result] < 2} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ ui_warn [msgcat::mc "Port %s not found: %s" [$port name] $lookup_result]
</span> continue
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ array set portinfo [lindex $lookup_result 1]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set parse_needed yes
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile_hash [sha256 file [file join [macports::getportdir $portinfo(porturl)] Portfile]]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fingerprint $portinfo(version)_$portinfo(revision)_${portfile_hash}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[info exists distfile_cache_prev($cache_key)]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array set cacheinfo $distfile_cache_prev($cache_key)
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$cacheinfo(fingerprint) eq $fingerprint} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set parse_needed no
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfile_cache_new($cache_key) $distfile_cache_prev($cache_key)
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span>
<span style='display:block; white-space:pre;background:#ffe0e0;'>- # Get sub-Tcl-interpreter that executed the installed port
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set workername [ditem_key $mport workername]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$parse_needed} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set cacheinfo(fingerprint) $fingerprint
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # Get mport reference
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ try -pass_signal {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set mport [mportopen $portinfo(porturl) [list subport $portinfo(name)] [get_variations [$port requested_variants]]]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } catch {{*} eCode eMessage} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ $progress intermission
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ ui_warn [msgcat::mc "Failed to open port %s %s: %s" [$port name] [$port requested_variants] $eMessage]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ #registry::entry close $port
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ continue
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # Get sub-Tcl-interpreter that executed the installed port
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set workername [ditem_key $mport workername]
</span>
<span style='display:block; white-space:pre;background:#ffe0e0;'>- # Append that port's distfiles to the list
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set dist_subdir [$workername eval {set dist_subdir}]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set distfiles [$workername eval {set distfiles}]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- if {[catch {$workername eval {set patchfiles}} patchfiles]} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set patchfiles {}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # Append that port's distfiles to the list
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set cacheinfo(dist_subdir) [$workername eval {set dist_subdir}]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfiles [$workername eval {set distfiles}]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[catch {$workername eval {set patchfiles}} patchfiles]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set patchfiles [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set filespath [$workername eval {set filespath}]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set cacheinfo(distfiles) [list]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ foreach file [concat $distfiles $patchfiles] {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # get filename without any tag
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfile [$workername eval [list getdistname $file]]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {![file exists [file join $filespath $distfile]]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ lappend cacheinfo(distfiles) $distfile
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set distfile_cache_new($cache_key) [array get cacheinfo]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ mportclose $mport
</span> }
<span style='display:block; white-space:pre;background:#ffe0e0;'>- foreach file [concat $distfiles $patchfiles] {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- # split distfile into filename and disttag
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set distfile [$workername eval [list getdistname $file]]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set root_path [file join $root_dist $dist_subdir $distfile]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set home_path [file join $home_dist $dist_subdir $distfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ foreach distfile $cacheinfo(distfiles) {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set root_path [file join $root_dist $cacheinfo(dist_subdir) $distfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set home_path [file join $home_dist $cacheinfo(dist_subdir) $distfile]
</span>
# Add the full file path to the list, depending where it's located.
if {[file isfile $root_path]} {
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -320,14 +382,13 @@ namespace eval reclaim {
</span> }
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>- mportclose $mport
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>-
</span> $progress update $i $port_count
#registry::entry close $port
incr i
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>- array unset seen
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- array unset portinfo
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array unset distfile_cache_prev
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ save_distfile_cache distfile_cache_new
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ array unset distfile_cache_new
</span>
$progress finish
</pre><pre style='margin:0'>
</pre>