<pre style='margin:0'>
Joshua Root (jmroot) pushed a commit to branch remote-db
in repository mpbb.
</pre>
<p><a href="https://github.com/macports/mpbb/commit/0fe4e799c649029517945693896240f452ff4330">https://github.com/macports/mpbb/commit/0fe4e799c649029517945693896240f452ff4330</a></p>
<pre style="white-space: pre; background: #F8F8F8"><span style='display:block; white-space:pre;color:#808000;'>commit 0fe4e799c649029517945693896240f452ff4330
</span>Author: Joshua Root <jmr@macports.org>
AuthorDate: Tue Mar 11 10:59:33 2025 +1100
<span style='display:block; white-space:pre;color:#404040;'> mirror-multi: add remote db support
</span>---
tools/mirror-multi.tcl | 133 +++++++++++++++++++++++++++++++++++++---------
tools/update-mirrordb.tcl | 24 +++++++++
2 files changed, 131 insertions(+), 26 deletions(-)
<span style='display:block; white-space:pre;color:#808080;'>diff --git a/tools/mirror-multi.tcl b/tools/mirror-multi.tcl
</span><span style='display:block; white-space:pre;color:#808080;'>index ba0747e..e6b665f 100755
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>--- a/tools/mirror-multi.tcl
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>+++ b/tools/mirror-multi.tcl
</span><span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -61,8 +61,32 @@ set deptypes [list depends_fetch depends_extract depends_patch depends_build dep
</span> set processed [dict create]
set mirror_done [dict create]
set distfiles_results [dict create]
<span style='display:block; white-space:pre;background:#e0ffe0;'>+set portname_portfile_map [dict create]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+set portfile_hash_cache [dict create]
</span>
<span style='display:block; white-space:pre;background:#ffe0e0;'>-proc check_mirror_done {portname} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc get_portfile_hash {portname} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global portfile_hash_cache portname_portfile_map
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[dict exists $portname_portfile_map $portname]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile [dict get $portname_portfile_map $portname]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } else {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set result [mportlookup $portname]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[llength $result] < 2} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return {}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portinfo [lindex $result 1]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile [file join [macports::getportdir [dict get $portinfo porturl]] Portfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[dict exists $portfile_hash_cache $portfile]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return [dict get $portfile_hash_cache $portfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } elseif {[file isfile $portfile]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile_hash [sha256 file $portfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set portfile_hash_cache $portfile $portfile_hash
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return $portfile_hash
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return {}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc check_mirror_done_local {portname} {
</span> global mirror_done
if {[dict exists $mirror_done $portname]} {
return [dict get $mirror_done $portname]
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -70,30 +94,58 @@ proc check_mirror_done {portname} {
</span> global mirrorcache_dir
set cache_entry [file join $mirrorcache_dir [string toupper [string index $portname 0]] $portname]
if {[file isfile $cache_entry]} {
<span style='display:block; white-space:pre;background:#ffe0e0;'>- set result [mportlookup $portname]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- if {[llength $result] < 2} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile_hash [get_portfile_hash $portname]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$portfile_hash eq {}} {
</span> return 0
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>- set portinfo [lindex $result 1]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set portfile [file join [macports::getportdir [dict get $portinfo porturl]] Portfile]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- if {[file isfile $portfile]} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set portfile_hash [sha256 file $portfile]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set fd [open $cache_entry]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set entry_hash [gets $fd]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set partial [gets $fd]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- close $fd
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- if {$portfile_hash eq $entry_hash} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- if {$partial eq ""} {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- dict set mirror_done $portname 1
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- return 1
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- } else {
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- dict set mirror_done $portname $partial
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- return $partial
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fd [open $cache_entry]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set entry_hash [gets $fd]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set partial [gets $fd]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ close $fd
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$portfile_hash eq $entry_hash} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$partial eq ""} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set mirror_done $portname 1
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return 1
</span> } else {
<span style='display:block; white-space:pre;background:#ffe0e0;'>- file delete -force $cache_entry
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- dict set mirror_done $portname 0
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set mirror_done $portname $partial
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return $partial
</span> }
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ } else {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ file delete -force $cache_entry
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set mirror_done $portname 0
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } else {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set mirror_done $portname 0
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return 0
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc get_remote_db_value {key} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global mirrorcache_baseurl mirrorcache_credentials
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fullurl ${mirrorcache_baseurl}GET/${key}?type=txt
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ try {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ curl fetch -u $mirrorcache_credentials $fullurl mirror_db_response
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fd [open mirror_db_response r]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ gets $fd result
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ close $fd
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } on error {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set result {}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ } finally {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ file delete -force mirror_db_response
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return $result
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc check_mirror_done_remote {portname} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global mirror_done
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[dict exists $mirror_done $portname]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return [dict get $mirror_done $portname]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[get_remote_db_value mirror.sha256.${portname}] eq [get_portfile_hash $portname]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set result [get_remote_db_value mirror.status.${portname}]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {$result ne {}} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set mirror_done $portname $result
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ return $result
</span> }
} else {
dict set mirror_done $portname 0
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -101,14 +153,32 @@ proc check_mirror_done {portname} {
</span> return 0
}
<span style='display:block; white-space:pre;background:#ffe0e0;'>-proc set_mirror_done {portname value} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc set_mirror_done_remote {portname value} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # We actually need to upload the files before updating their
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ # status in the remote db, so just update the dict here.
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global mirror_done
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {![dict exists $mirror_done $portname] || [dict get $mirror_done $portname] != 1} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ dict set mirror_done $portname $value
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# Write out info needed to update the remote db.
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc write_status_dicts {} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ foreach d {mirror_done portfile_hash_cache portname_portfile_map} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ global $d
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fd [open $d w]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ puts -nonewline $fd [set $d]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ close $fd
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+proc set_mirror_done_local {portname value} {
</span> global mirror_done
if {![dict exists $mirror_done $portname] || [dict get $mirror_done $portname] != 1} {
global mirrorcache_dir
<span style='display:block; white-space:pre;background:#ffe0e0;'>- set result [mportlookup $portname]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set portinfo [lindex $result 1]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set portfile [file join [macports::getportdir [dict get $portinfo porturl]] Portfile]
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>- set portfile_hash [sha256 file $portfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile_hash [get_portfile_hash $portname]
</span>
set cache_dir [file join $mirrorcache_dir [string toupper [string index $portname 0]]]
file mkdir $cache_dir
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -331,6 +401,14 @@ set mirrorcache_dir /tmp/mirrorcache
</span> if {[lindex $argv 0] eq "-c"} {
set mirrorcache_dir [lindex $argv 1]
set argv [lrange $argv 2 end]
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ rename check_mirror_done_local check_mirror_done
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ rename set_mirror_done_local set_mirror_done
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+} elseif {[lindex $argv 0] eq "-r"} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set mirrorcache_baseurl [lindex $argv 1]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set mirrorcache_credentials [lindex $argv 2]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set argv [lrange $argv 3 end]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ rename check_mirror_done_remote check_mirror_done
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ rename set_mirror_done_remote set_mirror_done
</span> }
set exitval 0
<span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -353,6 +431,9 @@ foreach portname $argv {
</span> if {[mirror_port [lindex $result 1]] != 0} {
set exitval 1
}
<span style='display:block; white-space:pre;background:#e0ffe0;'>+ if {[info exists mirrorcache_baseurl]} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ write_status_dicts
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ }
</span> }
exit $exitval
<span style='display:block; white-space:pre;color:#808080;'>diff --git a/tools/update-mirrordb.tcl b/tools/update-mirrordb.tcl
</span>new file mode 100755
<span style='display:block; white-space:pre;color:#808080;'>index 0000000..6dfe081
</span><span style='display:block; white-space:pre;background:#ffe0e0;'>--- /dev/null
</span><span style='display:block; white-space:pre;background:#e0e0ff;'>+++ b/tools/update-mirrordb.tcl
</span><span style='display:block; white-space:pre;background:#e0e0e0;'>@@ -0,0 +1,24 @@
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+#!/usr/bin/env port-tclsh
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# args:
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# mirrorcache_baseurl mirrorcache_credentials
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# then paths to three files containing tcl dicts:
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+# mirror_done portfile_hash_cache portname_portfile_map
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+package require Pextlib
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+lassign $argv mirrorcache_baseurl mirrorcache_credentials mirror_done_p portfile_hash_cache_p portname_portfile_map_p
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+foreach var {mirror_done portfile_hash_cache portname_portfile_map} {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set fd [open [set ${var}_p] r]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set $var [gets $fd]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ close $fd
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+dict for {portname status} $mirror_done {
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set portfile [dict get $portname_portfile_map $portname]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set hash [dict get $portfile_hash_cache $portfile]
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set hashurl ${mirrorcache_baseurl}SET/mirror.sha256.${portname}/${hash}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ set statusurl ${mirrorcache_baseurl}SET/mirror.status.${portname}/${status}
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ curl fetch -u $mirrorcache_credentials $hashurl /dev/null
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+ curl fetch -u $mirrorcache_credentials $statusurl /dev/null
</span><span style='display:block; white-space:pre;background:#e0ffe0;'>+}
</span></pre><pre style='margin:0'>
</pre>