[53051] branches/images-and-archives/base
blb at macports.org
blb at macports.org
Sun Jun 28 14:26:51 PDT 2009
Revision: 53051
http://trac.macports.org/changeset/53051
Author: blb at macports.org
Date: 2009-06-28 14:26:50 -0700 (Sun, 28 Jun 2009)
Log Message:
-----------
Merge from trunk
Modified Paths:
--------------
branches/images-and-archives/base/portmgr/jobs/PortIndex2MySQL.tcl
branches/images-and-archives/base/src/pextlib1.0/curl.c
branches/images-and-archives/base/src/port1.0/portdistcheck.tcl
branches/images-and-archives/base/src/port1.0/portdistfiles.tcl
branches/images-and-archives/base/src/port1.0/portfetch.tcl
branches/images-and-archives/base/src/registry1.0/receipt_flat.tcl
Removed Paths:
-------------
branches/images-and-archives/base/portmgr/bots/
Property Changed:
----------------
branches/images-and-archives/base/
Property changes on: branches/images-and-archives/base
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/gsoc08-privileges/base:37343-46937
/branches/universal-sanity/base:51872-52323
/branches/variant-descs-14482/base:34469-34855,34900-37508,37511-37512,41040-41463,42575-42626,42640-42659
/trunk/base:50249-52731
/users/perry/base-bugs_and_notes:45682-46060
/users/perry/base-select:44044-44692
+ /branches/gsoc08-privileges/base:37343-46937
/branches/universal-sanity/base:51872-52323
/branches/variant-descs-14482/base:34469-34855,34900-37508,37511-37512,41040-41463,42575-42626,42640-42659
/trunk/base:50249-53050
/users/perry/base-bugs_and_notes:45682-46060
/users/perry/base-select:44044-44692
Modified: branches/images-and-archives/base/portmgr/jobs/PortIndex2MySQL.tcl
===================================================================
--- branches/images-and-archives/base/portmgr/jobs/PortIndex2MySQL.tcl 2009-06-28 20:53:16 UTC (rev 53050)
+++ branches/images-and-archives/base/portmgr/jobs/PortIndex2MySQL.tcl 2009-06-28 21:26:50 UTC (rev 53051)
@@ -48,7 +48,7 @@
# meaningful, which is accomplished simply by calling the 'mportsync' proc in macports1.0
# (which updates the ports tree in use) and by installing the script on cron/launchd to be
# run on a timely schedule (not any more frequent than the run of the PortIndexRegen.sh
-# script on that creates a new PortIndex file, which is every twelve hours).
+# script on that creates a new PortIndex file).
#
# Remaining requirement to successfully run this script is performing the necessary
# MySQL admin tasks on the host box to create the database in the first place and the
@@ -72,8 +72,13 @@
set FROM macports-mgr at lists.macosforge.org
set HEADERS "To: $SPAM_LOVERS\r\nFrom: $FROM\r\nSubject: $SUBJECT\r\n\r\n"
-# We first initialize the runlog with proper mail headers
-puts $runlog_fd $HEADERS
+# handle command line arguments
+set create_tables false
+if {[llength $argv]} {
+ if {[lindex $argv 0] == "--create-tables"} {
+ set create_tables true
+ }
+}
# House keeping on exit.
proc cleanup {args} {
@@ -97,29 +102,6 @@
exit $exit_status
}
-# Check if there are any stray sibling jobs before moving on, bail in such case.
-if {[file exists $lockfile]} {
- puts $runlog_fd "PortIndex2MySQL lock file found, is another job running?"
- terminate 1
-} else {
- set lockfile_fd [open $lockfile a]
-}
-
-
-# Load macports1.0 so that we can use some of its procs and the portinfo array.
-if {[catch { source [file join "@TCL_PACKAGE_DIR@" macports1.0 macports_fastload.tcl] } errstr]} {
- puts $runlog_fd "${::errorInfo}"
- puts $runlog_fd "Failed to locate the macports1.0 Tcl package file: $errstr"
- cleanup lockfile
- terminate 1
-}
-if {[catch { package require macports } errstr]} {
- puts $runlog_fd "${::errorInfo}"
- puts $runlog_fd "Failed to load the macports1.0 Tcl package: $errstr"
- cleanup lockfile
- terminate 1
-}
-
# macports1.0 UI instantiation to route information/error messages wherever we want.
# This is a custom ui_channels proc because we want to get reported information on
# channels other than the default stdout/stderr that the macports1.0 API provides,
@@ -157,18 +139,6 @@
}
}
-# Initialize macports1.0 and its UI, in order to find the sources.conf file
-# (which is what will point us to the PortIndex we're gonna use) and use
-# the runtime information.
-array set ui_options {ports_verbose yes}
-if {[catch {mportinit ui_options} errstr]} {
- puts $runlog_fd "${::errorInfo}"
- puts $runlog_fd "Failed to initialize MacPorts: $errstr"
- cleanup lockfile
- terminate 1
-}
-
-
# Procedure to catch the database password from a protected file.
proc getpasswd {passwdfile} {
if {[catch {open $passwdfile r} passwdfile_fd]} {
@@ -188,6 +158,51 @@
return $passwd
}
+# SQL string escaping.
+proc sql_escape {str} {
+ regsub -all -- {'} $str {\\'} str
+ regsub -all -- {"} $str {\\"} str
+ regsub -all -- {\n} $str {\\n} str
+ return $str
+}
+
+# We first initialize the runlog with proper mail headers
+puts $runlog_fd $HEADERS
+
+# Check if there are any stray sibling jobs before moving on, bail in such case.
+if {[file exists $lockfile]} {
+ puts $runlog_fd "PortIndex2MySQL lock file found, is another job running?"
+ terminate 1
+} else {
+ set lockfile_fd [open $lockfile a]
+}
+
+# Load macports1.0 so that we can use some of its procs and the portinfo array.
+if {[catch { source [file join "/Library/Tcl" macports1.0 macports_fastload.tcl] } errstr]} {
+ puts $runlog_fd "${::errorInfo}"
+ puts $runlog_fd "Failed to locate the macports1.0 Tcl package file: $errstr"
+ cleanup lockfile
+ terminate 1
+}
+if {[catch { package require macports } errstr]} {
+ puts $runlog_fd "${::errorInfo}"
+ puts $runlog_fd "Failed to load the macports1.0 Tcl package: $errstr"
+ cleanup lockfile
+ terminate 1
+}
+
+# Initialize macports1.0 and its UI, in order to find the sources.conf file
+# (which is what will point us to the PortIndex we're gonna use) and use
+# the runtime information.
+array set ui_options {ports_verbose yes}
+if {[catch {mportinit ui_options} errstr]} {
+ puts $runlog_fd "${::errorInfo}"
+ puts $runlog_fd "Failed to initialize MacPorts: $errstr"
+ cleanup lockfile
+ terminate 1
+}
+
+
# Database abstraction variables:
set sqlfile "/tmp/portsdb.sql"
set portsdb_host localhost
@@ -222,39 +237,40 @@
terminate 1
}
-
-# SQL string escaping.
-proc sql_escape {str} {
- regsub -all -- {'} $str {\\'} str
- regsub -all -- {"} $str {\\"} str
- regsub -all -- {\n} $str {\\n} str
- return $str
+if {$create_tables} {
+ # Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms.
+ # Do we need any other?
+ puts $sqlfile_fd "DROP TABLE IF EXISTS log;"
+ puts $sqlfile_fd "CREATE TABLE log (activity VARCHAR(255), activity_time TIMESTAMP(14)) DEFAULT CHARSET=utf8;"
+
+ puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;"
+ puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255), description TEXT) DEFAULT CHARSET=utf8;"
+
+ puts $sqlfile_fd "DROP TABLE IF EXISTS categories;"
+ puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
+
+ puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;"
+ puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
+
+ puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;"
+ puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255)) DEFAULT CHARSET=utf8;"
+
+ puts $sqlfile_fd "DROP TABLE IF EXISTS variants;"
+ puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255)) DEFAULT CHARSET=utf8;"
+
+ puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;"
+ puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255)) DEFAULT CHARSET=utf8;"
+} else {
+ # if we are not creating tables from scratch, remove the old data
+ puts $sqlfile_fd "TRUNCATE log;"
+ puts $sqlfile_fd "TRUNCATE portfiles;"
+ puts $sqlfile_fd "TRUNCATE categories;"
+ puts $sqlfile_fd "TRUNCATE maintainers;"
+ puts $sqlfile_fd "TRUNCATE dependencies;"
+ puts $sqlfile_fd "TRUNCATE variants;"
+ puts $sqlfile_fd "TRUNCATE platforms;"
}
-
-# Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms.
-# Do we need any other?
-puts $sqlfile_fd "DROP TABLE IF EXISTS log;"
-puts $sqlfile_fd "CREATE TABLE log (activity VARCHAR(255), activity_time TIMESTAMP(14)) DEFAULT CHARSET=utf8;"
-
-puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;"
-puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255), description TEXT) DEFAULT CHARSET=utf8;"
-
-puts $sqlfile_fd "DROP TABLE IF EXISTS categories;"
-puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
-
-puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;"
-puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
-
-puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;"
-puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255)) DEFAULT CHARSET=utf8;"
-
-puts $sqlfile_fd "DROP TABLE IF EXISTS variants;"
-puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255)) DEFAULT CHARSET=utf8;"
-
-puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;"
-puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255)) DEFAULT CHARSET=utf8;"
-
-
+
# Iterate over each matching port, extracting its information from the
# portinfo array.
foreach {name array} $ports {
@@ -376,7 +392,6 @@
# Mark the db regen as done only once we're done processing all ports:
puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());"
-
# Pipe the contents of the generated sql file to the database command,
# reading from the file descriptor for the raw sql file to assure completeness.
if {[catch {seek $sqlfile_fd 0 start} errstr]} {
@@ -384,13 +399,13 @@
cleanup sqlfile lockfile
terminate 1
}
+
if {[catch {exec -- $portsdb_cmd --host=$portsdb_host --user=$portsdb_user --password=$portsdb_passwd --database=$portsdb_name <@ $sqlfile_fd} errstr]} {
ui_error "${::errorCode}: $errstr"
cleanup sqlfile lockfile
terminate 1
}
-
-# And we're done regen'ing the MacPorts dabase! Cleanup and exit successfully.
+# done regenerating the database. Cleanup and exit successfully.
cleanup sqlfile lockfile
terminate 0
Modified: branches/images-and-archives/base/src/pextlib1.0/curl.c
===================================================================
--- branches/images-and-archives/base/src/pextlib1.0/curl.c 2009-06-28 20:53:16 UTC (rev 53050)
+++ branches/images-and-archives/base/src/pextlib1.0/curl.c 2009-06-28 21:26:50 UTC (rev 53051)
@@ -54,9 +54,9 @@
* Some compiled-in constants that we may wish to change later, given more
* empirical data. These represent "best guess" values for now.
*/
-#define _CURL_CONNECTION_TIMEOUT ((long)(5 * 60)) /* 5 minutes */
+#define _CURL_CONNECTION_TIMEOUT ((long)(30)) /* 30 seconds */
#define _CURL_MINIMUM_XFER_SPEED ((long)1024) /* 1Kb/sec */
-#define _CURL_MINIMUM_XFER_TIMEOUT ((long)(10 * 60)) /* 10 minutes */
+#define _CURL_MINIMUM_XFER_TIMEOUT ((long)(60)) /* 1 minute */
/* ========================================================================= **
* Definitions
Modified: branches/images-and-archives/base/src/port1.0/portdistcheck.tcl
===================================================================
--- branches/images-and-archives/base/src/port1.0/portdistcheck.tcl 2009-06-28 20:53:16 UTC (rev 53050)
+++ branches/images-and-archives/base/src/port1.0/portdistcheck.tcl 2009-06-28 21:26:50 UTC (rev 53051)
@@ -64,21 +64,20 @@
if {"${distcheck.check}" != "none"
&& "${fetch.type}" == "standard"} {
# portfetch 1.0::checkfiles sets fetch_urls list.
- global fetch_urls
- checkfiles
+ set fetch_urls {}
+ portfetch::checkfiles fetch_urls
set totalsize 0
# Check all the files.
foreach {url_var distfile} $fetch_urls {
- global portfetch::$url_var
- if {![info exists $url_var]} {
+ global portfetch::urlmap
+ if {![info exists urlmap($url_var)]} {
ui_error [format [msgcat::mc "No defined site for tag: %s, using master_sites"] $url_var]
- set url_var master_sites
- global portfetch::$url_var
+ set urlmap($url_var) $master_sites
}
if {${distcheck.check} == "moddate"} {
set count 0
- foreach site [set $url_var] {
+ foreach site $urlmap($url_var) {
ui_debug [format [msgcat::mc "Checking %s from %s"] $distfile $site]
set file_url [portfetch::assemble_url $site $distfile]
if {[catch {set urlnewer [curl isnewer $file_url $port_moddate]} error]} {
@@ -95,7 +94,7 @@
}
} elseif {${distcheck.check} == "filesize"} {
set count 0
- foreach site [set $url_var] {
+ foreach site $urlmap($url_var) {
ui_debug [format [msgcat::mc "Checking %s from %s"] $distfile $site]
set file_url [portfetch::assemble_url $site $distfile]
if {[catch {set urlsize [curl getsize $file_url]} error]} {
Modified: branches/images-and-archives/base/src/port1.0/portdistfiles.tcl
===================================================================
--- branches/images-and-archives/base/src/port1.0/portdistfiles.tcl 2009-06-28 20:53:16 UTC (rev 53050)
+++ branches/images-and-archives/base/src/port1.0/portdistfiles.tcl 2009-06-28 21:26:50 UTC (rev 53051)
@@ -53,7 +53,7 @@
}
proc portdistfiles::distfiles_main {args} {
- global UI_PREFIX master_sites fetch_urls url_var distfile checksums_array
+ global UI_PREFIX master_sites checksums_array
# give up on ports that do not provide URLs
if {$master_sites == "{}"} {
@@ -61,13 +61,15 @@
}
# from portfetch... process the sites, files and patches
- portfetch::checkfiles
+ set fetch_urls {}
+ portfetch::checkfiles fetch_urls
# get checksum data from the portfile and parse it
set checksums_str [option checksums]
set result [portchecksum::parse_checksums $checksums_str]
- foreach {url_var distfile} $portfetch::fetch_urls {
+ foreach {url_var distfile} $fetch_urls {
+ global portfetch::urlmap
ui_msg "\[$distfile\]"
@@ -79,13 +81,12 @@
}
# determine sites to download from
- namespace import ::portfetch::$url_var
- if {![info exists $url_var]} {
- set url_var master_sites
+ if {![info exists urlmap($url_var)]} {
+ set urlmap($url_var) $master_sites
}
# determine URLs to download
- foreach site [set $url_var] {
+ foreach site $urlmap($url_var) {
set file_url [portfetch::assemble_url $site $distfile]
ui_msg " $file_url"
}
Modified: branches/images-and-archives/base/src/port1.0/portfetch.tcl
===================================================================
--- branches/images-and-archives/base/src/port1.0/portfetch.tcl 2009-06-28 20:53:16 UTC (rev 53050)
+++ branches/images-and-archives/base/src/port1.0/portfetch.tcl 2009-06-28 21:26:50 UTC (rev 53051)
@@ -43,6 +43,8 @@
namespace eval portfetch {
namespace export suffix
variable fetch_urls {}
+ variable urlmap
+ array set urlmap {}
}
# Name space for internal site lists storage
@@ -272,6 +274,7 @@
proc portfetch::checksites {args} {
global patch_sites master_sites master_sites.mirror_subdir \
patch_sites.mirror_subdir fallback_mirror_site global_mirror_site env
+ variable urlmap
append master_sites " ${global_mirror_site} ${fallback_mirror_site}"
if {[info exists env(MASTER_SITE_LOCAL)]} {
@@ -325,9 +328,9 @@
foreach site $site_list {
if {[regexp {([a-zA-Z]+://.+/?):([0-9A-Za-z_-]+)$} $site match site tag]} {
- lappend portfetch::$tag $site
+ lappend urlmap($tag) $site
} else {
- lappend portfetch::$list $site
+ lappend urlmap($list) $site
}
}
}
@@ -379,19 +382,18 @@
# sorts fetch_urls in order of ping time
proc portfetch::sortsites {urls} {
- global fallback_mirror_site
+ global fallback_mirror_site master_sites
upvar $urls fetch_urls
+ variable urlmap
set fallback_mirror_list [mirror_sites $fallback_mirror_site {} {}]
foreach {url_var distfile} $fetch_urls {
- variable portfetch::$url_var
- if {![info exists $url_var]} {
+ if {![info exists urlmap($url_var)]} {
ui_error [format [msgcat::mc "No defined site for tag: %s, using master_sites"] $url_var]
- set url_var master_sites
- variable portfetch::$url_var
+ set urlmap($url_var) $master_sites
}
- set urllist [set $url_var]
+ set urllist $urlmap($url_var)
set hosts {}
set hostregex {[a-zA-Z]+://([a-zA-Z0-9\.\-_]+)}
@@ -445,9 +447,9 @@
set pinglist [ lsort -real -index 1 $pinglist ]
- set $url_var {}
+ set urlmap($url_var) {}
foreach pair $pinglist {
- lappend $url_var [lindex $pair 0]
+ lappend urlmap($url_var) [lindex $pair 0]
}
}
}
@@ -587,6 +589,7 @@
global distfile site
global portverbose
variable fetch_urls
+ variable urlmap
if {![file isdirectory $distpath]} {
if {[catch {file mkdir $distpath} result]} {
@@ -631,14 +634,12 @@
sortsites fetch_urls
set sorted yes
}
- variable portfetch::$url_var
- if {![info exists $url_var]} {
+ if {![info exists urlmap($url_var)]} {
ui_error [format [msgcat::mc "No defined site for tag: %s, using master_sites"] $url_var]
- set url_var master_sites
- variable portfetch::$url_var
+ set urlmap($url_var) $master_sites
}
unset -nocomplain fetched
- foreach site [set $url_var] {
+ foreach site $urlmap($url_var) {
ui_msg "$UI_PREFIX [format [msgcat::mc "Attempting to fetch %s from %s"] $distfile $site]"
set file_url [portfetch::assemble_url $site $distfile]
set effectiveURL ""
Modified: branches/images-and-archives/base/src/registry1.0/receipt_flat.tcl
===================================================================
--- branches/images-and-archives/base/src/registry1.0/receipt_flat.tcl 2009-06-28 20:53:16 UTC (rev 53050)
+++ branches/images-and-archives/base/src/registry1.0/receipt_flat.tcl 2009-06-28 21:26:50 UTC (rev 53051)
@@ -74,7 +74,7 @@
# regex match case
if {$portversion == 0} {
- set x [glob -nocomplain [file join ${macports::registry.path} receipts ${portname}-*]]
+ set x [glob -nocomplain -directory [file join ${macports::registry.path} receipts] -- ${portname}-*]
if {[string length $x]} {
set matchfile [lindex $x 0]
# Remove trailing .bz2, if any.
@@ -134,7 +134,7 @@
if { $version == "" } {
# xxx: We really should have had the version given to us, how
# should we handle this?
- set x [glob -nocomplain [file join ${receipt_path} *]]
+ set x [glob -nocomplain -directory ${receipt_path} *]
if { [string length $x] } {
set v [lindex [file split [lindex $x 0]] end]
regexp {([-_a-zA-Z0-9\.]+)_([0-9]*)([+-_a-zA-Z0-9]*)$} $v match version revision variants
@@ -178,7 +178,7 @@
# move the old receipt
set convertedDirPath [file join ${macports::registry.path} receipts_converted]
file mkdir $convertedDirPath
- file rename $receipt_file $convertedDirPath
+ file rename -- $receipt_file $convertedDirPath
} elseif {[string match "# Version: *" $receipt_contents]} {
# This is new format
if {![string match "# Version: 1.0*" $receipt_contents]} {
@@ -326,12 +326,12 @@
close $receipt_handle
if { [file exists ${receipt_file}] } {
- file delete -force "${receipt_file}"
+ file delete -force -- "${receipt_file}"
} elseif { [file exists ${receipt_file}.bz2] } {
- file delete -force "${receipt_file}.bz2"
+ file delete -force -- "${receipt_file}.bz2"
}
- file rename -force "${receipt_file}.tmp" "${receipt_file}"
+ file rename -force -- "${receipt_file}.tmp" "${receipt_file}"
if { [file exists ${receipt_file}] && [file exists ${registry::autoconf::bzip2_path}] && ![info exists registry.nobzip] } {
system "${registry::autoconf::bzip2_path} -f ${receipt_file}"
@@ -420,14 +420,14 @@
if { [file exists ${receipt_path}] } {
# remove port receipt directory
ui_debug "deleting directory: ${receipt_path}"
- file delete -force ${receipt_path}
+ file delete -force -- ${receipt_path}
# remove port receipt parent directory (if empty)
set receipt_dir [file join ${macports::registry.path} receipts ${name}]
if { [file isdirectory ${receipt_dir}] } {
# 0 item means empty.
if { [llength [readdir ${receipt_dir}]] == 0 } {
ui_debug "deleting directory: ${receipt_dir}"
- file delete -force ${receipt_dir}
+ file delete -force -- ${receipt_dir}
} else {
ui_debug "${receipt_dir} is not empty"
}
@@ -845,9 +845,9 @@
close $map_handle
# don't both checking for presence, file delete doesn't error if file doesn't exist
- file delete ${map_file} ${map_file}.bz2
+ file delete -- ${map_file} ${map_file}.bz2
- file rename ${map_file}.tmp ${map_file}
+ file rename -- ${map_file}.tmp ${map_file}
if { [file exists ${map_file}] && [file exists ${registry::autoconf::bzip2_path}] && ![info exists registry.nobzip] } {
system "${registry::autoconf::bzip2_path} -f ${map_file}"
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/macports-changes/attachments/20090628/500a7b14/attachment.html>
More information about the macports-changes
mailing list