[24823] users/jberry/mpwa

source_changes at macosforge.org source_changes at macosforge.org
Sat May 5 15:02:51 PDT 2007


Revision: 24823
          http://trac.macosforge.org/projects/macports/changeset/24823
Author:   jberry at macports.org
Date:     2007-05-05 15:02:50 -0700 (Sat, 05 May 2007)

Log Message:
-----------
mpwa: use multiple file 64K file blobs per file to avoid in memory issues and mysql buffer sizing issues.

Modified Paths:
--------------
    users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb
    users/jberry/mpwa/app/models/port_pkg.rb
    users/jberry/mpwa/app/models/port_pkg_file.rb
    users/jberry/mpwa/doc/schema-thoughts.sql
    users/jberry/mpwa/lib/mpwa-conf.rb

Modified: users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb
===================================================================
--- users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb	2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb	2007-05-05 22:02:50 UTC (rev 24823)
@@ -51,6 +51,7 @@
   
   def emit
     file = PortPkgFile.find(params[:id])
-    send_data file.file_blob.data, :filename => file.file_path, :type => file.mime_type, :disposition => 'inline'
+    send_data file.data, :filename => file.file_path, :type => file.mime_type, :disposition => 'inline'
   end
+  
 end

Modified: users/jberry/mpwa/app/models/port_pkg.rb
===================================================================
--- users/jberry/mpwa/app/models/port_pkg.rb	2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/app/models/port_pkg.rb	2007-05-05 22:02:50 UTC (rev 24823)
@@ -109,11 +109,11 @@
         meta.categories.each { |c| self.add_tag(c) }
         
         # Save unpacked data into a file
-        self.files << PortPkgFile.from_path(pkgPath, tempDirPath, :mimetype => 'application/x-macports-portpkg')
+        self.files << PortPkgFile.from_path(self, pkgPath, tempDirPath, :mimetype => 'application/vnd.macports.portpkg')
         
         # Save files from the expanded package
         expandedPkgPath.find do |p|
-            self.files << PortPkgFile.from_path(p, tempDirPath) if p.file?
+            self.files << PortPkgFile.from_path(self, p, tempDirPath) if p.file?
         end
         
         # Save the pkg (maybe we shouldn't?)

Modified: users/jberry/mpwa/app/models/port_pkg_file.rb
===================================================================
--- users/jberry/mpwa/app/models/port_pkg_file.rb	2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/app/models/port_pkg_file.rb	2007-05-05 22:02:50 UTC (rev 24823)
@@ -1,3 +1,5 @@
+require 'stringio'
+
 require 'file_blob'
 require 'mpwa-conf'
 
@@ -3,8 +5,8 @@
 class PortPkgFile < ActiveRecord::Base
     belongs_to :port_pkg
-    has_one :file_blob
+    #has_many :file_blob -- we don't use this association to avoid keeping many blobs in memory
     
-    def PortPkgFile.from_path(path, path_root = nil, options = {})
-        port_pkg_file = PortPkgFile.new
+    def PortPkgFile.from_path(port_pkg, path, path_root = nil, options = {})
+        port_pkg_file = PortPkgFile.new(:port_pkg => port_pkg)
         return port_pkg_file.read_from_path(path, path_root, options)
     end
@@ -18,24 +20,63 @@
         mimetype = options[:mimetype] || PortPkgFile.mimetype_from_path(path)
         reported_path = options[:filename] || path_root.nil? ?
             path.to_s : Pathname.new(path).relative_path_from(path_root).to_s
+            
         File.open(path, "r") { |f| read_from_file(f, :path => reported_path, :mimetype => mimetype) }
         return self
     end
     
     def read_from_file(file, options = {})
-        # Create a new blob
-        blob = FileBlob.new
-        blob.read(file)
-        self.file_blob = blob
-        
-        # Save other file information
+        # Save file meta information
         self.file_path = options[:path]
-        self.length = blob.data.length
+        self.length = 0
         self.mime_type = options[:mimetype] || 'application/octet-stream'
         
-        self.md5 = Digest::MD5.hexdigest(blob.data)
-        self.sha256 = Digest::SHA256.hexdigest(blob.data)
+        # Save so that we get a primary id for the blob associations
+        self.save
         
+        # Create digesters for our digests
+        md5 = Digest::MD5.new
+        sha256 = Digest::SHA256.new
+        
+        # Read the file, creating blobs of data as we go
+        buf = ''
+        length = 0
+        seq = 0
+        while (file.read(MPWA::MAX_BLOB_SIZE, buf))
+            # Update the digests
+            md5.update buf
+            sha256.update buf
+            
+            # Create a new bob
+            blob = FileBlob.create(:port_pkg_file => self, :data => buf, :sequence => seq)
+            length = length + buf.length
+            
+            seq = seq + 1
+        end
+        
+        # Finish up
+        self.md5 = md5.hexdigest
+        self.sha256 = sha256.hexdigest
+        self.length = length
+    
+        self.save
         return self  
     end
+    
+    def write_to_file(file)
+        seq = 0
+        while (blob = FileBlob.find(:first, :conditions => "port_pkg_file_id=#{self.id} and sequence=#{seq}"))
+            file.write(blob.data)
+            seq = seq + 1
+        end
+    end
+    
+    def data()
+        StringIO.open("rw") do |f|
+            write_to_file(f)
+            f.rewind
+            f.read
+        end
+    end
+    
 end

Modified: users/jberry/mpwa/doc/schema-thoughts.sql
===================================================================
--- users/jberry/mpwa/doc/schema-thoughts.sql	2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/doc/schema-thoughts.sql	2007-05-05 22:02:50 UTC (rev 24823)
@@ -72,6 +72,7 @@
 create table File_Blobs (
     id                  bigint not null primary key auto_increment,
     port_pkg_file_id    bigint not null,
+    sequence            int not null,
     data                blob
 );
 

Modified: users/jberry/mpwa/lib/mpwa-conf.rb
===================================================================
--- users/jberry/mpwa/lib/mpwa-conf.rb	2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/lib/mpwa-conf.rb	2007-05-05 22:02:50 UTC (rev 24823)
@@ -2,4 +2,6 @@
 	FILETOOL = "/usr/bin/file"
 	XARTOOL = "/opt/local/bin/xar"
 	PORTTOOL = "/opt/local/bin/port"
+	
+	MAX_BLOB_SIZE = 1024*64
 end
\ No newline at end of file

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/macports-changes/attachments/20070505/a4793927/attachment.html


More information about the macports-changes mailing list