1 # -*- encoding: binary -*-
3 require 'mogilefs/util'
5 # Used for reading deprecated "bigfile" objects generated by the deprecated
6 # mogtool(1) utility. This is for reading legacy data and not recommended for
7 # new projects. MogileFS itself is capable of storing standalone objects
8 # of arbitrary length (as long as the underlying database and underlying
9 # filesystem on the DAV devices accept them).
11 module MogileFS::Bigfile
12 # VALID_TYPES = %w(file tarball partition).map { |x| x.freeze }.freeze
14 # returns a big_info hash if successful
16 bigfile_parse_info(get_file_data(key))
19 # returns total bytes written and the big_info hash if successful, raises an
20 # exception if not. wr_io is expected to be an IO-like object capable of
21 # receiving the write method.
22 def bigfile_write(key, wr_io, opts = { :verify => false })
23 info = bigfile_stat(key)
25 t = @get_file_data_timeout
27 # we only decode raw zlib deflated streams that mogtool (unfortunately)
28 # generates. tarballs and gzip(1) are up to to the application to decrypt.
29 if info[:compressed] || opts[:verify]
30 wr_io = MogileFS::Bigfile::Filter.new(wr_io, info, opts)
33 info[:parts].each_with_index do |part,part_nr|
34 next if part_nr == 0 # info[:parts][0] is always empty
37 sock = MogileFS::HTTPReader.first(part[:paths], "GET", t)
39 # part[:paths] may not be valid anymore due to rebalancing, however we
40 # can get_keys on key,<part_nr> and retry paths if all paths fail
41 part_key = "#{key.sub(/^_big_info:/, '')},#{part_nr}"
42 paths = get_paths(part_key)
44 raise MogileFS::Backend::NoDevices,
45 "no device for key=#{part_key.inspect}", []
46 sock = MogileFS::HTTPReader.first(paths, "GET", t)
49 w = MogileFS::X.copy_stream(sock, wr_io)
51 wr_io.respond_to?(:md5_check!) and wr_io.md5_check!(part[:md5])
55 total += wr_io.flushed_bytes if wr_io.respond_to?(:flushed_bytes)
61 # parses the contents of a _big_info: string or IO object
62 def bigfile_parse_info(info) # :nodoc:
64 info.each_line do |line|
67 when /^(des|type|filename)\s+(.+)$/
69 when /^compressed\s+([01])$/
70 rv[:compressed] = ($1 == '1')
71 when /^(chunks|size)\s+(\d+)$/
72 rv[$1.to_sym] = $2.to_i
73 when /^part\s+(\d+)\s+bytes=(\d+)\s+md5=(.+)\s+paths:\s+(.+)$/
74 rv[:parts][$1.to_i] = {
77 :paths => $4.split(/\s*,\s*/),
85 require "mogilefs/bigfile/filter"
88 # Copied from mogtool:
89 # http://code.sixapart.com/svn/mogilefs/utils/mogtool, r1221
91 # this is a temporary file that we delete when we're doing recording all chunks
95 starttime=UNIXTIMESTAMP
97 # when done, we write the _info file and delete the _pre.
101 des Cow's ljdb backup as of 2004-11-17
102 type { partition, file, tarball }
104 filename ljbinlog.305.gz
105 partblocks 234324324324
108 part 1 <bytes> <md5hex>
109 part 2 <bytes> <md5hex>
110 part 3 <bytes> <md5hex>
111 part 4 <bytes> <md5hex>
112 part 5 <bytes> <md5hex>
121 BEGIN MOGTOOL RECIEPT
126 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
127 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
128 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
129 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid