1 # -*- encoding: binary -*-
3 require 'mogilefs/util'
5 # Used for reading deprecated "bigfile" objects generated by the deprecated
6 # mogtool(1) utility. This is for reading legacy data and not recommended for
7 # new projects. MogileFS itself is capable of storing standalone objects
8 # of arbitrary length (as long as the underlying database and underlying
9 # filesystem on the DAV devices accept them).
11 module MogileFS::Bigfile
12 # VALID_TYPES = %w(file tarball partition).map { |x| x.freeze }.freeze
14 # returns a big_info hash if successful
16 bigfile_parse_info(get_file_data(key))
19 # returns total bytes written and the big_info hash if successful, raises an
20 # exception if not. wr_io is expected to be an IO-like object capable of
21 # receiving the write method.
22 def bigfile_write(key, wr_io, opts = { :verify => false })
23 info = bigfile_stat(key)
25 t = @get_file_data_timeout
27 # we only decode raw zlib deflated streams that mogtool (unfortunately)
28 # generates. tarballs and gzip(1) are up to to the application to decrypt.
29 if info[:compressed] || opts[:verify]
30 wr_io = MogileFS::Bigfile::Filter.new(wr_io, info, opts)
33 info[:parts].each_with_index do |part,part_nr|
34 next if part_nr == 0 # info[:parts][0] is always empty
37 sock = MogileFS::HTTPReader.first(part[:paths], "GET", t)
39 # part[:paths] may not be valid anymore due to rebalancing, however we
40 # can get_keys on key,<part_nr> and retry paths if all paths fail
41 paths = get_paths(part_key)
43 raise MogileFS::Backend::NoDevices,
44 "no device for key=#{part_key.inspect}", []
45 sock = MogileFS::HTTPReader.first(paths, "GET", t)
48 w = copy_stream(sock, wr_io)
50 wr_io.respond_to?(:md5_check!) and wr_io.md5_check!(part[:md5])
54 total += wr_io.flushed_bytes if wr_io.respond_to?(:flushed_bytes)
60 # parses the contents of a _big_info: string or IO object
61 def bigfile_parse_info(info) # :nodoc:
63 info.each_line do |line|
66 when /^(des|type|filename)\s+(.+)$/
68 when /^compressed\s+([01])$/
69 rv[:compressed] = ($1 == '1')
70 when /^(chunks|size)\s+(\d+)$/
71 rv[$1.to_sym] = $2.to_i
72 when /^part\s+(\d+)\s+bytes=(\d+)\s+md5=(.+)\s+paths:\s+(.+)$/
73 rv[:parts][$1.to_i] = {
76 :paths => $4.split(/\s*,\s*/),
84 require "mogilefs/bigfile/filter"
87 # Copied from mogtool:
88 # http://code.sixapart.com/svn/mogilefs/utils/mogtool, r1221
90 # this is a temporary file that we delete when we're doing recording all chunks
94 starttime=UNIXTIMESTAMP
96 # when done, we write the _info file and delete the _pre.
100 des Cow's ljdb backup as of 2004-11-17
101 type { partition, file, tarball }
103 filename ljbinlog.305.gz
104 partblocks 234324324324
107 part 1 <bytes> <md5hex>
108 part 2 <bytes> <md5hex>
109 part 3 <bytes> <md5hex>
110 part 4 <bytes> <md5hex>
111 part 5 <bytes> <md5hex>
120 BEGIN MOGTOOL RECIEPT
125 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
126 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
127 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid
128 part 1 bytes=23423432 md5=2349823948239423984 paths: http://dev5/2/23/23/.fid, http://dev6/23/423/4/324.fid