1 # -*- encoding: binary -*-
4 # MogileFS File manipulation client.
6 class MogileFS::MogileFS < MogileFS::Client
8 include MogileFS::Bigfile
11 # The domain of keys for this MogileFS client.
15 # The timeout for get_file_data. Defaults to five seconds.
17 attr_accessor :get_file_data_timeout
20 # Creates a new MogileFS::MogileFS instance. +args+ must include a key
21 # :domain specifying the domain of this client.
23 def initialize(args = {})
24 @domain = args[:domain]
26 @get_file_data_timeout = 5
28 raise ArgumentError, "you must specify a domain" unless @domain
30 if @backend = args[:db_backend]
38 # Enumerates keys starting with +key+.
40 def each_key(prefix = "")
43 keys, after = list_keys prefix
45 until keys.nil? or keys.empty? do
46 keys.each { |k| yield k }
47 keys, after = list_keys prefix, after
54 # Retrieves the contents of +key+. If +dest+ is specified, +dest+
55 # should be an IO-like object capable of receiving the +write+ method
58 def get_file_data(key, dest = nil, count = nil, offset = nil)
59 paths = get_paths(key)
60 sock = MogileFS::HTTPReader.first(paths, @get_file_data_timeout,
70 sock.close if sock && ! sock.closed?
74 # Get the paths for +key+.
76 def get_paths(key, *args)
85 opts[:noverify] = args[:noverify]
86 opts[:zone] = args[:zone]
87 pathcount = args[:pathcount] and opts[:pathcount] = pathcount.to_i
90 opts[:noverify] = false == opts[:noverify] ? 0 : 1
91 @backend.respond_to?(:_get_paths) and return @backend._get_paths(opts)
92 res = @backend.get_paths(opts)
93 (1..res['paths'].to_i).map { |i| res["path#{i}"] }.compact
97 # Get the URIs for +key+.
99 def get_uris(key, *args)
100 get_paths(key, *args).map! { |path| URI.parse(path) }
104 # Creates a new file +key+ in +klass+. +bytes+ is currently unused.
106 # The +block+ operates like File.open.
108 def new_file(key, klass = nil, bytes = 0) # :yields: file
109 raise MogileFS::ReadOnlyError if readonly?
110 opts = { :domain => @domain, :key => key, :multi_dest => 1 }
111 opts[:class] = klass if klass && klass != "default"
112 res = @backend.create_open(opts)
114 dests = if dev_count = res['dev_count'] # multi_dest succeeded
115 (1..dev_count.to_i).map do |i|
116 [res["devid_#{i}"], res["path_#{i}"]]
118 else # single destination returned
119 # 0x0040: d0e4 4f4b 2064 6576 6964 3d31 2666 6964 ..OK.devid=1&fid
120 # 0x0050: 3d33 2670 6174 683d 6874 7470 3a2f 2f31 =3&path=http://1
121 # 0x0060: 3932 2e31 3638 2e31 2e37 323a 3735 3030 92.168.1.72:7500
122 # 0x0070: 2f64 6576 312f 302f 3030 302f 3030 302f /dev1/0/000/000/
123 # 0x0080: 3030 3030 3030 3030 3033 2e66 6964 0d0a 0000000003.fid..
125 [[res['devid'], res['path']]]
128 case (dests[0][1] rescue nil)
129 when /^http:\/\// then
130 http_file = MogileFS::HTTPFile.new(dests, bytes)
132 rv = http_file.commit
133 @backend.create_close(:fid => res['fid'],
134 :devid => http_file.devid,
137 :path => http_file.uri.to_s,
141 raise MogileFS::EmptyPathError,
142 "Empty path for mogile upload res=#{res.inspect}"
144 raise MogileFS::UnsupportedPathError,
145 "paths '#{dests.inspect}' returned by backend is not supported"
150 # Copies the contents of +file+ into +key+ in class +klass+. +file+ can be
151 # either a path name (String or Pathname object) or an IO-like object that
152 # responds to #read or #readpartial. Returns size of +file+ stored.
154 def store_file(key, klass, file)
155 raise MogileFS::ReadOnlyError if readonly?
157 new_file(key, klass) { |mfp| mfp.big_io = file }
161 # Stores +content+ into +key+ in class +klass+.
163 def store_content(key, klass, content)
164 raise MogileFS::ReadOnlyError if readonly?
166 new_file key, klass do |mfp|
167 if content.is_a?(MogileFS::Util::StoreContent)
168 mfp.streaming_io = content
179 raise MogileFS::ReadOnlyError if readonly?
181 @backend.delete :domain => @domain, :key => key
189 @backend.sleep :duration => duration
193 # Renames a key +from+ to key +to+.
196 raise MogileFS::ReadOnlyError if readonly?
198 @backend.rename :domain => @domain, :from_key => from, :to_key => to
203 # Returns the size of +key+.
205 @backend.respond_to?(:_size) and return @backend._size(domain, key)
207 file_info(key)["length"].to_i
208 rescue MogileFS::Backend::UnknownCommandError
209 paths_size(get_paths(key))
213 def paths_size(paths) # :nodoc:
214 require "mogilefs/paths_size"
215 MogileFS::PathsSize.call(paths)
219 # Lists keys starting with +prefix+ follwing +after+ up to +limit+. If
220 # +after+ is nil the list starts at the beginning.
222 def list_keys(prefix = "", after = nil, limit = 1000)
223 if @backend.respond_to?(:_list_keys)
224 block_given? or return @backend._list_keys(domain, prefix, after, limit)
225 return @backend._list_keys(domain, prefix, after, limit) do |*a|
231 @backend.list_keys(:domain => domain, :prefix => prefix,
232 :after => after, :limit => limit)
233 rescue MogileFS::Backend::NoneMatchError
237 keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
239 # emulate the MogileFS::Mysql interface, slowly...
243 rescue MogileFS::Backend::UnknownCommandError # MogileFS < 2.45
244 paths = get_paths(key)
245 res = { "length" => paths_size(paths), "devcount" => paths.size }
247 yield key, res["length"], res["devcount"]
251 [ keys, res['next_after'] ]
254 # Return metadata about a file as a hash.
255 # Returns the domain, class, expected length, devcount, etc.
256 # Optionally device ids (not paths) can be returned as
257 # well if :devices is specified and +true+.
259 # This should only be used for informational purposes, and not usually
260 # for dynamically serving files.
261 def file_info(key, args = nil)
262 opts = { :domain => @domain, :key => key }
263 args and devices = args[:devices] and opts[:devices] = devices ? 1 : 0
264 rv = @backend.file_info(opts)
265 %w(fid length devcount).each { |f| rv[f] = rv[f].to_i }
266 devids = rv["devids"] and
267 rv["devids"] = devids.split(/,/).map! { |x| x.to_i }
271 # Given an Integer +fid+ or String +key+ and domain, thorougly search
272 # the database for all occurences of a particular fid.
274 # Use this sparingly, this command hits the master database numerous
275 # times and is very expensive. This is not for production use, only
276 # troubleshooting and debugging.
278 # Searches for fid=666:
280 # client.file_debug(666)
282 # Search for key=foo using the default domain for this object:
284 # client.file_debug("foo")
286 # Search for key=foo in domain="bar":
288 # client.file_debug(:key => "foo", :domain => "bar")
292 when Integer then args = { "fid" => args }
293 when String then args = { "key" => args }
295 opts = { :domain => args[:domain] || @domain }.merge!(args)
297 rv = @backend.file_debug(opts)
300 when /_(?:classid|devcount|dmid|fid|length|
301 nexttry|fromdevid|failcount|flags|devid|type)\z/x
304 rv[k] = v.split(/,/).map! { |x| x.to_i }