1 # -*- encoding: binary -*-
2 require './test/integration'
4 class TestMogileFSIntegration < TestMogIntegration
7 @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
11 assert ! @client.exist?("CRUD")
12 assert_equal 4, @client.store_content("CRUD", "default", "DATA")
13 assert @client.exist?("CRUD")
14 assert_equal 4, @client.size("CRUD")
15 assert_equal "DATA", @client.get_file_data("CRUD")
16 assert_equal "DAT", @client.get_file_data("CRUD", nil, 3)
17 assert_equal "AT", @client.get_file_data("CRUD", nil, 2, 1)
19 tmp = Tempfile.new("z")
22 assert_equal 4, @client.get_file_data("CRUD", tmp_path)
23 assert_equal "DATA", File.read(tmp_path)
26 sio = StringIO.new("")
27 rv = @client.get_file_data("CRUD", sio)
29 assert_equal "DATA", sio.string
30 assert_equal 8, @client.store_content("CRUD", "default", "MOARDATA")
31 assert_equal "MOARDATA", @client.get_file_data("CRUD")
32 assert_equal true, @client.delete("CRUD")
33 assert_raises(MogileFS::Backend::UnknownKeyError) { @client.delete("CRUD") }
35 data = "hello world\n".freeze
40 assert_equal tmp.size, @client.store_file("blob", nil, tmp)
41 assert_equal(data, @client.get_file_data("blob"))
43 data = "pipe!\n".freeze
49 assert_equal data.size, @client.store_file("pipe", nil, r)
50 assert_nothing_raised do
54 assert_equal(data, @client.get_file_data("pipe"))
56 cbk = MogileFS::Util::StoreContent.new(nil) do |write_callback|
57 10.times { write_callback.call("data") }
60 nr = @client.store_content('store_content', nil, cbk)
62 assert_equal("data" * 10, @client.get_file_data('store_content'))
65 def test_store_non_rewindable
70 def tmp.read(len, buf = "")
74 assert_raises(MogileFS::HTTPFile::NonRetryableError) do
75 @client.store_file("non_rewindable", nil, tmp)
80 assert_equal 3, @client.store_content("file_info", "default", "FOO")
81 res = @client.file_info("file_info")
82 assert_kind_of Integer, res["fid"]
83 assert_equal 3, res["length"]
84 assert ! res.include?("devids")
85 assert_kind_of Integer, res["devcount"]
87 res = @client.file_info("file_info", :devices => true)
88 assert_kind_of Integer, res["fid"]
89 assert_equal 3, res["length"]
90 assert_kind_of Integer, res["devcount"]
91 devids = res.delete("devids")
92 assert_instance_of Array, devids
93 devids.each { |devid| assert_kind_of Integer, devid }
94 assert_equal res["devcount"], devids.size
98 assert_equal 3, @client.store_content("file_debug", "default", "BUG")
99 a = @client.file_debug("file_debug")
100 b = @client.file_debug(:key => "file_debug")
101 fid = @client.file_info("file_debug")["fid"]
102 c = @client.file_debug(fid)
103 d = @client.file_debug(:fid => fid)
105 [ a, b, c, d ].each do |res|
106 assert_equal fid, res["fid_fid"]
107 assert_equal 0, res["fid_classid"]
108 assert_equal "file_debug", res["fid_dkey"]
109 assert_equal 3, res["fid_length"]
110 assert_kind_of Array, res["devids"]
111 assert_kind_of Integer, res["devids"][0]
112 res["devids"].each do |devid|
113 uri = URI.parse(res["devpath_#{devid}"])
114 assert_equal "http", uri.scheme
116 assert_equal "default", res["fid_class"]
118 @client.delete("file_debug")
119 rv = @client.file_debug(fid)
120 assert rv.keys.grep(/\Afid_/).empty?, rv.inspect
123 def test_file_debug_in_progress
124 rv = @client.new_file("file_debug_in_progress") do |http_file|
126 dests = http_file.instance_variable_get(:@dests)
127 dests[0][1] =~ %r{/(\d+)\.fid\z}
129 rv = @client.file_debug(fid)
130 devids = dests.map { |x| x[0].to_i }.sort
131 assert_equal devids, rv["tempfile_devids"].sort
132 assert_equal "file_debug_in_progress", rv["tempfile_dkey"]
137 def test_admin_get_devices
138 admin = MogileFS::Admin.new(:hosts => @trackers)
139 devices = admin.get_devices
140 if any_device = devices[0]
141 %w(mb_asof mb_free mb_used mb_total devid weight hostid).each do |field|
142 case value = any_device[field]
145 assert value >= 0, "#{field}=#{value.inspect} is negative"
147 assert false, "#{field}=#{value.inspect} is #{value.class}"
151 field = "utilization"
152 case value = any_device[field]
155 assert value >= 0.0, "#{field}=#{value.inspect} is negative"
157 assert false, "#{field}=#{value.inspect} is #{value.class}"
162 # TODO: move this to a fresh instance
163 def test_admin_each_fid
164 admin = MogileFS::Admin.new(:hosts => @trackers)
166 count = admin.each_fid do |info|
167 seen[info["fid"]] = true
168 assert_kind_of Integer, info["fid"]
169 assert_kind_of Integer, info["length"]
170 assert_kind_of Integer, info["devcount"]
171 assert_kind_of String, info["key"]
172 assert_kind_of String, info["class"]
173 assert_kind_of String, info["domain"]
175 assert_equal count, seen.size
176 end if ENV["TEST_EXPENSIVE"]
178 def test_new_file_no_block
179 rv = @client.new_file("no_block")
180 assert_nothing_raised { rv.write "HELLO" }
182 assert_equal "HELLO", @client.get_file_data("no_block")
185 def test_new_file_known_content_length
186 rv = @client.new_file("a", :content_length => 5)
187 assert_nothing_raised { rv.write "HELLO" }
189 assert_equal "HELLO", @client.get_file_data("a")
191 rv = @client.new_file("a", :content_length => 6)
192 assert_nothing_raised { rv.write "GOOD" }
193 assert_raises(MogileFS::SizeMismatchError) { rv.close }
194 assert_equal "HELLO", @client.get_file_data("a")
196 rv = @client.new_file("large", :content_length => 6, :largefile => true)
197 assert_instance_of MogileFS::NewFile::Stream, rv
198 assert_equal 6, rv.write("HIHIHI")
200 assert_equal "HIHIHI", @client.get_file_data("large")
203 def test_new_file_content_md5
205 b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
206 rv = @client.new_file("a", :content_md5 => b64digest, :content_length => 5)
207 assert_nothing_raised { rv.write "HELLO" }
209 assert_equal "HELLO", @client.get_file_data("a")
211 assert_nothing_raised { w.write "HIHI"; w.close }
212 assert_raises(ArgumentError) do
213 @client.new_file("a", :content_md5 => b64digest) { |f| f.big_io = r }
215 assert_equal "HELLO", @client.get_file_data("a")
217 assert_nothing_raised do
218 @client.new_file("a", :content_md5 => :trailer) { |f| f.big_io = r }
220 assert_equal "HIHI", @client.get_file_data("a")
222 # legacy, in case anybody used it
223 rv = @client.new_file("a",{:class => "default"}, 6)
224 assert_equal 2, rv.write("HI")
225 assert_raises(MogileFS::SizeMismatchError) { rv.close }
226 assert_equal "HIHI", @client.get_file_data("a")
228 rv = @client.new_file("a",{:class => "default"}, 2)
229 assert_equal 2, rv.write("HI")
231 assert_equal "HI", @client.get_file_data("a")
232 assert_raises(MogileFS::Backend::UnregClassError) {
233 @client.new_file("a", "non-existent", 2)
235 assert_raises(MogileFS::Backend::UnregClassError) {
236 @client.new_file("a", :class => "non-existent")
242 def test_store_content_opts
243 b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
244 assert_nothing_raised do
245 @client.store_content("c", nil, "HELLO", :content_md5 => b64digest)
247 assert_raises(MogileFS::SizeMismatchError) do
248 @client.store_content("c", nil, "GOODBYE", :content_length => 2)
250 assert_equal "HELLO", @client.get_file_data("c")
253 def test_store_file_opts
254 b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
255 io = StringIO.new("HELLO")
256 assert_nothing_raised do
257 @client.store_file("c", nil, io, :content_md5 => b64digest)
260 io = StringIO.new("GOODBYE")
261 assert_raises(MogileFS::SizeMismatchError) do
262 @client.store_content("c", nil, io, :content_length => 2)
264 assert_equal "HELLO", @client.get_file_data("c")
267 def test_store_file_content_md5_lambda
269 expect_md5 = lambda do
271 [ Digest::MD5.digest("HELLO") ].pack('m').strip
273 io = StringIO.new("HELLO")
274 assert_nothing_raised do
275 @client.store_file("c", nil, io, :content_md5 => expect_md5)
278 assert_equal true, checked, "expect_md5 lambda called"
281 def test_store_file_unlinked_tempfile
282 tmp = Tempfile.new("store_file_unlinked")
287 assert_nothing_raised do
288 @client.store_file("unlinked", nil, tmp)
291 assert_equal "HIHI", @client.get_file_data("unlinked")
295 admin = MogileFS::Admin.new(:hosts => @trackers)
296 admin.create_class(@domain, "one", 1)
297 admin.create_class(@domain, "two", 2)
298 4.times { admin.clear_cache }
300 assert_equal 4, @client.store_content("uc", "default", "DATA")
301 assert_equal true, @client.updateclass("uc", "one")
302 assert_equal true, @client.updateclass("uc", "two")
303 assert_raises(MogileFS::Backend::ClassNotFoundError) do
304 @client.updateclass("uc", "wtf")
306 assert_raises(MogileFS::Backend::InvalidKeyError) do
307 @client.updateclass("nonexistent", "one")
311 admin.delete_class @domain, "one"
312 admin.delete_class @domain, "two"