1 # -*- encoding: binary -*-
7 class TestMogileFS__MogileFS < TestMogileFS
9 @klass = MogileFS::MogileFS
14 assert_equal 'test', @client.domain
16 assert_raises ArgumentError do
17 MogileFS::MogileFS.new :hosts => ['kaa:6001']
21 def test_get_file_data_http
22 tmp = Tempfile.new('accept')
23 accept = File.open(tmp.path, "ab")
24 svr = Proc.new do |serv, port|
25 client, client_addr = serv.accept
27 readed = client.recv(4096, 0)
29 %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
31 client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
34 t1 = TempServer.new(svr)
35 t2 = TempServer.new(svr)
36 path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
37 path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
39 @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
41 assert_equal 'data!', @client.get_file_data('key')
42 assert_equal 1, accept.stat.size
44 TempServer.destroy_all!
47 def test_get_file_data_http_not_found_failover
48 tmp = Tempfile.new('accept')
49 accept = File.open(tmp.path, 'ab')
50 svr1 = Proc.new do |serv, port|
51 client, client_addr = serv.accept
53 readed = client.recv(4096, 0)
55 %r{\AGET /dev1/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
57 client.send("HTTP/1.0 404 Not Found\r\n\r\ndata!", 0)
61 svr2 = Proc.new do |serv, port|
62 client, client_addr = serv.accept
64 readed = client.recv(4096, 0)
66 %r{\AGET /dev2/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
68 client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
72 t1 = TempServer.new(svr1)
73 t2 = TempServer.new(svr2)
74 path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
75 path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
76 @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
78 assert_equal 'data!', @client.get_file_data('key')
79 assert_equal 2, accept.stat.size
81 TempServer.destroy_all!
84 def test_get_file_data_http_block
85 tmpfp = Tempfile.new('test_mogilefs.open_data')
87 chunk_size = 1024 * 1024
88 expect_size = nr * chunk_size
89 header = "HTTP/1.0 200 OK\r\n" \
90 "Content-Length: #{expect_size}\r\n\r\n"
91 assert_equal header.size, tmpfp.syswrite(header)
92 nr.times { assert_equal chunk_size, tmpfp.syswrite(' ' * chunk_size) }
93 assert_equal expect_size + header.size, File.size(tmpfp.path)
96 accept = Tempfile.new('accept')
97 svr = Proc.new do |serv, port|
98 client, client_addr = serv.accept
101 readed = client.recv(4096, 0)
103 %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
104 MogileFS::X.copy_stream(tmpfp, client)
108 t1 = TempServer.new(svr)
109 t2 = TempServer.new(svr)
110 path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
111 path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
113 @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
115 data = Tempfile.new('test_mogilefs.dest_data')
117 @client.get_file_data('key') do |fp|
121 fp.sysread(16384, buf)
124 assert_equal read_nr, data.syswrite(buf), "partial write"
132 assert_equal expect_size, nr, "size mismatch"
133 assert_equal 1, accept.stat.size
137 path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
138 path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
140 @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
142 expected = [ path1, path2 ]
144 assert_equal expected, @client.get_paths('key').sort
148 path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
149 path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
151 @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
153 expected = [ URI.parse(path1), URI.parse(path2) ]
155 assert_equal expected, @client.get_uris('key')
159 def test_get_paths_unknown_key
160 @backend.get_paths = ['unknown_key', '']
162 assert_raises MogileFS::Backend::UnknownKeyError do
163 assert_equal nil, @client.get_paths('key')
167 def test_delete_existing
168 @backend.delete = { }
169 assert_nothing_raised do
170 @client.delete 'no_such_key'
174 def test_delete_nonexisting
175 @backend.delete = 'unknown_key', ''
176 assert_raises MogileFS::Backend::UnknownKeyError do
177 @client.delete('no_such_key')
181 def test_delete_readonly
182 @client.readonly = true
183 assert_raises MogileFS::ReadOnlyError do
184 @client.delete 'no_such_key'
189 @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
190 'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
191 @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_4',
192 'key_1' => 'new_key_3', 'key_2' => 'new_key_4' }
193 @backend.list_keys = { 'key_count' => 0, 'next_after' => 'new_key_4' }
195 @client.each_key 'new' do |key|
199 assert_equal %w[new_key_1 new_key_2 new_key_3 new_key_4], keys
203 @backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
204 'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
206 keys, next_after = @client.list_keys 'new'
207 assert_equal ['new_key_1', 'new_key_2'], keys.sort
208 assert_equal 'new_key_2', next_after
211 def test_list_keys_block
212 @backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
213 'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
214 http_resp = "HTTP/1.0 200 OK\r\nContent-Length: %u\r\n"
215 srv = Proc.new do |serv, port, size|
216 client, client_addr = serv.accept
218 readed = client.readpartial(4096)
219 assert %r{\AHEAD } =~ readed
220 client.send(http_resp % size, 0)
223 t1 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
224 t2 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
225 t3 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 10) })
226 @backend.get_paths = { 'paths' => '2',
227 'path1' => "http://127.0.0.1:#{t1.port}/",
228 'path2' => "http://127.0.0.1:#{t2.port}/" }
229 @backend.get_paths = { 'paths' => '1',
230 'path1' => "http://127.0.0.1:#{t3.port}/" }
233 keys, next_after = @client.list_keys('new') do |key,length,devcount|
234 res << [ key, length, devcount ]
237 expect_res = [ [ 'new_key_1', 5, 2 ], [ 'new_key_2', 10, 1 ] ]
238 assert_equal expect_res, res
239 assert_equal ['new_key_1', 'new_key_2'], keys.sort
240 assert_equal 'new_key_2', next_after
242 TempServer.destroy_all!
245 def test_new_file_http
246 @client.readonly = true
247 assert_raises MogileFS::ReadOnlyError do
248 @client.new_file 'new_key', 'test'
252 def test_new_file_readonly
253 @client.readonly = true
254 assert_raises MogileFS::ReadOnlyError do
255 @client.new_file 'new_key', 'test'
260 accept = Tempfile.new('accept')
261 t = TempServer.new(Proc.new do |serv,port|
262 client, client_addr = serv.accept
264 readed = client.recv(4096, 0) rescue nil
266 assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
267 client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", 0)
271 path = "http://127.0.0.1:#{t.port}/path"
272 @backend.get_paths = { 'paths' => 1, 'path1' => path }
274 assert_equal 5, @client.size('key')
275 assert_equal 1, accept.stat.size
278 def test_bad_size_http
279 tmp = Tempfile.new('accept')
280 t = TempServer.new(Proc.new do |serv,port|
281 client, client_addr = serv.accept
283 readed = client.recv(4096, 0) rescue nil
284 assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
286 client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
290 path = "http://127.0.0.1:#{t.port}/path"
291 @backend.get_paths = { 'paths' => 1, 'path1' => path }
293 assert_raises(MogileFS::Error) { @client.size('key') }
294 assert_equal 1, tmp.stat.size
297 def test_store_file_small_http
298 received = Tempfile.new('received')
299 to_store = Tempfile.new('small')
300 to_store.syswrite('data')
302 expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
303 t = TempServer.new(Proc.new do |serv, accept|
304 client, client_addr = serv.accept
306 received.syswrite(client.recv(4096, 0))
307 client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
311 @backend.create_open = {
313 'path' => "http://127.0.0.1:#{t.port}/path",
315 nr = @client.store_file 'new_key', 'test', to_store.path
318 assert_equal expected, received.sysread(4096)
320 TempServer.destroy_all!
323 def test_store_content_http
324 received = Tempfile.new('recieved')
325 expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
327 t = TempServer.new(Proc.new do |serv, accept|
328 client, client_addr = serv.accept
330 received.syswrite(client.recv(4096, 0))
331 client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
335 @backend.create_open = {
337 'path' => "http://127.0.0.1:#{t.port}/path",
340 nr = @client.store_content 'new_key', 'test', 'data'
345 assert_equal expected, received.sysread(4096)
347 TempServer.destroy_all!
351 def test_store_content_with_writer_callback
352 received = Tempfile.new('recieved')
353 expected = "PUT /path HTTP/1.0\r\nContent-Length: 40\r\n\r\n"
357 t = TempServer.new(Proc.new do |serv, accept|
358 client, client_addr = serv.accept
362 buf = client.readpartial(8192) or break
363 break if buf.length == 0
364 assert_equal buf.length, received.syswrite(buf)
366 break if nr >= expected.size
368 client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
372 @backend.create_open = {
374 'path' => "http://127.0.0.1:#{t.port}/path",
377 cbk = MogileFS::Util::StoreContent.new(40) do |write_callback|
379 write_callback.call("data")
382 assert_equal 40, cbk.length
383 nr = @client.store_content('new_key', 'test', cbk)
387 assert_equal expected, received.sysread(4096)
389 TempServer.destroy_all!
392 def test_store_content_multi_dest_failover
393 received1 = Tempfile.new('received')
394 received2 = Tempfile.new('received')
395 expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
397 t1 = TempServer.new(Proc.new do |serv, accept|
398 client, client_addr = serv.accept
400 received1.syswrite(client.recv(4096, 0))
401 client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
405 t2 = TempServer.new(Proc.new do |serv, accept|
406 client, client_addr = serv.accept
408 received2.syswrite(client.recv(4096, 0))
409 client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
413 @backend.create_open = {
416 'path_1' => "http://127.0.0.1:#{t1.port}/path",
418 'path_2' => "http://127.0.0.1:#{t2.port}/path",
421 nr = @client.store_content 'new_key', 'test', 'data'
425 assert_equal expected, received1.sysread(4096)
426 assert_equal expected, received2.sysread(4096)
428 TempServer.destroy_all!
431 def test_store_content_http_fail
432 t = TempServer.new(Proc.new do |serv, accept|
433 client, client_addr = serv.accept
436 client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
440 @backend.create_open = {
442 'path' => "http://127.0.0.1:#{t.port}/path",
445 assert_raises MogileFS::HTTPFile::NoStorageNodesError do
446 @client.store_content 'new_key', 'test', 'data'
450 def test_store_content_http_empty
451 received = Tempfile.new('received')
452 expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
453 t = TempServer.new(Proc.new do |serv, accept|
454 client, client_addr = serv.accept
456 received.syswrite(client.recv(4096, 0))
457 client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
461 @backend.create_open = {
463 'path' => "http://127.0.0.1:#{t.port}/path",
466 nr = @client.store_content 'new_key', 'test', ''
469 assert_equal expected, received.sysread(4096)
472 def test_store_content_nfs
473 @backend.create_open = {
478 assert_raises MogileFS::UnsupportedPathError do
479 @client.store_content 'new_key', 'test', 'data'
483 def test_new_file_http_large
484 expect = Tempfile.new('test_mogilefs.expect')
485 to_put = Tempfile.new('test_mogilefs.to_put')
486 received = Tempfile.new('test_mogilefs.received')
489 chunk_size = 1024 * 1024
490 expect_size = nr * chunk_size
492 header = "PUT /path HTTP/1.0\r\n" \
493 "Content-Length: #{expect_size}\r\n\r\n"
494 assert_equal header.size, expect.syswrite(header)
496 assert_equal chunk_size, expect.syswrite(' ' * chunk_size)
497 assert_equal chunk_size, to_put.syswrite(' ' * chunk_size)
499 assert_equal expect_size + header.size, expect.stat.size
500 assert_equal expect_size, to_put.stat.size
502 readed = Tempfile.new('readed')
503 t = TempServer.new(Proc.new do |serv, accept|
504 client, client_addr = serv.accept
508 buf = client.readpartial(8192) or break
509 break if buf.length == 0
510 assert_equal buf.length, received.syswrite(buf)
512 break if nr >= expect.stat.size
514 readed.syswrite("#{nr}")
515 client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
519 @backend.create_open = {
521 'path' => "http://127.0.0.1:#{t.port}/path",
524 orig_size = to_put.size
525 nr = @client.store_file('new_key', 'test', to_put.path)
526 assert nr, nr.inspect
527 assert_equal orig_size, nr
528 assert_equal orig_size, to_put.size
530 assert_equal expect.stat.size, readed.sysread(4096).to_i
532 ENV['PATH'].split(/:/).each do |path|
533 cmp_bin = "#{path}/cmp"
534 File.executable?(cmp_bin) or next
535 # puts "running #{cmp_bin} #{expect.path} #{received.path}"
536 assert( system(cmp_bin, expect.path, received.path) )
541 TempServer.destroy_all!
544 def test_store_content_readonly
545 @client.readonly = true
547 assert_raises MogileFS::ReadOnlyError do
548 @client.store_content 'new_key', 'test', nil
552 def test_store_file_readonly
553 @client.readonly = true
554 assert_raises MogileFS::ReadOnlyError do
555 @client.store_file 'new_key', 'test', nil
559 def test_rename_existing
562 assert_nil @client.rename('from_key', 'to_key')
565 def test_rename_nonexisting
566 @backend.rename = 'unknown_key', ''
568 assert_raises MogileFS::Backend::UnknownKeyError do
569 @client.rename('from_key', 'to_key')
573 def test_rename_no_key
574 @backend.rename = 'no_key', 'no_key'
576 e = assert_raises MogileFS::Backend::NoKeyError do
577 @client.rename 'new_key', 'test'
580 assert_equal 'no_key', e.message
583 def test_rename_readonly
584 @client.readonly = true
586 e = assert_raises MogileFS::ReadOnlyError do
587 @client.rename 'new_key', 'test'
590 assert_equal 'readonly mogilefs', e.message
595 assert_nothing_raised do
596 assert_equal({}, @client.sleep(2))
602 # tested with 1000, though it takes a while
604 ENV['NR_CHUNKS'] ? ENV['NR_CHUNKS'].to_i : 10