test/test_mogtool_bigfile.rb

   1 # -*- encoding: binary -*-
   2 require "./test/integration"
   3 require "net/http"
   4 ok = true
   5 unless File.executable?(`which mogtool 2>/dev/null`.strip)
   6   warn "mogtool not found, skipping #{__FILE__}"
   7   ok = false
   8 end
   9
  10 class TestMogtoolBigfile < TestMogIntegration
  11   buf = File.open("/dev/urandom") { |fp| fp.read(1024) }
  12   buf *= 1024
  13   RAND = Tempfile.new("rand")
  14   RAND.sync = true
  15   sha1 = Digest::SHA1.new
  16   100.times { sha1 << buf; RAND.write(buf) }
  17   buf = nil
  18   RAND_SHA1 = sha1.hexdigest
  19
  20   def setup
  21     super
  22     RAND.rewind
  23     @big_uuid = "big-#{uuid}"
  24     @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
  25   end
  26
  27   def mogtool!(*args)
  28     x!("mogtool", "--trackers=#{@trackers.join(',')}",
  29        "--domain=#@domain", *args)
  30   end
  31
  32   # the mogtool definition of gzip is wrong and just raw zlib deflate
  33   def test_bigfile_gzip_mogtool
  34     mogtool!("inject", "--gzip", "--bigfile", RAND.path, @big_uuid)
  35     sha1_check
  36   end
  37
  38   def test_bigfile_mogtool
  39     mogtool!("inject", "--bigfile", RAND.path, @big_uuid)
  40     sha1_check
  41
  42     # ensure fallback works for rebalanced/replaced files
  43     part1 = "#@big_uuid,1"
  44     tmp = tmpfile("part1")
  45     before_uris = @client.get_uris(part1)
  46     @client.get_file_data(part1, tmp)
  47     @client.delete(part1)
  48     @client.store_file(part1, nil, tmp.path)
  49     wait_for_DELETE(before_uris)
  50     sha1_check
  51
  52     # corrupt the existing data in part1
  53     @client.store_content(part1, nil, "HELLO")
  54     @client.get_uris(part1)
  55
  56     # corruption is detected on verify
  57     junk = tmpfile("junk")
  58     assert_raises(MogileFS::ChecksumMismatchError) do
  59       @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
  60     end
  61
  62     # corruption is NOT detected on verify
  63     junk = tmpfile("junk")
  64     @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => false)
  65
  66     # restoring no-corrupted data succeeds!
  67     @client.store_file(part1, nil, tmp.path)
  68     sha1_check
  69
  70     # missing parts fail
  71     before_uris = @client.get_uris(part1)
  72     @client.delete(part1)
  73     junk = tmpfile("junk")
  74     assert_raises(MogileFS::Backend::UnknownKeyError) do
  75       @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
  76     end
  77   end
  78
  79   def wait_for_DELETE(uris)
  80     uris.each do |uri|
  81       tries = 0
  82       begin
  83         Net::HTTP.start(uri.host, uri.port) do |http|
  84           sleep(0.1) while Net::HTTPOK === http.head(uri.path)
  85         end
  86       rescue
  87         if (tries += 1) < 666
  88           sleep(0.1)
  89           retry
  90         end
  91         raise
  92       end
  93     end
  94   end
  95
  96   def sha1_check
  97     r, w = IO.pipe
  98     @to_close << r
  99     @to_close << w
 100     th = Thread.new do
 101       sha1 = Digest::SHA1.new
 102       buf = ""
 103       while r.read(16384, buf)
 104         sha1 << buf
 105       end
 106       sha1.hexdigest
 107     end
 108     res = @client.bigfile_write("_big_info:#@big_uuid", w, :verify => true)
 109     w.close
 110     read_sha1 = th.value
 111     assert_equal RAND_SHA1, read_sha1
 112     assert_equal RAND.size, res[0]
 113   end
 114 end if ok