test/unit/test_upload.rb

   1 # -*- encoding: binary -*-
   2
   3 # Copyright (c) 2009 Eric Wong
   4 require 'test/test_helper'
   5 require 'digest/md5'
   6
   7 include Unicorn
   8
   9 class UploadTest < Test::Unit::TestCase
  10
  11   def setup
  12     @addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
  13     @port = unused_port
  14     @hdr = {'Content-Type' => 'text/plain', 'Content-Length' => '0'}
  15     @bs = 4096
  16     @count = 256
  17     @server = nil
  18
  19     # we want random binary data to test 1.9 encoding-aware IO craziness
  20     @random = File.open('/dev/urandom','rb')
  21     @sha1 = Digest::SHA1.new
  22     @sha1_app = lambda do |env|
  23       input = env['rack.input']
  24       resp = {}
  25
  26       @sha1.reset
  27       while buf = input.read(@bs)
  28         @sha1.update(buf)
  29       end
  30       resp[:sha1] = @sha1.hexdigest
  31
  32       # rewind and read again
  33       input.rewind
  34       @sha1.reset
  35       while buf = input.read(@bs)
  36         @sha1.update(buf)
  37       end
  38
  39       if resp[:sha1] == @sha1.hexdigest
  40         resp[:sysread_read_byte_match] = true
  41       end
  42
  43       if expect_size = env['HTTP_X_EXPECT_SIZE']
  44         if expect_size.to_i == input.size
  45           resp[:expect_size_match] = true
  46         end
  47       end
  48       resp[:size] = input.size
  49       resp[:content_md5] = env['HTTP_CONTENT_MD5']
  50
  51       [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ]
  52     end
  53   end
  54
  55   def teardown
  56     redirect_test_io { @server.stop(true) } if @server
  57     @random.close
  58     reset_sig_handlers
  59   end
  60
  61   def test_put
  62     start_server(@sha1_app)
  63     sock = TCPSocket.new(@addr, @port)
  64     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
  65     @count.times do |i|
  66       buf = @random.sysread(@bs)
  67       @sha1.update(buf)
  68       sock.syswrite(buf)
  69     end
  70     read = sock.read.split(/\r\n/)
  71     assert_equal "HTTP/1.1 200 OK", read[0]
  72     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
  73     assert_equal length, resp[:size]
  74     assert_equal @sha1.hexdigest, resp[:sha1]
  75   end
  76
  77   def test_put_content_md5
  78     md5 = Digest::MD5.new
  79     start_server(@sha1_app)
  80     sock = TCPSocket.new(@addr, @port)
  81     sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \
  82                   "Trailer: Content-MD5\r\n\r\n")
  83     @count.times do |i|
  84       buf = @random.sysread(@bs)
  85       @sha1.update(buf)
  86       md5.update(buf)
  87       sock.syswrite("#{'%x' % buf.size}\r\n")
  88       sock.syswrite(buf << "\r\n")
  89     end
  90     sock.syswrite("0\r\n")
  91
  92     content_md5 = [ md5.digest! ].pack('m').strip.freeze
  93     sock.syswrite("Content-MD5: #{content_md5}\r\n\r\n")
  94     read = sock.read.split(/\r\n/)
  95     assert_equal "HTTP/1.1 200 OK", read[0]
  96     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
  97     assert_equal length, resp[:size]
  98     assert_equal @sha1.hexdigest, resp[:sha1]
  99     assert_equal content_md5, resp[:content_md5]
 100   end
 101
 102   def test_put_trickle_small
 103     @count, @bs = 2, 128
 104     start_server(@sha1_app)
 105     assert_equal 256, length
 106     sock = TCPSocket.new(@addr, @port)
 107     hdr = "PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n"
 108     @count.times do
 109       buf = @random.sysread(@bs)
 110       @sha1.update(buf)
 111       hdr << buf
 112       sock.syswrite(hdr)
 113       hdr = ''
 114       sleep 0.6
 115     end
 116     read = sock.read.split(/\r\n/)
 117     assert_equal "HTTP/1.1 200 OK", read[0]
 118     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
 119     assert_equal length, resp[:size]
 120     assert_equal @sha1.hexdigest, resp[:sha1]
 121   end
 122
 123   def test_put_keepalive_truncates_small_overwrite
 124     start_server(@sha1_app)
 125     sock = TCPSocket.new(@addr, @port)
 126     to_upload = length + 1
 127     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{to_upload}\r\n\r\n")
 128     @count.times do
 129       buf = @random.sysread(@bs)
 130       @sha1.update(buf)
 131       sock.syswrite(buf)
 132     end
 133     sock.syswrite('12345') # write 4 bytes more than we expected
 134     @sha1.update('1')
 135
 136     buf = sock.readpartial(4096)
 137     while buf !~ /\r\n\r\n/
 138       buf << sock.readpartial(4096)
 139     end
 140     read = buf.split(/\r\n/)
 141     assert_equal "HTTP/1.1 200 OK", read[0]
 142     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
 143     assert_equal to_upload, resp[:size]
 144     assert_equal @sha1.hexdigest, resp[:sha1]
 145   end
 146
 147   def test_put_excessive_overwrite_closed
 148     start_server(lambda { |env|
 149       while env['rack.input'].read(65536); end
 150       [ 200, @hdr, [] ]
 151     })
 152     sock = TCPSocket.new(@addr, @port)
 153     buf = ' ' * @bs
 154     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
 155
 156     @count.times { sock.syswrite(buf) }
 157     assert_raise(Errno::ECONNRESET, Errno::EPIPE) do
 158       ::Unicorn::Const::CHUNK_SIZE.times { sock.syswrite(buf) }
 159     end
 160     assert_equal "HTTP/1.1 200 OK\r\n", sock.gets
 161   end
 162
 163   # Despite reading numerous articles and inspecting the 1.9.1-p0 C
 164   # source, Eric Wong will never trust that we're always handling
 165   # encoding-aware IO objects correctly.  Thus this test uses shell
 166   # utilities that should always operate on files/sockets on a
 167   # byte-level.
 168   def test_uncomfortable_with_onenine_encodings
 169     # POSIX doesn't require all of these to be present on a system
 170     which('curl') or return
 171     which('sha1sum') or return
 172     which('dd') or return
 173
 174     start_server(@sha1_app)
 175
 176     tmp = Tempfile.new('dd_dest')
 177     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 178                         "bs=#{@bs}", "count=#{@count}"),
 179            "dd #@random to #{tmp}")
 180     sha1_re = %r!\b([a-f0-9]{40})\b!
 181     sha1_out = `sha1sum #{tmp.path}`
 182     assert $?.success?, 'sha1sum ran OK'
 183
 184     assert_match(sha1_re, sha1_out)
 185     sha1 = sha1_re.match(sha1_out)[1]
 186     resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
 187     assert $?.success?, 'curl ran OK'
 188     assert_match(%r!\b#{sha1}\b!, resp)
 189     assert_match(/sysread_read_byte_match/, resp)
 190
 191     # small StringIO path
 192     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 193                         "bs=1024", "count=1"),
 194            "dd #@random to #{tmp}")
 195     sha1_re = %r!\b([a-f0-9]{40})\b!
 196     sha1_out = `sha1sum #{tmp.path}`
 197     assert $?.success?, 'sha1sum ran OK'
 198
 199     assert_match(sha1_re, sha1_out)
 200     sha1 = sha1_re.match(sha1_out)[1]
 201     resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
 202     assert $?.success?, 'curl ran OK'
 203     assert_match(%r!\b#{sha1}\b!, resp)
 204     assert_match(/sysread_read_byte_match/, resp)
 205   end
 206
 207   def test_chunked_upload_via_curl
 208     # POSIX doesn't require all of these to be present on a system
 209     which('curl') or return
 210     which('sha1sum') or return
 211     which('dd') or return
 212
 213     start_server(@sha1_app)
 214
 215     tmp = Tempfile.new('dd_dest')
 216     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 217                         "bs=#{@bs}", "count=#{@count}"),
 218            "dd #@random to #{tmp}")
 219     sha1_re = %r!\b([a-f0-9]{40})\b!
 220     sha1_out = `sha1sum #{tmp.path}`
 221     assert $?.success?, 'sha1sum ran OK'
 222
 223     assert_match(sha1_re, sha1_out)
 224     sha1 = sha1_re.match(sha1_out)[1]
 225     cmd = "curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
 226            -isSf --no-buffer -T- " \
 227           "http://#@addr:#@port/"
 228     resp = Tempfile.new('resp')
 229     resp.sync = true
 230
 231     rd, wr = IO.pipe
 232     wr.sync = rd.sync = true
 233     pid = fork {
 234       STDIN.reopen(rd)
 235       rd.close
 236       wr.close
 237       STDOUT.reopen(resp)
 238       exec cmd
 239     }
 240     rd.close
 241
 242     tmp.rewind
 243     @count.times { |i|
 244       wr.write(tmp.read(@bs))
 245       sleep(rand / 10) if 0 == i % 8
 246     }
 247     wr.close
 248     pid, status = Process.waitpid2(pid)
 249
 250     resp.rewind
 251     resp = resp.read
 252     assert status.success?, 'curl ran OK'
 253     assert_match(%r!\b#{sha1}\b!, resp)
 254     assert_match(/sysread_read_byte_match/, resp)
 255     assert_match(/expect_size_match/, resp)
 256   end
 257
 258   def test_curl_chunked_small
 259     # POSIX doesn't require all of these to be present on a system
 260     which('curl') or return
 261     which('sha1sum') or return
 262     which('dd') or return
 263
 264     start_server(@sha1_app)
 265
 266     tmp = Tempfile.new('dd_dest')
 267     # small StringIO path
 268     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 269                         "bs=1024", "count=1"),
 270            "dd #@random to #{tmp}")
 271     sha1_re = %r!\b([a-f0-9]{40})\b!
 272     sha1_out = `sha1sum #{tmp.path}`
 273     assert $?.success?, 'sha1sum ran OK'
 274
 275     assert_match(sha1_re, sha1_out)
 276     sha1 = sha1_re.match(sha1_out)[1]
 277     resp = `curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
 278             -isSf --no-buffer -T- http://#@addr:#@port/ < #{tmp.path}`
 279     assert $?.success?, 'curl ran OK'
 280     assert_match(%r!\b#{sha1}\b!, resp)
 281     assert_match(/sysread_read_byte_match/, resp)
 282     assert_match(/expect_size_match/, resp)
 283   end
 284
 285   private
 286
 287   def length
 288     @bs * @count
 289   end
 290
 291   def start_server(app)
 292     redirect_test_io do
 293       @server = HttpServer.new(app, :listeners => [ "#{@addr}:#{@port}" ] )
 294       @server.start
 295     end
 296   end
 297
 298 end