test/unit/test_upload.rb

   1 # Copyright (c) 2009 Eric Wong
   2 require 'test/test_helper'
   3 require 'digest/md5'
   4
   5 include Unicorn
   6
   7 class UploadTest < Test::Unit::TestCase
   8
   9   def setup
  10     @addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
  11     @port = unused_port
  12     @hdr = {'Content-Type' => 'text/plain', 'Content-Length' => '0'}
  13     @bs = 4096
  14     @count = 256
  15     @server = nil
  16
  17     # we want random binary data to test 1.9 encoding-aware IO craziness
  18     @random = File.open('/dev/urandom','rb')
  19     @sha1 = Digest::SHA1.new
  20     @sha1_app = lambda do |env|
  21       input = env['rack.input']
  22       resp = {}
  23
  24       @sha1.reset
  25       while buf = input.read(@bs)
  26         @sha1.update(buf)
  27       end
  28       resp[:sha1] = @sha1.hexdigest
  29
  30       # rewind and read again
  31       input.rewind
  32       @sha1.reset
  33       while buf = input.read(@bs)
  34         @sha1.update(buf)
  35       end
  36
  37       if resp[:sha1] == @sha1.hexdigest
  38         resp[:sysread_read_byte_match] = true
  39       end
  40
  41       if expect_size = env['HTTP_X_EXPECT_SIZE']
  42         if expect_size.to_i == input.size
  43           resp[:expect_size_match] = true
  44         end
  45       end
  46       resp[:size] = input.size
  47       resp[:content_md5] = env['HTTP_CONTENT_MD5']
  48
  49       [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ]
  50     end
  51   end
  52
  53   def teardown
  54     redirect_test_io { @server.stop(true) } if @server
  55     @random.close
  56   end
  57
  58   def test_put
  59     start_server(@sha1_app)
  60     sock = TCPSocket.new(@addr, @port)
  61     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
  62     @count.times do |i|
  63       buf = @random.sysread(@bs)
  64       @sha1.update(buf)
  65       sock.syswrite(buf)
  66     end
  67     read = sock.read.split(/\r\n/)
  68     assert_equal "HTTP/1.1 200 OK", read[0]
  69     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
  70     assert_equal length, resp[:size]
  71     assert_equal @sha1.hexdigest, resp[:sha1]
  72   end
  73
  74   def test_put_content_md5
  75     md5 = Digest::MD5.new
  76     start_server(@sha1_app)
  77     sock = TCPSocket.new(@addr, @port)
  78     sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \
  79                   "Trailer: Content-MD5\r\n\r\n")
  80     @count.times do |i|
  81       buf = @random.sysread(@bs)
  82       @sha1.update(buf)
  83       md5.update(buf)
  84       sock.syswrite("#{'%x' % buf.size}\r\n")
  85       sock.syswrite(buf << "\r\n")
  86     end
  87     sock.syswrite("0\r\n")
  88
  89     content_md5 = [ md5.digest! ].pack('m').strip.freeze
  90     sock.syswrite("Content-MD5: #{content_md5}\r\n\r\n")
  91     read = sock.read.split(/\r\n/)
  92     assert_equal "HTTP/1.1 200 OK", read[0]
  93     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
  94     assert_equal length, resp[:size]
  95     assert_equal @sha1.hexdigest, resp[:sha1]
  96     assert_equal content_md5, resp[:content_md5]
  97   end
  98
  99   def test_put_trickle_small
 100     @count, @bs = 2, 128
 101     start_server(@sha1_app)
 102     assert_equal 256, length
 103     sock = TCPSocket.new(@addr, @port)
 104     hdr = "PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n"
 105     @count.times do
 106       buf = @random.sysread(@bs)
 107       @sha1.update(buf)
 108       hdr << buf
 109       sock.syswrite(hdr)
 110       hdr = ''
 111       sleep 0.6
 112     end
 113     read = sock.read.split(/\r\n/)
 114     assert_equal "HTTP/1.1 200 OK", read[0]
 115     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
 116     assert_equal length, resp[:size]
 117     assert_equal @sha1.hexdigest, resp[:sha1]
 118   end
 119
 120   def test_put_keepalive_truncates_small_overwrite
 121     start_server(@sha1_app)
 122     sock = TCPSocket.new(@addr, @port)
 123     to_upload = length + 1
 124     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{to_upload}\r\n\r\n")
 125     @count.times do
 126       buf = @random.sysread(@bs)
 127       @sha1.update(buf)
 128       sock.syswrite(buf)
 129     end
 130     sock.syswrite('12345') # write 4 bytes more than we expected
 131     @sha1.update('1')
 132
 133     buf = sock.readpartial(4096)
 134     while buf !~ /\r\n\r\n/
 135       buf << sock.readpartial(4096)
 136     end
 137     read = buf.split(/\r\n/)
 138     assert_equal "HTTP/1.1 200 OK", read[0]
 139     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
 140     assert_equal to_upload, resp[:size]
 141     assert_equal @sha1.hexdigest, resp[:sha1]
 142   end
 143
 144   def test_put_excessive_overwrite_closed
 145     start_server(lambda { |env|
 146       while env['rack.input'].read(65536); end
 147       [ 200, @hdr, [] ]
 148     })
 149     sock = TCPSocket.new(@addr, @port)
 150     buf = ' ' * @bs
 151     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
 152
 153     @count.times { sock.syswrite(buf) }
 154     assert_raise(Errno::ECONNRESET, Errno::EPIPE) do
 155       ::Unicorn::Const::CHUNK_SIZE.times { sock.syswrite(buf) }
 156     end
 157     assert_equal "HTTP/1.1 200 OK\r\n", sock.gets
 158   end
 159
 160   # Despite reading numerous articles and inspecting the 1.9.1-p0 C
 161   # source, Eric Wong will never trust that we're always handling
 162   # encoding-aware IO objects correctly.  Thus this test uses shell
 163   # utilities that should always operate on files/sockets on a
 164   # byte-level.
 165   def test_uncomfortable_with_onenine_encodings
 166     # POSIX doesn't require all of these to be present on a system
 167     which('curl') or return
 168     which('sha1sum') or return
 169     which('dd') or return
 170
 171     start_server(@sha1_app)
 172
 173     tmp = Tempfile.new('dd_dest')
 174     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 175                         "bs=#{@bs}", "count=#{@count}"),
 176            "dd #@random to #{tmp}")
 177     sha1_re = %r!\b([a-f0-9]{40})\b!
 178     sha1_out = `sha1sum #{tmp.path}`
 179     assert $?.success?, 'sha1sum ran OK'
 180
 181     assert_match(sha1_re, sha1_out)
 182     sha1 = sha1_re.match(sha1_out)[1]
 183     resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
 184     assert $?.success?, 'curl ran OK'
 185     assert_match(%r!\b#{sha1}\b!, resp)
 186     assert_match(/sysread_read_byte_match/, resp)
 187
 188     # small StringIO path
 189     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 190                         "bs=1024", "count=1"),
 191            "dd #@random to #{tmp}")
 192     sha1_re = %r!\b([a-f0-9]{40})\b!
 193     sha1_out = `sha1sum #{tmp.path}`
 194     assert $?.success?, 'sha1sum ran OK'
 195
 196     assert_match(sha1_re, sha1_out)
 197     sha1 = sha1_re.match(sha1_out)[1]
 198     resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
 199     assert $?.success?, 'curl ran OK'
 200     assert_match(%r!\b#{sha1}\b!, resp)
 201     assert_match(/sysread_read_byte_match/, resp)
 202   end
 203
 204   def test_chunked_upload_via_curl
 205     # POSIX doesn't require all of these to be present on a system
 206     which('curl') or return
 207     which('sha1sum') or return
 208     which('dd') or return
 209
 210     start_server(@sha1_app)
 211
 212     tmp = Tempfile.new('dd_dest')
 213     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 214                         "bs=#{@bs}", "count=#{@count}"),
 215            "dd #@random to #{tmp}")
 216     sha1_re = %r!\b([a-f0-9]{40})\b!
 217     sha1_out = `sha1sum #{tmp.path}`
 218     assert $?.success?, 'sha1sum ran OK'
 219
 220     assert_match(sha1_re, sha1_out)
 221     sha1 = sha1_re.match(sha1_out)[1]
 222     cmd = "curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
 223            -isSf --no-buffer -T- " \
 224           "http://#@addr:#@port/"
 225     resp = Tempfile.new('resp')
 226     resp.sync = true
 227
 228     rd, wr = IO.pipe
 229     wr.sync = rd.sync = true
 230     pid = fork {
 231       STDIN.reopen(rd)
 232       rd.close
 233       wr.close
 234       STDOUT.reopen(resp)
 235       exec cmd
 236     }
 237     rd.close
 238
 239     tmp.rewind
 240     @count.times { |i|
 241       wr.write(tmp.read(@bs))
 242       sleep(rand / 10) if 0 == i % 8
 243     }
 244     wr.close
 245     pid, status = Process.waitpid2(pid)
 246
 247     resp.rewind
 248     resp = resp.read
 249     assert status.success?, 'curl ran OK'
 250     assert_match(%r!\b#{sha1}\b!, resp)
 251     assert_match(/sysread_read_byte_match/, resp)
 252     assert_match(/expect_size_match/, resp)
 253   end
 254
 255   def test_curl_chunked_small
 256     # POSIX doesn't require all of these to be present on a system
 257     which('curl') or return
 258     which('sha1sum') or return
 259     which('dd') or return
 260
 261     start_server(@sha1_app)
 262
 263     tmp = Tempfile.new('dd_dest')
 264     # small StringIO path
 265     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 266                         "bs=1024", "count=1"),
 267            "dd #@random to #{tmp}")
 268     sha1_re = %r!\b([a-f0-9]{40})\b!
 269     sha1_out = `sha1sum #{tmp.path}`
 270     assert $?.success?, 'sha1sum ran OK'
 271
 272     assert_match(sha1_re, sha1_out)
 273     sha1 = sha1_re.match(sha1_out)[1]
 274     resp = `curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
 275             -isSf --no-buffer -T- http://#@addr:#@port/ < #{tmp.path}`
 276     assert $?.success?, 'curl ran OK'
 277     assert_match(%r!\b#{sha1}\b!, resp)
 278     assert_match(/sysread_read_byte_match/, resp)
 279     assert_match(/expect_size_match/, resp)
 280   end
 281
 282   private
 283
 284   def length
 285     @bs * @count
 286   end
 287
 288   def start_server(app)
 289     redirect_test_io do
 290       @server = HttpServer.new(app, :listeners => [ "#{@addr}:#{@port}" ] )
 291       @server.start
 292     end
 293   end
 294
 295 end