test/unit/test_upload.rb

   1 # -*- encoding: binary -*-
   2
   3 # Copyright (c) 2009 Eric Wong
   4 require 'test/test_helper'
   5 require 'digest/md5'
   6
   7 include Unicorn
   8
   9 class UploadTest < Test::Unit::TestCase
  10
  11   def setup
  12     @addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
  13     @port = unused_port
  14     @hdr = {'Content-Type' => 'text/plain', 'Content-Length' => '0'}
  15     @bs = 4096
  16     @count = 256
  17     @server = nil
  18
  19     # we want random binary data to test 1.9 encoding-aware IO craziness
  20     @random = File.open('/dev/urandom','rb')
  21     @sha1 = Digest::SHA1.new
  22     @sha1_app = lambda do |env|
  23       input = env['rack.input']
  24       resp = {}
  25
  26       @sha1.reset
  27       while buf = input.read(@bs)
  28         @sha1.update(buf)
  29       end
  30       resp[:sha1] = @sha1.hexdigest
  31
  32       # rewind and read again
  33       input.rewind
  34       @sha1.reset
  35       while buf = input.read(@bs)
  36         @sha1.update(buf)
  37       end
  38
  39       if resp[:sha1] == @sha1.hexdigest
  40         resp[:sysread_read_byte_match] = true
  41       end
  42
  43       if expect_size = env['HTTP_X_EXPECT_SIZE']
  44         if expect_size.to_i == input.size
  45           resp[:expect_size_match] = true
  46         end
  47       end
  48       resp[:size] = input.size
  49       resp[:content_md5] = env['HTTP_CONTENT_MD5']
  50
  51       [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ]
  52     end
  53   end
  54
  55   def teardown
  56     redirect_test_io { @server.stop(false) } if @server
  57     @random.close
  58     reset_sig_handlers
  59   end
  60
  61   def test_put
  62     start_server(@sha1_app)
  63     sock = TCPSocket.new(@addr, @port)
  64     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
  65     @count.times do |i|
  66       buf = @random.sysread(@bs)
  67       @sha1.update(buf)
  68       sock.syswrite(buf)
  69     end
  70     read = sock.read.split(/\r\n/)
  71     assert_equal "HTTP/1.1 200 OK", read[0]
  72     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
  73     assert_equal length, resp[:size]
  74     assert_equal @sha1.hexdigest, resp[:sha1]
  75   end
  76
  77   def test_put_content_md5
  78     md5 = Digest::MD5.new
  79     start_server(@sha1_app)
  80     sock = TCPSocket.new(@addr, @port)
  81     sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \
  82                   "Trailer: Content-MD5\r\n\r\n")
  83     @count.times do |i|
  84       buf = @random.sysread(@bs)
  85       @sha1.update(buf)
  86       md5.update(buf)
  87       sock.syswrite("#{'%x' % buf.size}\r\n")
  88       sock.syswrite(buf << "\r\n")
  89     end
  90     sock.syswrite("0\r\n")
  91
  92     content_md5 = [ md5.digest! ].pack('m').strip.freeze
  93     sock.syswrite("Content-MD5: #{content_md5}\r\n\r\n")
  94     read = sock.read.split(/\r\n/)
  95     assert_equal "HTTP/1.1 200 OK", read[0]
  96     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
  97     assert_equal length, resp[:size]
  98     assert_equal @sha1.hexdigest, resp[:sha1]
  99     assert_equal content_md5, resp[:content_md5]
 100   end
 101
 102   def test_put_trickle_small
 103     @count, @bs = 2, 128
 104     start_server(@sha1_app)
 105     assert_equal 256, length
 106     sock = TCPSocket.new(@addr, @port)
 107     hdr = "PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n"
 108     @count.times do
 109       buf = @random.sysread(@bs)
 110       @sha1.update(buf)
 111       hdr << buf
 112       sock.syswrite(hdr)
 113       hdr = ''
 114       sleep 0.6
 115     end
 116     read = sock.read.split(/\r\n/)
 117     assert_equal "HTTP/1.1 200 OK", read[0]
 118     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
 119     assert_equal length, resp[:size]
 120     assert_equal @sha1.hexdigest, resp[:sha1]
 121   end
 122
 123   def test_put_keepalive_truncates_small_overwrite
 124     start_server(@sha1_app)
 125     sock = TCPSocket.new(@addr, @port)
 126     to_upload = length + 1
 127     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{to_upload}\r\n\r\n")
 128     @count.times do
 129       buf = @random.sysread(@bs)
 130       @sha1.update(buf)
 131       sock.syswrite(buf)
 132     end
 133     sock.syswrite('12345') # write 4 bytes more than we expected
 134     @sha1.update('1')
 135
 136     buf = sock.readpartial(4096)
 137     while buf !~ /\r\n\r\n/
 138       buf << sock.readpartial(4096)
 139     end
 140     read = buf.split(/\r\n/)
 141     assert_equal "HTTP/1.1 200 OK", read[0]
 142     resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
 143     assert_equal to_upload, resp[:size]
 144     assert_equal @sha1.hexdigest, resp[:sha1]
 145   end
 146
 147   def test_put_excessive_overwrite_closed
 148     tmp = Tempfile.new('overwrite_check')
 149     tmp.sync = true
 150     start_server(lambda { |env|
 151       nr = 0
 152       while buf = env['rack.input'].read(65536)
 153         nr += buf.size
 154       end
 155       tmp.write(nr.to_s)
 156       [ 200, @hdr, [] ]
 157     })
 158     sock = TCPSocket.new(@addr, @port)
 159     buf = ' ' * @bs
 160     sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
 161
 162     @count.times { sock.syswrite(buf) }
 163     assert_raise(Errno::ECONNRESET, Errno::EPIPE) do
 164       ::Unicorn::Const::CHUNK_SIZE.times { sock.syswrite(buf) }
 165     end
 166     sock.gets
 167     tmp.rewind
 168     assert_equal length, tmp.read.to_i
 169   end
 170
 171   # Despite reading numerous articles and inspecting the 1.9.1-p0 C
 172   # source, Eric Wong will never trust that we're always handling
 173   # encoding-aware IO objects correctly.  Thus this test uses shell
 174   # utilities that should always operate on files/sockets on a
 175   # byte-level.
 176   def test_uncomfortable_with_onenine_encodings
 177     # POSIX doesn't require all of these to be present on a system
 178     which('curl') or return
 179     which('sha1sum') or return
 180     which('dd') or return
 181
 182     start_server(@sha1_app)
 183
 184     tmp = Tempfile.new('dd_dest')
 185     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 186                         "bs=#{@bs}", "count=#{@count}"),
 187            "dd #@random to #{tmp}")
 188     sha1_re = %r!\b([a-f0-9]{40})\b!
 189     sha1_out = `sha1sum #{tmp.path}`
 190     assert $?.success?, 'sha1sum ran OK'
 191
 192     assert_match(sha1_re, sha1_out)
 193     sha1 = sha1_re.match(sha1_out)[1]
 194     resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
 195     assert $?.success?, 'curl ran OK'
 196     assert_match(%r!\b#{sha1}\b!, resp)
 197     assert_match(/sysread_read_byte_match/, resp)
 198
 199     # small StringIO path
 200     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 201                         "bs=1024", "count=1"),
 202            "dd #@random to #{tmp}")
 203     sha1_re = %r!\b([a-f0-9]{40})\b!
 204     sha1_out = `sha1sum #{tmp.path}`
 205     assert $?.success?, 'sha1sum ran OK'
 206
 207     assert_match(sha1_re, sha1_out)
 208     sha1 = sha1_re.match(sha1_out)[1]
 209     resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
 210     assert $?.success?, 'curl ran OK'
 211     assert_match(%r!\b#{sha1}\b!, resp)
 212     assert_match(/sysread_read_byte_match/, resp)
 213   end
 214
 215   def test_chunked_upload_via_curl
 216     # POSIX doesn't require all of these to be present on a system
 217     which('curl') or return
 218     which('sha1sum') or return
 219     which('dd') or return
 220
 221     start_server(@sha1_app)
 222
 223     tmp = Tempfile.new('dd_dest')
 224     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 225                         "bs=#{@bs}", "count=#{@count}"),
 226            "dd #@random to #{tmp}")
 227     sha1_re = %r!\b([a-f0-9]{40})\b!
 228     sha1_out = `sha1sum #{tmp.path}`
 229     assert $?.success?, 'sha1sum ran OK'
 230
 231     assert_match(sha1_re, sha1_out)
 232     sha1 = sha1_re.match(sha1_out)[1]
 233     cmd = "curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
 234            -isSf --no-buffer -T- " \
 235           "http://#@addr:#@port/"
 236     resp = Tempfile.new('resp')
 237     resp.sync = true
 238
 239     rd, wr = IO.pipe
 240     wr.sync = rd.sync = true
 241     pid = fork {
 242       STDIN.reopen(rd)
 243       rd.close
 244       wr.close
 245       STDOUT.reopen(resp)
 246       exec cmd
 247     }
 248     rd.close
 249
 250     tmp.rewind
 251     @count.times { |i|
 252       wr.write(tmp.read(@bs))
 253       sleep(rand / 10) if 0 == i % 8
 254     }
 255     wr.close
 256     pid, status = Process.waitpid2(pid)
 257
 258     resp.rewind
 259     resp = resp.read
 260     assert status.success?, 'curl ran OK'
 261     assert_match(%r!\b#{sha1}\b!, resp)
 262     assert_match(/sysread_read_byte_match/, resp)
 263     assert_match(/expect_size_match/, resp)
 264   end
 265
 266   def test_curl_chunked_small
 267     # POSIX doesn't require all of these to be present on a system
 268     which('curl') or return
 269     which('sha1sum') or return
 270     which('dd') or return
 271
 272     start_server(@sha1_app)
 273
 274     tmp = Tempfile.new('dd_dest')
 275     # small StringIO path
 276     assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
 277                         "bs=1024", "count=1"),
 278            "dd #@random to #{tmp}")
 279     sha1_re = %r!\b([a-f0-9]{40})\b!
 280     sha1_out = `sha1sum #{tmp.path}`
 281     assert $?.success?, 'sha1sum ran OK'
 282
 283     assert_match(sha1_re, sha1_out)
 284     sha1 = sha1_re.match(sha1_out)[1]
 285     resp = `curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
 286             -isSf --no-buffer -T- http://#@addr:#@port/ < #{tmp.path}`
 287     assert $?.success?, 'curl ran OK'
 288     assert_match(%r!\b#{sha1}\b!, resp)
 289     assert_match(/sysread_read_byte_match/, resp)
 290     assert_match(/expect_size_match/, resp)
 291   end
 292
 293   private
 294
 295   def length
 296     @bs * @count
 297   end
 298
 299   def start_server(app)
 300     redirect_test_io do
 301       @server = HttpServer.new(app, :listeners => [ "#{@addr}:#{@port}" ] )
 302       @server.start
 303     end
 304   end
 305
 306 end