test/open-uri/test_open-uri.rb

   1 # frozen_string_literal: true
   2 require 'test/unit'
   3 require 'open-uri'
   4 require 'webrick'
   5 require 'webrick/httpproxy'
   6 begin
   7   require 'zlib'
   8 rescue LoadError
   9 end
  10
  11 class TestOpenURI < Test::Unit::TestCase
  12
  13   NullLog = Object.new
  14   def NullLog.<<(arg)
  15     #puts arg if / INFO / !~ arg
  16   end
  17
  18   def with_http(log_tester=lambda {|log| assert_equal([], log) })
  19     log = []
  20     logger = WEBrick::Log.new(log, WEBrick::BasicLog::WARN)
  21     Dir.mktmpdir {|dr|
  22       srv = WEBrick::HTTPServer.new({
  23         :DocumentRoot => dr,
  24         :ServerType => Thread,
  25         :Logger => logger,
  26         :AccessLog => [[NullLog, ""]],
  27         :BindAddress => '127.0.0.1',
  28         :Port => 0})
  29       _, port, _, host = srv.listeners[0].addr
  30       server_thread = srv.start
  31       server_thread2 = Thread.new {
  32         server_thread.join
  33         if log_tester
  34           log_tester.call(log)
  35         end
  36       }
  37       client_thread = Thread.new {
  38         begin
  39           yield srv, dr, "http://#{host}:#{port}", server_thread, log
  40         ensure
  41           srv.shutdown
  42         end
  43       }
  44       assert_join_threads([client_thread, server_thread2])
  45     }
  46   ensure
  47     WEBrick::Utils::TimeoutHandler.terminate
  48   end
  49
  50   def with_env(h)
  51     begin
  52       old = {}
  53       h.each_key {|k| old[k] = ENV[k] }
  54       h.each {|k, v| ENV[k] = v }
  55       yield
  56     ensure
  57       h.each_key {|k| ENV[k] = old[k] }
  58     end
  59   end
  60
  61   def setup
  62     @proxies = %w[http_proxy HTTP_PROXY ftp_proxy FTP_PROXY no_proxy]
  63     @old_proxies = @proxies.map {|k| ENV[k] }
  64     @proxies.each {|k| ENV[k] = nil }
  65   end
  66
  67   def teardown
  68     @proxies.each_with_index {|k, i| ENV[k] = @old_proxies[i] }
  69   end
  70
  71   def test_200_uri_open
  72     with_http {|srv, dr, url|
  73       srv.mount_proc("/urifoo200", lambda { |req, res| res.body = "urifoo200" } )
  74       URI.open("#{url}/urifoo200") {|f|
  75         assert_equal("200", f.status[0])
  76         assert_equal("urifoo200", f.read)
  77       }
  78     }
  79   end
  80
  81   def test_200
  82     with_http {|srv, dr, url|
  83       srv.mount_proc("/foo200", lambda { |req, res| res.body = "foo200" } )
  84       URI.open("#{url}/foo200") {|f|
  85         assert_equal("200", f.status[0])
  86         assert_equal("foo200", f.read)
  87       }
  88     }
  89   end
  90
  91   def test_200big
  92     with_http {|srv, dr, url|
  93       content = "foo200big"*10240
  94       srv.mount_proc("/foo200big", lambda { |req, res| res.body = content } )
  95       URI.open("#{url}/foo200big") {|f|
  96         assert_equal("200", f.status[0])
  97         assert_equal(content, f.read)
  98       }
  99     }
 100   end
 101
 102   def test_404
 103     log_tester = lambda {|server_log|
 104       assert_equal(1, server_log.length)
 105       assert_match(%r{ERROR `/not-exist' not found}, server_log[0])
 106     }
 107     with_http(log_tester) {|srv, dr, url, server_thread, server_log|
 108       exc = assert_raise(OpenURI::HTTPError) { URI.open("#{url}/not-exist") {} }
 109       assert_equal("404", exc.io.status[0])
 110     }
 111   end
 112
 113   def test_open_uri
 114     with_http {|srv, dr, url|
 115       srv.mount_proc("/foo_ou", lambda { |req, res| res.body = "foo_ou" } )
 116       u = URI("#{url}/foo_ou")
 117       URI.open(u) {|f|
 118         assert_equal("200", f.status[0])
 119         assert_equal("foo_ou", f.read)
 120       }
 121     }
 122   end
 123
 124   def test_open_too_many_arg
 125     assert_raise(ArgumentError) { URI.open("http://192.0.2.1/tma", "r", 0666, :extra) {} }
 126   end
 127
 128   def test_read_timeout
 129     TCPServer.open("127.0.0.1", 0) {|serv|
 130       port = serv.addr[1]
 131       th = Thread.new {
 132         sock = serv.accept
 133         begin
 134           req = sock.gets("\r\n\r\n")
 135           assert_match(%r{\AGET /foo/bar }, req)
 136           sock.print "HTTP/1.0 200 OK\r\n"
 137           sock.print "Content-Length: 4\r\n\r\n"
 138           sleep 1
 139           sock.print "ab\r\n"
 140         ensure
 141           sock.close
 142         end
 143       }
 144       begin
 145         assert_raise(Net::ReadTimeout) { URI("http://127.0.0.1:#{port}/foo/bar").read(:read_timeout=>0.1) }
 146       ensure
 147         Thread.kill(th)
 148         th.join
 149       end
 150     }
 151   end
 152
 153   def test_open_timeout
 154     assert_raise(Net::OpenTimeout) do
 155       URI("http://example.com/").read(open_timeout: 0.000001)
 156     end if false # avoid external resources in tests
 157
 158     with_http {|srv, dr, url|
 159       url += '/'
 160       srv.mount_proc('/', lambda { |_, res| res.body = 'hi' })
 161       begin
 162         URI(url).read(open_timeout: 0.000001)
 163       rescue Net::OpenTimeout
 164         # not guaranteed to fire, since the kernel negotiates the
 165         # TCP connection even if the server thread is sleeping
 166       end
 167       assert_equal 'hi', URI(url).read(open_timeout: 60), 'should not timeout'
 168     }
 169   end
 170
 171   def test_invalid_option
 172     assert_raise(ArgumentError) { URI.open("http://127.0.0.1/", :invalid_option=>true) {} }
 173   end
 174
 175   def test_pass_keywords
 176     begin
 177       f = URI.open(File::NULL, mode: 0666)
 178       assert_kind_of File, f
 179     ensure
 180       f&.close
 181     end
 182
 183     o = Object.new
 184     def o.open(foo: ) foo end
 185     assert_equal 1, URI.open(o, foo: 1)
 186   end
 187
 188   def test_mode
 189     with_http {|srv, dr, url|
 190       srv.mount_proc("/mode", lambda { |req, res| res.body = "mode" } )
 191       URI.open("#{url}/mode", "r") {|f|
 192         assert_equal("200", f.status[0])
 193         assert_equal("mode", f.read)
 194       }
 195       URI.open("#{url}/mode", "r", 0600) {|f|
 196         assert_equal("200", f.status[0])
 197         assert_equal("mode", f.read)
 198       }
 199       assert_raise(ArgumentError) { URI.open("#{url}/mode", "a") {} }
 200       URI.open("#{url}/mode", "r:us-ascii") {|f|
 201         assert_equal(Encoding::US_ASCII, f.read.encoding)
 202       }
 203       URI.open("#{url}/mode", "r:utf-8") {|f|
 204         assert_equal(Encoding::UTF_8, f.read.encoding)
 205       }
 206       assert_raise(ArgumentError) { URI.open("#{url}/mode", "r:invalid-encoding") {} }
 207     }
 208   end
 209
 210   def test_without_block
 211     with_http {|srv, dr, url|
 212       srv.mount_proc("/without_block", lambda { |req, res| res.body = "without_block" } )
 213       begin
 214         f = URI.open("#{url}/without_block")
 215         assert_equal("200", f.status[0])
 216         assert_equal("without_block", f.read)
 217       ensure
 218         f.close if f && !f.closed?
 219       end
 220     }
 221   end
 222
 223   def test_close_in_block_small
 224     with_http {|srv, dr, url|
 225       srv.mount_proc("/close200", lambda { |req, res| res.body = "close200" } )
 226       assert_nothing_raised {
 227         URI.open("#{url}/close200") {|f|
 228           f.close
 229         }
 230       }
 231     }
 232   end
 233
 234   def test_close_in_block_big
 235     with_http {|srv, dr, url|
 236       content = "close200big"*10240
 237       srv.mount_proc("/close200big", lambda { |req, res| res.body = content } )
 238       assert_nothing_raised {
 239         URI.open("#{url}/close200big") {|f|
 240           f.close
 241         }
 242       }
 243     }
 244   end
 245
 246   def test_header
 247     myheader1 = 'barrrr'
 248     myheader2 = nil
 249     with_http {|srv, dr, url|
 250       srv.mount_proc("/h/") {|req, res| myheader2 = req['myheader']; res.body = "foo" }
 251       URI.open("#{url}/h/", 'MyHeader'=>myheader1) {|f|
 252         assert_equal("foo", f.read)
 253         assert_equal(myheader1, myheader2)
 254       }
 255     }
 256   end
 257
 258   def test_multi_proxy_opt
 259     assert_raise(ArgumentError) {
 260       URI.open("http://127.0.0.1/", :proxy_http_basic_authentication=>true, :proxy=>true) {}
 261     }
 262   end
 263
 264   def test_non_http_proxy
 265     assert_raise(RuntimeError) {
 266       URI.open("http://127.0.0.1/", :proxy=>URI("ftp://127.0.0.1/")) {}
 267     }
 268   end
 269
 270   def test_proxy
 271     with_http {|srv, dr, url|
 272       proxy_log = StringIO.new(''.dup)
 273       proxy_logger = WEBrick::Log.new(proxy_log, WEBrick::BasicLog::WARN)
 274       proxy_auth_log = ''.dup
 275       proxy = WEBrick::HTTPProxyServer.new({
 276         :ServerType => Thread,
 277         :Logger => proxy_logger,
 278         :AccessLog => [[NullLog, ""]],
 279         :ProxyAuthProc => lambda {|req, res|
 280           proxy_auth_log << req.request_line
 281         },
 282         :BindAddress => '127.0.0.1',
 283         :Port => 0})
 284       _, proxy_port, _, proxy_host = proxy.listeners[0].addr
 285       proxy_url = "http://#{proxy_host}:#{proxy_port}/"
 286       begin
 287         proxy_thread = proxy.start
 288         srv.mount_proc("/proxy", lambda { |req, res| res.body = "proxy" } )
 289         URI.open("#{url}/proxy", :proxy=>proxy_url) {|f|
 290           assert_equal("200", f.status[0])
 291           assert_equal("proxy", f.read)
 292         }
 293         assert_match(/#{Regexp.quote url}/, proxy_auth_log); proxy_auth_log.clear
 294         URI.open("#{url}/proxy", :proxy=>URI(proxy_url)) {|f|
 295           assert_equal("200", f.status[0])
 296           assert_equal("proxy", f.read)
 297         }
 298         assert_match(/#{Regexp.quote url}/, proxy_auth_log); proxy_auth_log.clear
 299         URI.open("#{url}/proxy", :proxy=>nil) {|f|
 300           assert_equal("200", f.status[0])
 301           assert_equal("proxy", f.read)
 302         }
 303         assert_equal("", proxy_auth_log); proxy_auth_log.clear
 304         assert_raise(ArgumentError) {
 305           URI.open("#{url}/proxy", :proxy=>:invalid) {}
 306         }
 307         assert_equal("", proxy_auth_log); proxy_auth_log.clear
 308         with_env("http_proxy"=>proxy_url) {
 309           # should not use proxy for 127.0.0.0/8.
 310           URI.open("#{url}/proxy") {|f|
 311             assert_equal("200", f.status[0])
 312             assert_equal("proxy", f.read)
 313           }
 314         }
 315         assert_equal("", proxy_auth_log); proxy_auth_log.clear
 316       ensure
 317         proxy.shutdown
 318         proxy_thread.join
 319       end
 320       assert_equal("", proxy_log.string)
 321     }
 322   end
 323
 324   def test_proxy_http_basic_authentication_failure
 325     with_http {|srv, dr, url|
 326       proxy_log = StringIO.new(''.dup)
 327       proxy_logger = WEBrick::Log.new(proxy_log, WEBrick::BasicLog::WARN)
 328       proxy_auth_log = ''.dup
 329       proxy = WEBrick::HTTPProxyServer.new({
 330         :ServerType => Thread,
 331         :Logger => proxy_logger,
 332         :AccessLog => [[NullLog, ""]],
 333         :ProxyAuthProc => lambda {|req, res|
 334           proxy_auth_log << req.request_line
 335           if req["Proxy-Authorization"] != "Basic #{['user:pass'].pack('m').chomp}"
 336             raise WEBrick::HTTPStatus::ProxyAuthenticationRequired
 337           end
 338         },
 339         :BindAddress => '127.0.0.1',
 340         :Port => 0})
 341       _, proxy_port, _, proxy_host = proxy.listeners[0].addr
 342       proxy_url = "http://#{proxy_host}:#{proxy_port}/"
 343       begin
 344         th = proxy.start
 345         srv.mount_proc("/proxy", lambda { |req, res| res.body = "proxy" } )
 346         exc = assert_raise(OpenURI::HTTPError) { URI.open("#{url}/proxy", :proxy=>proxy_url) {} }
 347         assert_equal("407", exc.io.status[0])
 348         assert_match(/#{Regexp.quote url}/, proxy_auth_log); proxy_auth_log.clear
 349       ensure
 350         proxy.shutdown
 351         th.join
 352       end
 353       assert_match(/ERROR WEBrick::HTTPStatus::ProxyAuthenticationRequired/, proxy_log.string)
 354     }
 355   end
 356
 357   def test_proxy_http_basic_authentication_success
 358     with_http {|srv, dr, url|
 359       proxy_log = StringIO.new(''.dup)
 360       proxy_logger = WEBrick::Log.new(proxy_log, WEBrick::BasicLog::WARN)
 361       proxy_auth_log = ''.dup
 362       proxy = WEBrick::HTTPProxyServer.new({
 363         :ServerType => Thread,
 364         :Logger => proxy_logger,
 365         :AccessLog => [[NullLog, ""]],
 366         :ProxyAuthProc => lambda {|req, res|
 367           proxy_auth_log << req.request_line
 368           if req["Proxy-Authorization"] != "Basic #{['user:pass'].pack('m').chomp}"
 369             raise WEBrick::HTTPStatus::ProxyAuthenticationRequired
 370           end
 371         },
 372         :BindAddress => '127.0.0.1',
 373         :Port => 0})
 374       _, proxy_port, _, proxy_host = proxy.listeners[0].addr
 375       proxy_url = "http://#{proxy_host}:#{proxy_port}/"
 376       begin
 377         th = proxy.start
 378         srv.mount_proc("/proxy", lambda { |req, res| res.body = "proxy" } )
 379         URI.open("#{url}/proxy",
 380             :proxy_http_basic_authentication=>[proxy_url, "user", "pass"]) {|f|
 381           assert_equal("200", f.status[0])
 382           assert_equal("proxy", f.read)
 383         }
 384         assert_match(/#{Regexp.quote url}/, proxy_auth_log); proxy_auth_log.clear
 385         assert_raise(ArgumentError) {
 386           URI.open("#{url}/proxy",
 387               :proxy_http_basic_authentication=>[true, "user", "pass"]) {}
 388         }
 389         assert_equal("", proxy_auth_log); proxy_auth_log.clear
 390       ensure
 391         proxy.shutdown
 392         th.join
 393       end
 394       assert_equal("", proxy_log.string)
 395     }
 396   end
 397
 398   def test_authenticated_proxy_http_basic_authentication_success
 399     with_http {|srv, dr, url|
 400       proxy_log = StringIO.new(''.dup)
 401       proxy_logger = WEBrick::Log.new(proxy_log, WEBrick::BasicLog::WARN)
 402       proxy_auth_log = ''.dup
 403       proxy = WEBrick::HTTPProxyServer.new({
 404         :ServerType => Thread,
 405         :Logger => proxy_logger,
 406         :AccessLog => [[NullLog, ""]],
 407         :ProxyAuthProc => lambda {|req, res|
 408           proxy_auth_log << req.request_line
 409           if req["Proxy-Authorization"] != "Basic #{['user:pass'].pack('m').chomp}"
 410             raise WEBrick::HTTPStatus::ProxyAuthenticationRequired
 411           end
 412         },
 413         :BindAddress => '127.0.0.1',
 414         :Port => 0})
 415       _, proxy_port, _, proxy_host = proxy.listeners[0].addr
 416       proxy_url = "http://user:pass@#{proxy_host}:#{proxy_port}/"
 417       begin
 418         th = proxy.start
 419         srv.mount_proc("/proxy", lambda { |req, res| res.body = "proxy" } )
 420         URI.open("#{url}/proxy", :proxy => proxy_url) {|f|
 421           assert_equal("200", f.status[0])
 422           assert_equal("proxy", f.read)
 423         }
 424         assert_match(/#{Regexp.quote url}/, proxy_auth_log); proxy_auth_log.clear
 425         assert_equal("", proxy_auth_log); proxy_auth_log.clear
 426       ensure
 427         proxy.shutdown
 428         th.join
 429       end
 430       assert_equal("", proxy_log.string)
 431     }
 432   end
 433
 434   def test_redirect
 435     with_http {|srv, dr, url|
 436       srv.mount_proc("/r1/") {|req, res| res.status = 301; res["location"] = "#{url}/r2"; res.body = "r1" }
 437       srv.mount_proc("/r2/") {|req, res| res.body = "r2" }
 438       srv.mount_proc("/to-file/") {|req, res| res.status = 301; res["location"] = "file:///foo" }
 439       URI.open("#{url}/r1/") {|f|
 440         assert_equal("#{url}/r2", f.base_uri.to_s)
 441         assert_equal("r2", f.read)
 442       }
 443       assert_raise(OpenURI::HTTPRedirect) { URI.open("#{url}/r1/", :redirect=>false) {} }
 444       assert_raise(RuntimeError) { URI.open("#{url}/to-file/") {} }
 445     }
 446   end
 447
 448   def test_redirect_loop
 449     with_http {|srv, dr, url|
 450       srv.mount_proc("/r1/") {|req, res| res.status = 301; res["location"] = "#{url}/r2"; res.body = "r1" }
 451       srv.mount_proc("/r2/") {|req, res| res.status = 301; res["location"] = "#{url}/r1"; res.body = "r2" }
 452       assert_raise(RuntimeError) { URI.open("#{url}/r1/") {} }
 453     }
 454   end
 455
 456   def test_redirect_relative
 457     TCPServer.open("127.0.0.1", 0) {|serv|
 458       port = serv.addr[1]
 459       th = Thread.new {
 460         sock = serv.accept
 461         begin
 462           req = sock.gets("\r\n\r\n")
 463           assert_match(%r{\AGET /foo/bar }, req)
 464           sock.print "HTTP/1.0 302 Found\r\n"
 465           sock.print "Location: ../baz\r\n\r\n"
 466         ensure
 467           sock.close
 468         end
 469         sock = serv.accept
 470         begin
 471           req = sock.gets("\r\n\r\n")
 472           assert_match(%r{\AGET /baz }, req)
 473           sock.print "HTTP/1.0 200 OK\r\n"
 474           sock.print "Content-Length: 4\r\n\r\n"
 475           sock.print "ab\r\n"
 476         ensure
 477           sock.close
 478         end
 479       }
 480       begin
 481         content = URI("http://127.0.0.1:#{port}/foo/bar").read
 482         assert_equal("ab\r\n", content)
 483       ensure
 484         Thread.kill(th)
 485         th.join
 486       end
 487     }
 488   end
 489
 490   def test_redirect_invalid
 491     TCPServer.open("127.0.0.1", 0) {|serv|
 492       port = serv.addr[1]
 493       th = Thread.new {
 494         sock = serv.accept
 495         begin
 496           req = sock.gets("\r\n\r\n")
 497           assert_match(%r{\AGET /foo/bar }, req)
 498           sock.print "HTTP/1.0 302 Found\r\n"
 499           sock.print "Location: ::\r\n\r\n"
 500         ensure
 501           sock.close
 502         end
 503       }
 504       begin
 505         assert_raise(OpenURI::HTTPError) {
 506           URI("http://127.0.0.1:#{port}/foo/bar").read
 507         }
 508       ensure
 509         Thread.kill(th)
 510         th.join
 511       end
 512     }
 513   end
 514
 515   def setup_redirect_auth(srv, url)
 516     srv.mount_proc("/r1/") {|req, res|
 517       res.status = 301
 518       res["location"] = "#{url}/r2"
 519     }
 520     srv.mount_proc("/r2/") {|req, res|
 521       if req["Authorization"] != "Basic #{['user:pass'].pack('m').chomp}"
 522         raise WEBrick::HTTPStatus::Unauthorized
 523       end
 524       res.body = "r2"
 525     }
 526   end
 527
 528   def test_redirect_auth_success
 529     with_http {|srv, dr, url|
 530       setup_redirect_auth(srv, url)
 531       URI.open("#{url}/r2/", :http_basic_authentication=>['user', 'pass']) {|f|
 532         assert_equal("r2", f.read)
 533       }
 534     }
 535   end
 536
 537   def test_redirect_auth_failure_r2
 538     log_tester = lambda {|server_log|
 539       assert_equal(1, server_log.length)
 540       assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, server_log[0])
 541     }
 542     with_http(log_tester) {|srv, dr, url, server_thread, server_log|
 543       setup_redirect_auth(srv, url)
 544       exc = assert_raise(OpenURI::HTTPError) { URI.open("#{url}/r2/") {} }
 545       assert_equal("401", exc.io.status[0])
 546     }
 547   end
 548
 549   def test_redirect_auth_failure_r1
 550     log_tester = lambda {|server_log|
 551       assert_equal(1, server_log.length)
 552       assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, server_log[0])
 553     }
 554     with_http(log_tester) {|srv, dr, url, server_thread, server_log|
 555       setup_redirect_auth(srv, url)
 556       exc = assert_raise(OpenURI::HTTPError) { URI.open("#{url}/r1/", :http_basic_authentication=>['user', 'pass']) {} }
 557       assert_equal("401", exc.io.status[0])
 558     }
 559   end
 560
 561   def test_max_redirects_success
 562     with_http {|srv, dr, url|
 563       srv.mount_proc("/r1/") {|req, res| res.status = 301; res["location"] = "#{url}/r2"; res.body = "r1" }
 564       srv.mount_proc("/r2/") {|req, res| res.status = 301; res["location"] = "#{url}/r3"; res.body = "r2" }
 565       srv.mount_proc("/r3/") {|req, res| res.body = "r3" }
 566       URI.open("#{url}/r1/", max_redirects: 2) { |f| assert_equal("r3", f.read) }
 567     }
 568   end
 569
 570   def test_max_redirects_too_many
 571     with_http {|srv, dr, url|
 572       srv.mount_proc("/r1/") {|req, res| res.status = 301; res["location"] = "#{url}/r2"; res.body = "r1" }
 573       srv.mount_proc("/r2/") {|req, res| res.status = 301; res["location"] = "#{url}/r3"; res.body = "r2" }
 574       srv.mount_proc("/r3/") {|req, res| res.body = "r3" }
 575       exc = assert_raise(OpenURI::TooManyRedirects) { URI.open("#{url}/r1/", max_redirects: 1) {} }
 576       assert_equal("Too many redirects", exc.message)
 577     }
 578   end
 579
 580   def test_userinfo
 581     assert_raise(ArgumentError) { URI.open("http://user:pass@127.0.0.1/") {} }
 582   end
 583
 584   def test_progress
 585     with_http {|srv, dr, url|
 586       content = "a" * 100000
 587       srv.mount_proc("/data/") {|req, res| res.body = content }
 588       length = []
 589       progress = []
 590       URI.open("#{url}/data/",
 591            :content_length_proc => lambda {|n| length << n },
 592            :progress_proc => lambda {|n| progress << n }
 593           ) {|f|
 594         assert_equal(1, length.length)
 595         assert_equal(content.length, length[0])
 596         assert(progress.length>1,"maybe test is wrong")
 597         assert(progress.sort == progress,"monotone increasing expected but was\n#{progress.inspect}")
 598         assert_equal(content.length, progress[-1])
 599         assert_equal(content, f.read)
 600       }
 601     }
 602   end
 603
 604   def test_progress_chunked
 605     with_http {|srv, dr, url|
 606       content = "a" * 100000
 607       srv.mount_proc("/data/") {|req, res| res.body = content; res.chunked = true }
 608       length = []
 609       progress = []
 610       URI.open("#{url}/data/",
 611            :content_length_proc => lambda {|n| length << n },
 612            :progress_proc => lambda {|n| progress << n }
 613           ) {|f|
 614         assert_equal(1, length.length)
 615         assert_equal(nil, length[0])
 616         assert(progress.length>1,"maybe test is wrong")
 617         assert(progress.sort == progress,"monotone increasing expected but was\n#{progress.inspect}")
 618         assert_equal(content.length, progress[-1])
 619         assert_equal(content, f.read)
 620       }
 621     }
 622   end
 623
 624   def test_uri_read
 625     with_http {|srv, dr, url|
 626       srv.mount_proc("/uriread", lambda { |req, res| res.body = "uriread" } )
 627       data = URI("#{url}/uriread").read
 628       assert_equal("200", data.status[0])
 629       assert_equal("uriread", data)
 630     }
 631   end
 632
 633   def test_encoding
 634     with_http {|srv, dr, url|
 635       content_u8 = "\u3042"
 636       content_ej = "\xa2\xa4".dup.force_encoding("euc-jp")
 637       srv.mount_proc("/u8/") {|req, res| res.body = content_u8; res['content-type'] = 'text/plain; charset=utf-8' }
 638       srv.mount_proc("/ej/") {|req, res| res.body = content_ej; res['content-type'] = 'TEXT/PLAIN; charset=EUC-JP' }
 639       srv.mount_proc("/nc/") {|req, res| res.body = "aa"; res['content-type'] = 'Text/Plain' }
 640       URI.open("#{url}/u8/") {|f|
 641         assert_equal(content_u8, f.read)
 642         assert_equal("text/plain", f.content_type)
 643         assert_equal("utf-8", f.charset)
 644       }
 645       URI.open("#{url}/ej/") {|f|
 646         assert_equal(content_ej, f.read)
 647         assert_equal("text/plain", f.content_type)
 648         assert_equal("euc-jp", f.charset)
 649         assert_equal(Encoding::EUC_JP, f.read.encoding)
 650       }
 651       URI.open("#{url}/ej/", 'r:utf-8') {|f|
 652         # override charset with encoding option
 653         assert_equal(content_ej.dup.force_encoding('utf-8'), f.read)
 654         assert_equal("text/plain", f.content_type)
 655         assert_equal("euc-jp", f.charset)
 656         assert_equal(Encoding::UTF_8, f.read.encoding)
 657       }
 658       URI.open("#{url}/ej/", :encoding=>'utf-8') {|f|
 659         # override charset with encoding option
 660         assert_equal(content_ej.dup.force_encoding('utf-8'), f.read)
 661         assert_equal("text/plain", f.content_type)
 662         assert_equal("euc-jp", f.charset)
 663         assert_equal(Encoding::UTF_8, f.read.encoding)
 664       }
 665       assert_raise(ArgumentError) {
 666         URI.open("#{url}/ej/", 'r:utf-8', :encoding=>'utf-8') {|f| }
 667       }
 668       URI.open("#{url}/nc/") {|f|
 669         assert_equal("aa", f.read)
 670         assert_equal("text/plain", f.content_type)
 671         assert_equal("utf-8", f.charset)
 672         assert_equal("unknown", f.charset { "unknown" })
 673       }
 674     }
 675   end
 676
 677   def test_quoted_attvalue
 678     with_http {|srv, dr, url|
 679       content_u8 = "\u3042"
 680       srv.mount_proc("/qu8/") {|req, res| res.body = content_u8; res['content-type'] = 'text/plain; charset="utf\-8"' }
 681       URI.open("#{url}/qu8/") {|f|
 682         assert_equal(content_u8, f.read)
 683         assert_equal("text/plain", f.content_type)
 684         assert_equal("utf-8", f.charset)
 685       }
 686     }
 687   end
 688
 689   def test_last_modified
 690     with_http {|srv, dr, url|
 691       srv.mount_proc("/data/") {|req, res| res.body = "foo"; res['last-modified'] = 'Fri, 07 Aug 2009 06:05:04 GMT' }
 692       URI.open("#{url}/data/") {|f|
 693         assert_equal("foo", f.read)
 694         assert_equal(Time.utc(2009,8,7,6,5,4), f.last_modified)
 695       }
 696     }
 697   end
 698
 699   def test_content_encoding
 700     with_http {|srv, dr, url|
 701       content = "abc" * 10000
 702       Zlib::GzipWriter.wrap(StringIO.new(content_gz="".b)) {|z| z.write content }
 703       srv.mount_proc("/data/") {|req, res| res.body = content_gz; res['content-encoding'] = 'gzip' }
 704       srv.mount_proc("/data2/") {|req, res| res.body = content_gz; res['content-encoding'] = 'gzip'; res.chunked = true }
 705       srv.mount_proc("/noce/") {|req, res| res.body = content_gz }
 706       URI.open("#{url}/data/") {|f|
 707         assert_equal [], f.content_encoding
 708         assert_equal(content, f.read)
 709       }
 710       URI.open("#{url}/data2/") {|f|
 711         assert_equal [], f.content_encoding
 712         assert_equal(content, f.read)
 713       }
 714       URI.open("#{url}/noce/") {|f|
 715         assert_equal [], f.content_encoding
 716         assert_equal(content_gz, f.read.force_encoding("ascii-8bit"))
 717       }
 718     }
 719   end if defined?(Zlib::GzipWriter)
 720
 721   def test_multiple_cookies
 722     with_http {|srv, dr, url|
 723       srv.mount_proc("/mcookie/") {|req, res|
 724         res.cookies << "name1=value1; blabla"
 725         res.cookies << "name2=value2; blabla"
 726         res.body = "foo"
 727       }
 728       URI.open("#{url}/mcookie/") {|f|
 729         assert_equal("foo", f.read)
 730         assert_equal(["name1=value1; blabla", "name2=value2; blabla"],
 731                      f.metas['set-cookie'].sort)
 732       }
 733     }
 734   end
 735
 736   # 192.0.2.0/24 is TEST-NET.  [RFC3330]
 737
 738   begin
 739     require 'net/ftp'
 740
 741     def test_ftp_invalid_request
 742       assert_raise(ArgumentError) { URI("ftp://127.0.0.1/").read }
 743       assert_raise(ArgumentError) { URI("ftp://127.0.0.1/a%0Db").read }
 744       assert_raise(ArgumentError) { URI("ftp://127.0.0.1/a%0Ab").read }
 745       assert_raise(ArgumentError) { URI("ftp://127.0.0.1/a%0Db/f").read }
 746       assert_raise(ArgumentError) { URI("ftp://127.0.0.1/a%0Ab/f").read }
 747       assert_nothing_raised(URI::InvalidComponentError) { URI("ftp://127.0.0.1/d/f;type=x") }
 748     end
 749
 750     def test_ftp
 751       TCPServer.open("127.0.0.1", 0) {|serv|
 752         _, port, _, host = serv.addr
 753         th = Thread.new {
 754           s = serv.accept
 755           begin
 756             s.print "220 Test FTP Server\r\n"
 757             assert_equal("USER anonymous\r\n", s.gets); s.print "331 name ok\r\n"
 758             assert_match(/\APASS .*\r\n\z/, s.gets); s.print "230 logged in\r\n"
 759             assert_equal("TYPE I\r\n", s.gets); s.print "200 type set to I\r\n"
 760             assert_equal("CWD foo\r\n", s.gets); s.print "250 CWD successful\r\n"
 761             assert_equal("PASV\r\n", s.gets)
 762             TCPServer.open("127.0.0.1", 0) {|data_serv|
 763               _, data_serv_port, _, _ = data_serv.addr
 764               hi = data_serv_port >> 8
 765               lo = data_serv_port & 0xff
 766               s.print "227 Entering Passive Mode (127,0,0,1,#{hi},#{lo}).\r\n"
 767               assert_equal("RETR bar\r\n", s.gets); s.print "150 file okay\r\n"
 768               data_sock = data_serv.accept
 769               begin
 770                 data_sock << "content"
 771               ensure
 772                 data_sock.close
 773               end
 774               s.print "226 transfer complete\r\n"
 775               assert_nil(s.gets)
 776             }
 777           ensure
 778             s.close if s
 779           end
 780         }
 781         begin
 782           content = URI("ftp://#{host}:#{port}/foo/bar").read
 783           assert_equal("content", content)
 784         ensure
 785           Thread.kill(th)
 786           th.join
 787         end
 788       }
 789     end
 790
 791     def test_ftp_active
 792       TCPServer.open("127.0.0.1", 0) {|serv|
 793         _, port, _, host = serv.addr
 794         th = Thread.new {
 795           s = serv.accept
 796           begin
 797             content = "content"
 798             s.print "220 Test FTP Server\r\n"
 799             assert_equal("USER anonymous\r\n", s.gets); s.print "331 name ok\r\n"
 800             assert_match(/\APASS .*\r\n\z/, s.gets); s.print "230 logged in\r\n"
 801             assert_equal("TYPE I\r\n", s.gets); s.print "200 type set to I\r\n"
 802             assert_equal("CWD foo\r\n", s.gets); s.print "250 CWD successful\r\n"
 803             assert(m = /\APORT 127,0,0,1,(\d+),(\d+)\r\n\z/.match(s.gets))
 804             active_port = m[1].to_i << 8 | m[2].to_i
 805             TCPSocket.open("127.0.0.1", active_port) {|data_sock|
 806               s.print "200 data connection opened\r\n"
 807               assert_equal("RETR bar\r\n", s.gets); s.print "150 file okay\r\n"
 808               begin
 809                 data_sock << content
 810               ensure
 811                 data_sock.close
 812               end
 813               s.print "226 transfer complete\r\n"
 814               assert_nil(s.gets)
 815             }
 816           ensure
 817             s.close if s
 818           end
 819         }
 820         begin
 821           content = URI("ftp://#{host}:#{port}/foo/bar").read(:ftp_active_mode=>true)
 822           assert_equal("content", content)
 823         ensure
 824           Thread.kill(th)
 825           th.join
 826         end
 827       }
 828     end
 829
 830     def test_ftp_ascii
 831       TCPServer.open("127.0.0.1", 0) {|serv|
 832         _, port, _, host = serv.addr
 833         th = Thread.new {
 834           s = serv.accept
 835           begin
 836             content = "content"
 837             s.print "220 Test FTP Server\r\n"
 838             assert_equal("USER anonymous\r\n", s.gets); s.print "331 name ok\r\n"
 839             assert_match(/\APASS .*\r\n\z/, s.gets); s.print "230 logged in\r\n"
 840             assert_equal("TYPE I\r\n", s.gets); s.print "200 type set to I\r\n"
 841             assert_equal("CWD /foo\r\n", s.gets); s.print "250 CWD successful\r\n"
 842             assert_equal("TYPE A\r\n", s.gets); s.print "200 type set to A\r\n"
 843             assert_equal("SIZE bar\r\n", s.gets); s.print "213 #{content.bytesize}\r\n"
 844             assert_equal("PASV\r\n", s.gets)
 845             TCPServer.open("127.0.0.1", 0) {|data_serv|
 846               _, data_serv_port, _, _ = data_serv.addr
 847               hi = data_serv_port >> 8
 848               lo = data_serv_port & 0xff
 849               s.print "227 Entering Passive Mode (127,0,0,1,#{hi},#{lo}).\r\n"
 850               assert_equal("RETR bar\r\n", s.gets); s.print "150 file okay\r\n"
 851               data_sock = data_serv.accept
 852               begin
 853                 data_sock << content
 854               ensure
 855                 data_sock.close
 856               end
 857               s.print "226 transfer complete\r\n"
 858               assert_nil(s.gets)
 859             }
 860           ensure
 861             s.close if s
 862           end
 863         }
 864         begin
 865           length = []
 866           progress = []
 867           content = URI("ftp://#{host}:#{port}/%2Ffoo/b%61r;type=a").read(
 868           :content_length_proc => lambda {|n| length << n },
 869           :progress_proc => lambda {|n| progress << n })
 870           assert_equal("content", content)
 871           assert_equal([7], length)
 872           assert_equal(7, progress.inject(&:+))
 873         ensure
 874           Thread.kill(th)
 875           th.join
 876         end
 877       }
 878     end
 879   rescue LoadError
 880     # net-ftp is the bundled gems at Ruby 3.1
 881   end
 882
 883   def test_ftp_over_http_proxy
 884     TCPServer.open("127.0.0.1", 0) {|proxy_serv|
 885       proxy_port = proxy_serv.addr[1]
 886       th = Thread.new {
 887         proxy_sock = proxy_serv.accept
 888         begin
 889           req = proxy_sock.gets("\r\n\r\n")
 890           assert_match(%r{\AGET ftp://192.0.2.1/foo/bar }, req)
 891           proxy_sock.print "HTTP/1.0 200 OK\r\n"
 892           proxy_sock.print "Content-Length: 4\r\n\r\n"
 893           proxy_sock.print "ab\r\n"
 894         ensure
 895           proxy_sock.close
 896         end
 897       }
 898       begin
 899         with_env('ftp_proxy'=>"http://127.0.0.1:#{proxy_port}") {
 900           content = URI("ftp://192.0.2.1/foo/bar").read
 901           assert_equal("ab\r\n", content)
 902         }
 903       ensure
 904         Thread.kill(th)
 905         th.join
 906       end
 907     }
 908   end
 909
 910   def test_ftp_over_http_proxy_auth
 911     TCPServer.open("127.0.0.1", 0) {|proxy_serv|
 912       proxy_port = proxy_serv.addr[1]
 913       th = Thread.new {
 914         proxy_sock = proxy_serv.accept
 915         begin
 916           req = proxy_sock.gets("\r\n\r\n")
 917           assert_match(%r{\AGET ftp://192.0.2.1/foo/bar }, req)
 918           assert_match(%r{Proxy-Authorization: Basic #{['proxy-user:proxy-password'].pack('m').chomp}\r\n}, req)
 919           proxy_sock.print "HTTP/1.0 200 OK\r\n"
 920           proxy_sock.print "Content-Length: 4\r\n\r\n"
 921           proxy_sock.print "ab\r\n"
 922         ensure
 923           proxy_sock.close
 924         end
 925       }
 926       begin
 927         content = URI("ftp://192.0.2.1/foo/bar").read(
 928           :proxy_http_basic_authentication => ["http://127.0.0.1:#{proxy_port}", "proxy-user", "proxy-password"])
 929         assert_equal("ab\r\n", content)
 930       ensure
 931         Thread.kill(th)
 932         th.join
 933       end
 934     }
 935   end
 936
 937   def test_meta_init_doesnt_bump_global_constant_state
 938     omit "RubyVM.stat not defined" unless defined? RubyVM.stat
 939     omit unless RubyVM.stat.has_key?(:global_constant_state)
 940
 941     OpenURI::Meta.init(Object.new) # prewarm
 942
 943     before = RubyVM.stat(:global_constant_state)
 944     OpenURI::Meta.init(Object.new)
 945     assert_equal 0, RubyVM.stat(:global_constant_state) - before
 946   end
 947 end