From 283050fc283868f79796f05901bd9149713ae282 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 5 Nov 2011 02:34:35 +0000 Subject: [PATCH] redo IO.copy_stream usage Avoid deepening stack depth and make it easier to migrate fully to 1.9 in the future (dropping 1.8 support). --- bin/mog | 5 ++--- lib/mogilefs.rb | 7 +++++++ lib/mogilefs/bigfile.rb | 2 +- lib/mogilefs/copy_stream.rb | 19 +++++++++++++++++++ lib/mogilefs/http_file.rb | 7 ++----- lib/mogilefs/http_reader.rb | 1 - lib/mogilefs/mogilefs.rb | 7 +++---- lib/mogilefs/util.rb | 23 ----------------------- test/test_mogilefs.rb | 4 +--- 9 files changed, 35 insertions(+), 40 deletions(-) create mode 100644 lib/mogilefs/copy_stream.rb diff --git a/bin/mog b/bin/mog index c62d818..c9f1542 100755 --- a/bin/mog +++ b/bin/mog @@ -97,7 +97,6 @@ unless cmd = ARGV.shift end cfg[:timeout] ||= 30 # longer timeout for interactive use -include MogileFS::Util mg = MogileFS::MogileFS.new(cfg) def store_file_retry(mg, key, storage_class, filepath) @@ -129,7 +128,7 @@ begin if (!cat[:raw] && key =~ /^_big_info:/) mg.bigfile_write(key, STDOUT, {:verify => true}) else - mg.get_file_data(key) { |fp| copy_stream(fp, STDOUT) } + mg.get_file_data(key) { |fp| MogileFS::X.copy_stream(fp, STDOUT) } end end when 'ls' @@ -201,7 +200,7 @@ begin end end begin - copy_stream(STDIN, tee_obj) + MogileFS::X.copy_stream(STDIN, tee_obj) store_file_retry(mg, key, cfg[:class], tmp.path) ensure tmp.close! diff --git a/lib/mogilefs.rb b/lib/mogilefs.rb index c4ea147..d0f3e21 100644 --- a/lib/mogilefs.rb +++ b/lib/mogilefs.rb @@ -27,6 +27,13 @@ module MogileFS def message; "couldn't connect to mogilefsd backend"; end end + # IO.copy_stream was buggy in Ruby 1.9.2 and earlier + if RUBY_VERSION >= "1.9.3" + X = IO + else + require "mogilefs/copy_stream" + X = MogileFS::CopyStream + end end require 'mogilefs/backend' diff --git a/lib/mogilefs/bigfile.rb b/lib/mogilefs/bigfile.rb index 65c7eb6..bfedbde 100644 --- a/lib/mogilefs/bigfile.rb +++ b/lib/mogilefs/bigfile.rb @@ -45,7 +45,7 @@ module MogileFS::Bigfile sock = MogileFS::HTTPReader.first(paths, "GET", t) end - w = copy_stream(sock, wr_io) + w = MogileFS::X.copy_stream(sock, wr_io) wr_io.respond_to?(:md5_check!) and wr_io.md5_check!(part[:md5]) total += w diff --git a/lib/mogilefs/copy_stream.rb b/lib/mogilefs/copy_stream.rb new file mode 100644 index 0000000..922eb71 --- /dev/null +++ b/lib/mogilefs/copy_stream.rb @@ -0,0 +1,19 @@ +# -*- encoding: binary -*- + +# internal compatibility class for older Rubies +module MogileFS::CopyStream # :nodoc: + def self.copy_stream(src, dst) + src_io = String === src ? File.open(src) : src + buf = "" + written = 0 + begin + src_io.readpartial(0x4000, buf) + written += dst.write(buf) + rescue EOFError + break + end while true + written + ensure + src_io.close if String === src + end +end diff --git a/lib/mogilefs/http_file.rb b/lib/mogilefs/http_file.rb index 22692e7..261e209 100644 --- a/lib/mogilefs/http_file.rb +++ b/lib/mogilefs/http_file.rb @@ -3,7 +3,6 @@ require 'stringio' require 'uri' require 'mogilefs/backend' -require 'mogilefs/util' ## # HTTPFile wraps up the new file operations for storing files onto an HTTP @@ -13,8 +12,6 @@ require 'mogilefs/util' # create a new file using MogileFS::MogileFS.new_file. # class MogileFS::HTTPFile < StringIO - include MogileFS::Util - class EmptyResponseError < MogileFS::Error; end class BadResponseError < MogileFS::Error; end class UnparseableResponseError < MogileFS::Error; end @@ -63,11 +60,11 @@ class MogileFS::HTTPFile < StringIO end) elsif @big_io # Don't try to run out of memory - File.open(@big_io, "rb") do |fp| + File.open(@big_io) do |fp| file_size = fp.stat.size sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ "Content-Length: #{file_size}\r\n\r\n") - copy_stream(fp, sock) + MogileFS::X.copy_stream(fp, sock) end else sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \ diff --git a/lib/mogilefs/http_reader.rb b/lib/mogilefs/http_reader.rb index f1c5491..d3f57f7 100644 --- a/lib/mogilefs/http_reader.rb +++ b/lib/mogilefs/http_reader.rb @@ -5,7 +5,6 @@ # for reading huge response bodies over fast LANs. class MogileFS::HTTPReader < MogileFS::Socket attr_accessor :content_length, :uri - include MogileFS::Util # backwards compat, if anybody cares alias mogilefs_size content_length # :nodoc: diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb index 8679d2e..812fba6 100644 --- a/lib/mogilefs/mogilefs.rb +++ b/lib/mogilefs/mogilefs.rb @@ -8,7 +8,6 @@ require 'mogilefs/http_reader' class MogileFS::MogileFS < MogileFS::Client - include MogileFS::Util include MogileFS::Bigfile ## @@ -137,15 +136,15 @@ class MogileFS::MogileFS < MogileFS::Client raise MogileFS::ReadOnlyError if readonly? new_file key, klass do |mfp| - if file.respond_to?(:read) - copy_stream(file, mfp) + if file.respond_to?(:readpartial) + MogileFS::X.copy_stream(file, mfp) else size = File.size(file) if size > 0x10000 # Bigass file, handle differently mfp.big_io = file size else - File.open(file, "rb") { |fp| copy_stream(fp, mfp) } + MogileFS::X.copy_stream(file, mfp) end end end diff --git a/lib/mogilefs/util.rb b/lib/mogilefs/util.rb index 6b46c97..aaf7b7a 100644 --- a/lib/mogilefs/util.rb +++ b/lib/mogilefs/util.rb @@ -1,29 +1,7 @@ # -*- encoding: binary -*- require 'mogilefs' -require 'socket' module MogileFS::Util - - # here are internal implementation details, do not use them in your code - # TODO: cleanup - if IO.respond_to?(:copy_stream) - def copy_stream(src, dst) - IO.copy_stream(src, dst) - end - else - def copy_stream(src, dst) - buf = "" - written = 0 - begin - src.readpartial(0x4000, buf) - written += dst.write(buf) - rescue EOFError - break - end while true - written - end - end - class StoreContent < Proc def initialize(total_size, &writer_proc) @total_size = total_size @@ -33,7 +11,6 @@ module MogileFS::Util @total_size end end - end require 'timeout' diff --git a/test/test_mogilefs.rb b/test/test_mogilefs.rb index f85548f..cd413c7 100644 --- a/test/test_mogilefs.rb +++ b/test/test_mogilefs.rb @@ -5,8 +5,6 @@ require 'tempfile' require 'fileutils' class TestMogileFS__MogileFS < TestMogileFS - include MogileFS::Util - def setup @klass = MogileFS::MogileFS super @@ -103,7 +101,7 @@ class TestMogileFS__MogileFS < TestMogileFS readed = client.recv(4096, 0) assert(readed =~ \ %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z}) - copy_stream(tmpfp, client) + MogileFS::X.copy_stream(tmpfp, client) client.close exit 0 end -- 2.11.4.GIT