From f62e34251c1101b0fcdddea35dfa3f73c416a3ba Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 8 Dec 2011 16:57:08 -0800 Subject: [PATCH] reorganize largefile support code Putting this into the MogileFS::Put::* namespace will make it easier to document. --- bin/mog | 2 +- lib/mogilefs.rb | 2 +- lib/mogilefs/mogilefs.rb | 13 +--------- lib/mogilefs/put.rb | 29 ++++++++++++++++++++++ .../{http_range_put.rb => put/content_range.rb} | 2 +- lib/mogilefs/{http_stream.rb => put/stream.rb} | 2 +- lib/mogilefs/{http_tempfile.rb => put/tempfile.rb} | 2 +- test/test_mogstored_rack.rb | 20 +++++++-------- 8 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 lib/mogilefs/put.rb rename lib/mogilefs/{http_range_put.rb => put/content_range.rb} (98%) rename lib/mogilefs/{http_stream.rb => put/stream.rb} (98%) rename lib/mogilefs/{http_tempfile.rb => put/tempfile.rb} (91%) diff --git a/bin/mog b/bin/mog index 6ca3ca5..89b1b9b 100755 --- a/bin/mog +++ b/bin/mog @@ -210,7 +210,7 @@ begin skip_tee = File.stat('/dev/null') == $stdout.stat largefile = :tempfile largefile = :content_range if range - largefile = :chunked if chunk + largefile = :stream if chunk io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile) begin diff --git a/lib/mogilefs.rb b/lib/mogilefs.rb index 4002a54..ad0d6fb 100644 --- a/lib/mogilefs.rb +++ b/lib/mogilefs.rb @@ -73,9 +73,9 @@ require 'mogilefs/util' require 'mogilefs/socket' require 'mogilefs/backend' require 'mogilefs/http_file' -require 'mogilefs/http_stream' require 'mogilefs/http_reader' require 'mogilefs/client' require 'mogilefs/bigfile' +require 'mogilefs/put' require 'mogilefs/mogilefs' require 'mogilefs/version' # generated by ./GIT-VERSION-GEN diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb index 50484d8..c93d1a3 100644 --- a/lib/mogilefs/mogilefs.rb +++ b/lib/mogilefs/mogilefs.rb @@ -161,18 +161,7 @@ class MogileFS::MogileFS < MogileFS::Client case (dests[0][1] rescue nil) when %r{\Ahttp://} - http_file = case opts[:largefile] - when :chunked,:stream - MogileFS::HTTPStream - when :tempfile - require 'mogilefs/http_tempfile' - MogileFS::HTTPTempfile - when :content_range - require 'mogilefs/http_range_put' - MogileFS::HTTPRangePut - when nil, false - MogileFS::HTTPFile - end.new(dests, opts) + http_file = MogileFS::Put[opts[:largefile]].new(dests, opts) if block_given? yield http_file return http_file.commit # calls create_close diff --git a/lib/mogilefs/put.rb b/lib/mogilefs/put.rb new file mode 100644 index 0000000..3f13df2 --- /dev/null +++ b/lib/mogilefs/put.rb @@ -0,0 +1,29 @@ +# -*- encoding: binary -*- +module MogileFS::Put + + def self.[](largefile) + case largefile + when nil, false + MogileFS::HTTPFile + when :stream + Stream + when :content_range + ContentRange + when :tempfile + Tempfile + else + raise ArgumentError, "largefile: #{largefile.inspect} not understood" + end + end + + def self.const_missing(name) + case name + when :Stream, :ContentRange, :Tempfile + file = name.to_s.gsub(/([a-z])([A-Z])/, '\1_\2').downcase + require "mogilefs/put/#{file}" + const_get(name) + else + super(name) + end + end +end diff --git a/lib/mogilefs/http_range_put.rb b/lib/mogilefs/put/content_range.rb similarity index 98% rename from lib/mogilefs/http_range_put.rb rename to lib/mogilefs/put/content_range.rb index e191c23..e44a8e2 100644 --- a/lib/mogilefs/http_range_put.rb +++ b/lib/mogilefs/put/content_range.rb @@ -11,7 +11,7 @@ require 'mogilefs/new_file_common' require 'mogilefs/new_file_writer' # an IO-like object -class MogileFS::HTTPRangePut +class MogileFS::Put::ContentRange include MogileFS::NewFileWriter include MogileFS::NewFileCommon diff --git a/lib/mogilefs/http_stream.rb b/lib/mogilefs/put/stream.rb similarity index 98% rename from lib/mogilefs/http_stream.rb rename to lib/mogilefs/put/stream.rb index 3f84b9b..590a24f 100644 --- a/lib/mogilefs/http_stream.rb +++ b/lib/mogilefs/put/stream.rb @@ -4,7 +4,7 @@ require 'mogilefs/new_file_common' require 'mogilefs/new_file_writer' -class MogileFS::HTTPStream +class MogileFS::Put::Stream attr_reader :to_io attr_reader :md5 diff --git a/lib/mogilefs/http_tempfile.rb b/lib/mogilefs/put/tempfile.rb similarity index 91% rename from lib/mogilefs/http_tempfile.rb rename to lib/mogilefs/put/tempfile.rb index 5019522..8082f95 100644 --- a/lib/mogilefs/http_tempfile.rb +++ b/lib/mogilefs/put/tempfile.rb @@ -3,7 +3,7 @@ require 'tempfile' require 'mogilefs/http_file' -class MogileFS::HTTPTempfile < Tempfile +class MogileFS::Put::Tempfile < Tempfile def initialize(*args) @mogilefs_httpfile_args = args super("mogilefs-client") diff --git a/test/test_mogstored_rack.rb b/test/test_mogstored_rack.rb index f44e70a..ba4986f 100644 --- a/test/test_mogstored_rack.rb +++ b/test/test_mogstored_rack.rb @@ -37,15 +37,15 @@ class TestMogstoredRack < Test::Unit::TestCase def test_stream_new_file add_host_device_domain client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain - client.new_file("chunky", :largefile => :chunked) do |io| - assert_instance_of MogileFS::HTTPStream, io + client.new_file("chunky", :largefile => :stream) do |io| + assert_instance_of MogileFS::Put::Stream, io assert_equal(5, io.write("HELLO")) assert_nil io.md5 end assert_equal "HELLO", client.get_file_data("chunky") - io = client.new_file("puts", :largefile => :chunked) - assert_instance_of MogileFS::HTTPStream, io + io = client.new_file("puts", :largefile => :stream) + assert_instance_of MogileFS::Put::Stream, io assert_equal io, IO.select(nil, [io])[1][0], "IO.select-able" assert_nil(io.puts("PUTS!")) @@ -53,35 +53,35 @@ class TestMogstoredRack < Test::Unit::TestCase assert_nil io.close assert_equal "PUTS!\nPUTZ\n", client.get_file_data("puts") - io = client.new_file("putc", :largefile => :chunked) + io = client.new_file("putc", :largefile => :stream) assert_equal(0x20, io.putc(0x20)) assert_nil io.close assert_equal " ", client.get_file_data("putc") - io = client.new_file("print splat", :largefile => :chunked) + io = client.new_file("print splat", :largefile => :stream) io.print(1, 2, 3) assert_nil io.close assert_equal "123", client.get_file_data("print splat") - io = client.new_file("printf", :largefile => :chunked) + io = client.new_file("printf", :largefile => :stream) assert_nil io.printf("%x", 1638) assert_nil io.close assert_equal "666", client.get_file_data("printf") - io = client.new_file("syswrite", :largefile => :chunked) + io = client.new_file("syswrite", :largefile => :stream) assert_equal 4, io.syswrite("good") assert_equal 7, io.syswrite("morning") assert_nil io.close assert_equal "goodmorning", client.get_file_data("syswrite") - io = client.new_file("md5", :largefile=>:chunked, :content_md5=>:trailer) + io = client.new_file("md5", :largefile=>:stream, :content_md5=>:trailer) assert_instance_of Digest::MD5, io.md5 assert_nil io.puts("HIHI") assert_nil io.close assert_equal "HIHI\n", client.get_file_data("md5") assert_equal Digest::MD5.hexdigest("HIHI\n"), io.md5.hexdigest - io = client.new_file("<<", :largefile=>:chunked) + io = client.new_file("<<", :largefile=>:stream) assert_equal(io, io << ">>") assert_nil io.close assert_equal ">>", client.get_file_data("<<") -- 2.11.4.GIT