bin/mog

   1 #!/usr/bin/env ruby
   2 require 'mogilefs'
   3 require 'optparse'
   4 $stdin.binmode
   5 $stdout.binmode
   6 $stderr.sync = $stdout.sync = true
   7
   8 trap('INT') { exit 130 }
   9 trap('PIPE') { exit 0 }
  10 if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
  11   md5_trailer_nodes.split(/\s*,\s*/).each do |host|
  12     MogileFS::HTTPFile::MD5_TRAILER_NODES[host] = true
  13   end
  14 end
  15
  16 # this is to be compatible with config files used by the Perl tools
  17 def parse_config_file!(path, overwrite = false)
  18   dest = {}
  19   File.open(path).each_line do |line|
  20     case line
  21     when /^(domain|class)\s*=\s*(\S+)/
  22       dest[$1.to_sym] = $2
  23     when /^(?:trackers|hosts)\s*=\s*(.*)/
  24       dest[:hosts] = $1.split(/\s*,\s*/)
  25     when /^timeout\s*=\s*(.*)/
  26       dest[:timeout] = $1.to_f
  27     when /^noclobber\s*=\s*true\s*/
  28       dest[:noclobber] = true
  29     else
  30       warn "Ignored configuration line: #{line}" unless /^#/.match(line)
  31     end
  32   end
  33   dest
  34 end
  35
  36 # parse the default config file if one exists
  37 def_file = File.expand_path("~/.mogilefs-client.conf")
  38 def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
  39
  40 # parse the command-line first, these options take precedence over all else
  41 cli_cfg = {}
  42 config_file = nil
  43 ls_l = false
  44 ls_h = false
  45 chunk = false
  46 range = false
  47 test = {}
  48 cat = { :raw => false }
  49
  50 ARGV.options do |x|
  51   x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
  52   x.separator ''
  53
  54   x.on('-c', '--config=/path/to/config',
  55        'config file to load') { |file| config_file = file }
  56
  57   x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
  58        'hostnames/IP addresses of trackers') do |trackers|
  59     cli_cfg[:hosts] = trackers
  60   end
  61
  62   x.on('-e', 'True if key exists') { test[:e] = true }
  63   x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
  64   x.on('-n', '--no-clobber', 'do not clobber existing key') do
  65     cli_cfg[:noclobber] = true
  66   end
  67
  68   x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
  69   x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
  70   x.on('-l', "long listing format (`ls' command)") { ls_l = true }
  71   x.on('-h', '--human-readable',
  72        "print sizes in human-readable format (`ls' command)") { ls_h = true }
  73   x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
  74   x.on('--range', "stream partial uploads (`tee' command)") { range = true }
  75   x.separator ''
  76   x.on('--help', 'Show this help message.') { puts x; exit }
  77   x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
  78   x.parse!
  79 end
  80
  81 # parse the config file specified at the command-line
  82 file_cfg = config_file ? parse_config_file!(config_file, true) : {}
  83
  84 # read environment variables, too.  This Ruby API favors the term
  85 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
  86 # Favor the term more consistent with what the MogileFS inventors used.
  87 env_cfg = {}
  88 if ENV["MOG_TRACKERS"]
  89   env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
  90 end
  91 if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
  92   env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
  93 end
  94 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
  95 env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
  96
  97 # merge the configs, favoring them in order specified:
  98 cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
  99
 100 # error-checking
 101 err = []
 102 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
 103 err << "domain must be specified" unless cfg[:domain]
 104 if err.any?
 105   warn "Errors:\n  #{err.join("\n  ")}"
 106   warn ARGV.options
 107   exit 1
 108 end
 109
 110 unless cmd = ARGV.shift
 111   warn ARGV.options
 112   exit 1
 113 end
 114
 115 cfg[:timeout] ||= 30 # longer timeout for interactive use
 116 mg = MogileFS::MogileFS.new(cfg)
 117
 118 def store_file_retry(mg, key, storage_class, filepath)
 119   tries = 0
 120   begin
 121     mg.store_file(key, storage_class, filepath)
 122   rescue MogileFS::UnreadableSocketError,
 123          MogileFS::Backend::NoDevicesError => err
 124     if ((tries += 1) < 10)
 125       warn "Retrying on error: #{err}: #{err.message} tries: #{tries}"
 126       retry
 127     else
 128       warn "FATAL: #{err}: #{err.message} tries: #{tries}"
 129     end
 130     exit 1
 131   end
 132 end
 133
 134 def human_size(size)
 135   suff = ''
 136   %w(K M G).each do |s|
 137     size /= 1024.0
 138     if size <= 1024
 139       suff = s
 140       break
 141     end
 142   end
 143   sprintf("%.1f%s", size, suff)
 144 end
 145
 146 begin
 147   case cmd
 148   when 'cp'
 149     filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
 150     dkey = ARGV.shift or raise ArgumentError, '<filename> <key>'
 151     ARGV.shift and raise ArgumentError, '<filename> <key>'
 152     cfg[:noclobber] && mg.exist?(dkey) and
 153       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
 154     store_file_retry(mg, dkey, cfg[:class], filename)
 155   when 'cat'
 156     ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
 157     ARGV.each do |key|
 158       if (!cat[:raw] && key =~ /^_big_info:/)
 159         mg.bigfile_write(key, $stdout, {:verify => true})
 160       else
 161         mg.get_file_data(key, $stdout)
 162       end
 163     end
 164   when 'ls'
 165     prefixes = ARGV.empty? ? [ nil ] : ARGV
 166     if ls_l
 167       each_key = lambda do |key, size, devcount|
 168         size = ls_h && size > 1024 ? human_size(size) : size.to_s
 169         size = (' ' * (12 - size.length)) << size # right justify
 170         puts [ sprintf("% 2d", devcount), size, key ].pack("A4 A16 A*")
 171       end
 172     else
 173       each_key = lambda { |key| puts key }
 174     end
 175     prefixes.each { |prefix| mg.each_key(prefix, &each_key) }
 176   when 'rm'
 177     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 178     ARGV.each { |key| mg.delete(key) }
 179   when 'mv'
 180     from = ARGV.shift or raise ArgumentError, '<from> <to>'
 181     to = ARGV.shift or raise ArgumentError, '<from> <to>'
 182     ARGV.shift and raise ArgumentError, '<from> <to>'
 183     mg.rename(from, to)
 184   when 'stat' # this outputs a RFC822-like format
 185     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 186     ok = true
 187     ARGV.each_with_index do |key,j|
 188       begin
 189         info = mg.file_info(key)
 190         puts "Key: #{key}"
 191         puts "Size: #{info['length']}"
 192         puts "Class: #{info['class']}"
 193         checksum = info['checksum'] and puts "Checksum: #{checksum}"
 194         o = { :pathcount => info["devcount"] }
 195         mg.get_paths(key, o).each_with_index do |path,i|
 196           puts "URL-#{i}: #{path}"
 197         end
 198         puts "" if ARGV.size != (j + 1)
 199       rescue MogileFS::Backend::UnknownKeyError
 200         warn "No such key: #{key}"
 201         ok = false
 202       end
 203     end
 204     exit(ok)
 205   when 'tee'
 206     abort "--range and --chunk are incompatible" if range && chunk
 207     dkey = ARGV.shift or raise ArgumentError, '<key>'
 208     ARGV.shift and raise ArgumentError, '<key>'
 209     cfg[:noclobber] && mg.exist?(dkey) and
 210       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
 211     skip_tee = File.stat('/dev/null') == $stdout.stat
 212     largefile = :tempfile
 213     largefile = :content_range if range
 214     largefile = :chunked if chunk
 215
 216     io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile)
 217     begin
 218       buf = $stdin.readpartial(16384)
 219       begin
 220         io.write(buf)
 221         $stdout.write(buf) unless skip_tee
 222         $stdin.readpartial(16384, buf)
 223       end while true
 224     rescue EOFError
 225     end
 226     io.close
 227   when 'test'
 228     truth, ok = true, nil
 229     raise ArgumentError, "-e must be specified" unless (test.size == 1)
 230
 231     truth, key = case ARGV.size
 232     when 1
 233       [ true, ARGV[0] ]
 234     when 2
 235       if ARGV[0] != "!"
 236         raise ArgumentError, "#{ARGV[0]}: binary operator expected"
 237       end
 238       [ false, ARGV[1] ]
 239     else
 240       raise ArgumentError, "Too many arguments"
 241     end
 242
 243     test[:e] or raise ArgumentError, "Unknown flag: -#{test.keys.first}"
 244     ok = mg.exist?(key)
 245     truth or ok = ! ok
 246     exit ok ? 0 : 1
 247   else
 248     raise ArgumentError, "Unknown command: #{cmd}"
 249   end
 250 rescue ArgumentError => err
 251   warn "Usage: #{$0} #{cmd} #{err.message}"
 252   exit 1
 253 end
 254 exit 0