bin/mog

   1 #!/usr/bin/env ruby
   2 require 'mogilefs'
   3 require 'optparse'
   4 $stdin.binmode
   5 $stdout.binmode
   6 $stderr.sync = $stdout.sync = true
   7
   8 trap('INT') { exit 130 }
   9 trap('PIPE') { exit 0 }
  10 if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
  11   md5_trailer_nodes.split(/\s*,\s*/).each do |host|
  12     MogileFS::HTTPFile::MD5_TRAILER_NODES[host] = true
  13   end
  14 end
  15
  16 # this is to be compatible with config files used by the Perl tools
  17 def parse_config_file!(path, dest = {})
  18   File.open(path).each_line do |line|
  19     case line
  20     when /^(domain|class)\s*=\s*(\S+)/
  21       dest[$1.to_sym] = $2
  22     when /^(?:trackers|hosts)\s*=\s*(.*)/
  23       dest[:hosts] = $1.split(/\s*,\s*/)
  24     when /^timeout\s*=\s*(.*)/
  25       dest[:timeout] = $1.to_f
  26     when /^noclobber\s*=\s*true\s*/
  27       dest[:noclobber] = true
  28     else
  29       warn "Ignored configuration line: #{line}" unless /^#/.match(line)
  30     end
  31   end
  32   dest
  33 end
  34
  35 # parse the default config file if one exists
  36 def_file = File.expand_path("~/.mogilefs-client.conf")
  37 def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
  38
  39 # parse the command-line first, these options take precedence over all else
  40 cli_cfg = {}
  41 config_file = nil
  42 ls_l = false
  43 ls_h = false
  44 chunk = false
  45 range = false
  46 test = {}
  47 cat = { :raw => false }
  48
  49 ARGV.options do |x|
  50   x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
  51   x.separator ''
  52
  53   x.on('-c', '--config=/path/to/config',
  54        'config file to load') { |file| config_file = file }
  55
  56   x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
  57        'hostnames/IP addresses of trackers') do |trackers|
  58     cli_cfg[:hosts] = trackers
  59   end
  60
  61   x.on('-e', 'True if key exists') { test[:e] = true }
  62   x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
  63   x.on('-n', '--no-clobber', 'do not clobber existing key') do
  64     cli_cfg[:noclobber] = true
  65   end
  66
  67   x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
  68   x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
  69   x.on('-l', "long listing format (`ls' command)") { ls_l = true }
  70   x.on('-h', '--human-readable',
  71        "print sizes in human-readable format (`ls' command)") { ls_h = true }
  72   x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
  73   x.on('--range', "stream partial uploads (`tee' command)") { range = true }
  74   x.separator ''
  75   x.on('--help', 'Show this help message.') { puts x; exit }
  76   x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
  77   x.parse!
  78 end
  79
  80 # parse the config file specified at the command-line
  81 file_cfg = config_file ? parse_config_file!(config_file) : {}
  82
  83 # read environment variables, too.  This Ruby API favors the term
  84 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
  85 # Favor the term more consistent with what the MogileFS inventors used.
  86 env_cfg = {}
  87 if ENV["MOG_TRACKERS"]
  88   env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
  89 end
  90 if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
  91   env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
  92 end
  93 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
  94 env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
  95
  96 # merge the configs, favoring them in order specified:
  97 cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
  98
  99 # error-checking
 100 err = []
 101 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
 102 err << "domain must be specified" unless cfg[:domain]
 103 if err.any?
 104   warn "Errors:\n  #{err.join("\n  ")}"
 105   warn ARGV.options
 106   exit 1
 107 end
 108
 109 unless cmd = ARGV.shift
 110   warn ARGV.options
 111   exit 1
 112 end
 113
 114 cfg[:timeout] ||= 30 # longer timeout for interactive use
 115 mg = MogileFS::MogileFS.new(cfg)
 116
 117 def store_file_retry(mg, key, storage_class, filepath)
 118   tries = 0
 119   begin
 120     mg.store_file(key, storage_class, filepath)
 121   rescue MogileFS::UnreadableSocketError,
 122          MogileFS::Backend::NoDevicesError => err
 123     if ((tries += 1) < 10)
 124       warn "Retrying on error: #{err}: #{err.message} tries: #{tries}"
 125       retry
 126     else
 127       warn "FATAL: #{err}: #{err.message} tries: #{tries}"
 128     end
 129     exit 1
 130   end
 131 end
 132
 133 def human_size(size)
 134   suff = ''
 135   %w(K M G).each do |s|
 136     size /= 1024.0
 137     if size <= 1024
 138       suff = s
 139       break
 140     end
 141   end
 142   sprintf("%.1f%s", size, suff)
 143 end
 144
 145 begin
 146   case cmd
 147   when 'cp'
 148     filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
 149     dkey = ARGV.shift or raise ArgumentError, '<filename> <key>'
 150     ARGV.shift and raise ArgumentError, '<filename> <key>'
 151     cfg[:noclobber] && mg.exist?(dkey) and
 152       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
 153     store_file_retry(mg, dkey, cfg[:class], filename)
 154   when 'cat'
 155     ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
 156     ARGV.each do |key|
 157       if (!cat[:raw] && key =~ /^_big_info:/)
 158         mg.bigfile_write(key, $stdout, {:verify => true})
 159       else
 160         mg.get_file_data(key, $stdout)
 161       end
 162     end
 163   when 'updateclass'
 164     newclass = cfg[:class] or abort '-C/--class not specified'
 165     ARGV.each do |key|
 166       mg.updateclass(key, newclass)
 167     end
 168   when 'ls'
 169     prefixes = ARGV.empty? ? [ nil ] : ARGV
 170     if ls_l
 171       each_key = lambda do |key, size, devcount|
 172         size = ls_h && size > 1024 ? human_size(size) : size.to_s
 173         size = (' ' * (12 - size.length)) << size # right justify
 174         puts [ sprintf("% 2d", devcount), size, key ].pack("A4 A16 A*")
 175       end
 176     else
 177       each_key = lambda { |key| puts key }
 178     end
 179     prefixes.each { |prefix| mg.each_key(prefix, &each_key) }
 180   when 'rm'
 181     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 182     ARGV.each { |key| mg.delete(key) }
 183   when 'mv'
 184     from = ARGV.shift or raise ArgumentError, '<from> <to>'
 185     to = ARGV.shift or raise ArgumentError, '<from> <to>'
 186     ARGV.shift and raise ArgumentError, '<from> <to>'
 187     mg.rename(from, to)
 188   when 'stat' # this outputs a RFC822-like format
 189     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 190     ok = true
 191     ARGV.each_with_index do |key,j|
 192       begin
 193         info = mg.file_info(key)
 194         puts "Key: #{key}"
 195         puts "Size: #{info['length']}"
 196         puts "Class: #{info['class']}"
 197         checksum = info['checksum'] and puts "Checksum: #{checksum}"
 198         o = { :pathcount => info["devcount"] }
 199         mg.get_paths(key, o).each_with_index do |path,i|
 200           puts "URL-#{i}: #{path}"
 201         end
 202         puts "" if ARGV.size != (j + 1)
 203       rescue MogileFS::Backend::UnknownKeyError
 204         warn "No such key: #{key}"
 205         ok = false
 206       end
 207     end
 208     exit(ok)
 209   when 'tee'
 210     abort "--range and --chunk are incompatible" if range && chunk
 211     dkey = ARGV.shift or raise ArgumentError, '<key>'
 212     ARGV.shift and raise ArgumentError, '<key>'
 213     cfg[:noclobber] && mg.exist?(dkey) and
 214       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
 215     skip_tee = File.stat('/dev/null') == $stdout.stat
 216     largefile = :tempfile
 217     largefile = :content_range if range
 218     largefile = :stream if chunk
 219
 220     io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile)
 221     begin
 222       buf = $stdin.readpartial(16384)
 223       begin
 224         io.write(buf)
 225         $stdout.write(buf) unless skip_tee
 226         $stdin.readpartial(16384, buf)
 227       end while true
 228     rescue EOFError
 229     end
 230     io.close
 231   when 'test'
 232     truth, ok = true, nil
 233     raise ArgumentError, "-e must be specified" unless (test.size == 1)
 234
 235     truth, key = case ARGV.size
 236     when 1
 237       [ true, ARGV[0] ]
 238     when 2
 239       if ARGV[0] != "!"
 240         raise ArgumentError, "#{ARGV[0]}: binary operator expected"
 241       end
 242       [ false, ARGV[1] ]
 243     else
 244       raise ArgumentError, "Too many arguments"
 245     end
 246
 247     test[:e] or raise ArgumentError, "Unknown flag: -#{test.keys.first}"
 248     ok = mg.exist?(key)
 249     truth or ok = ! ok
 250     exit ok ? 0 : 1
 251   else
 252     raise ArgumentError, "Unknown command: #{cmd}"
 253   end
 254 rescue ArgumentError => err
 255   warn "Usage: #{$0} #{cmd} #{err.message}"
 256   exit 1
 257 end
 258 exit 0