bin/mog

   1 #!/usr/bin/env ruby
   2 require 'mogilefs'
   3 require 'optparse'
   4 $stdin.binmode
   5 $stdout.binmode
   6 $stderr.sync = $stdout.sync = true
   7
   8 trap('INT') { exit 130 }
   9 trap('PIPE') { exit 0 }
  10 if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
  11   md5_trailer_nodes.split(/\s*,\s*/).each do |host|
  12     MogileFS::HTTPFile::MD5_TRAILER_NODES[host] = true
  13   end
  14 end
  15
  16 # this is to be compatible with config files used by the Perl tools
  17 def parse_config_file!(path, dest = {})
  18   File.open(path).each_line do |line|
  19     case line
  20     when /^(domain|class)\s*=\s*(\S+)/
  21       dest[$1.to_sym] = $2
  22     when /^(?:trackers|hosts)\s*=\s*(.*)/
  23       dest[:hosts] = $1.split(/\s*,\s*/)
  24     when /^timeout\s*=\s*(.*)/
  25       dest[:timeout] = $1.to_f
  26     when /^noclobber\s*=\s*true\s*/
  27       dest[:noclobber] = true
  28     else
  29       warn "Ignored configuration line: #{line}" unless /^#/.match(line)
  30     end
  31   end
  32   dest
  33 end
  34
  35 # parse the default config file if one exists
  36 def_file = File.expand_path("~/.mogilefs-client.conf")
  37 def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
  38
  39 # parse the command-line first, these options take precedence over all else
  40 cli_cfg = {}
  41 config_file = nil
  42 ls_l = false
  43 ls_h = false
  44 chunk = false
  45 range = false
  46 test = {}
  47 cat = { :raw => false }
  48
  49 ARGV.options do |x|
  50   x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
  51   x.separator ''
  52
  53   x.on('-c', '--config=/path/to/config',
  54        'config file to load') { |file| config_file = file }
  55
  56   x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
  57        'hostnames/IP addresses of trackers') do |trackers|
  58     cli_cfg[:hosts] = trackers
  59   end
  60
  61   x.on('-e', 'True if key exists') { test[:e] = true }
  62   x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
  63   x.on('-n', '--no-clobber', 'do not clobber existing key') do
  64     cli_cfg[:noclobber] = true
  65   end
  66
  67   x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
  68   x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
  69   x.on('-l', "long listing format (`ls' command)") { ls_l = true }
  70   x.on('-h', '--human-readable',
  71        "print sizes in human-readable format (`ls' command)") { ls_h = true }
  72   x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
  73   x.on('--range', "stream partial uploads (`tee' command)") { range = true }
  74   x.separator ''
  75   x.on('--help', 'Show this help message.') { puts x; exit }
  76   x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
  77   x.parse!
  78 end
  79
  80 # parse the config file specified at the command-line
  81 file_cfg = config_file ? parse_config_file!(config_file) : {}
  82
  83 # read environment variables, too.  This Ruby API favors the term
  84 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
  85 # Favor the term more consistent with what the MogileFS inventors used.
  86 env_cfg = {}
  87 if ENV["MOG_TRACKERS"]
  88   env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
  89 end
  90 if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
  91   env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
  92 end
  93 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
  94 env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
  95
  96 # merge the configs, favoring them in order specified:
  97 cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
  98
  99 # error-checking
 100 err = []
 101 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
 102 err << "domain must be specified" unless cfg[:domain]
 103 if err.any?
 104   warn "Errors:\n  #{err.join("\n  ")}"
 105   warn ARGV.options
 106   exit 1
 107 end
 108
 109 unless cmd = ARGV.shift
 110   warn ARGV.options
 111   exit 1
 112 end
 113
 114 cfg[:timeout] ||= 30 # longer timeout for interactive use
 115 mg = MogileFS::MogileFS.new(cfg)
 116
 117 def store_file_retry(mg, key, storage_class, filepath)
 118   tries = 0
 119   begin
 120     mg.store_file(key, storage_class, filepath)
 121   rescue MogileFS::UnreadableSocketError,
 122          MogileFS::Backend::NoDevicesError => err
 123     if ((tries += 1) < 10)
 124       warn "Retrying on error: #{err}: #{err.message} tries: #{tries}"
 125       retry
 126     else
 127       warn "FATAL: #{err}: #{err.message} tries: #{tries}"
 128     end
 129     exit 1
 130   end
 131 end
 132
 133 def human_size(size)
 134   suff = ''
 135   %w(K M G).each do |s|
 136     size /= 1024.0
 137     if size <= 1024
 138       suff = s
 139       break
 140     end
 141   end
 142   sprintf("%.1f%s", size, suff)
 143 end
 144
 145 begin
 146   case cmd
 147   when 'cp'
 148     filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
 149     dkey = ARGV.shift or raise ArgumentError, '<filename> <key>'
 150     ARGV.shift and raise ArgumentError, '<filename> <key>'
 151     cfg[:noclobber] && mg.exist?(dkey) and
 152       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
 153     store_file_retry(mg, dkey, cfg[:class], filename)
 154   when 'cat'
 155     ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
 156     ARGV.each do |key|
 157       if (!cat[:raw] && key =~ /^_big_info:/)
 158         mg.bigfile_write(key, $stdout, {:verify => true})
 159       else
 160         mg.get_file_data(key, $stdout)
 161       end
 162     end
 163   when 'ls'
 164     prefixes = ARGV.empty? ? [ nil ] : ARGV
 165     if ls_l
 166       each_key = lambda do |key, size, devcount|
 167         size = ls_h && size > 1024 ? human_size(size) : size.to_s
 168         size = (' ' * (12 - size.length)) << size # right justify
 169         puts [ sprintf("% 2d", devcount), size, key ].pack("A4 A16 A*")
 170       end
 171     else
 172       each_key = lambda { |key| puts key }
 173     end
 174     prefixes.each { |prefix| mg.each_key(prefix, &each_key) }
 175   when 'rm'
 176     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 177     ARGV.each { |key| mg.delete(key) }
 178   when 'mv'
 179     from = ARGV.shift or raise ArgumentError, '<from> <to>'
 180     to = ARGV.shift or raise ArgumentError, '<from> <to>'
 181     ARGV.shift and raise ArgumentError, '<from> <to>'
 182     mg.rename(from, to)
 183   when 'stat' # this outputs a RFC822-like format
 184     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 185     ok = true
 186     ARGV.each_with_index do |key,j|
 187       begin
 188         info = mg.file_info(key)
 189         puts "Key: #{key}"
 190         puts "Size: #{info['length']}"
 191         puts "Class: #{info['class']}"
 192         checksum = info['checksum'] and puts "Checksum: #{checksum}"
 193         o = { :pathcount => info["devcount"] }
 194         mg.get_paths(key, o).each_with_index do |path,i|
 195           puts "URL-#{i}: #{path}"
 196         end
 197         puts "" if ARGV.size != (j + 1)
 198       rescue MogileFS::Backend::UnknownKeyError
 199         warn "No such key: #{key}"
 200         ok = false
 201       end
 202     end
 203     exit(ok)
 204   when 'tee'
 205     abort "--range and --chunk are incompatible" if range && chunk
 206     dkey = ARGV.shift or raise ArgumentError, '<key>'
 207     ARGV.shift and raise ArgumentError, '<key>'
 208     cfg[:noclobber] && mg.exist?(dkey) and
 209       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
 210     skip_tee = File.stat('/dev/null') == $stdout.stat
 211     largefile = :tempfile
 212     largefile = :content_range if range
 213     largefile = :stream if chunk
 214
 215     io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile)
 216     begin
 217       buf = $stdin.readpartial(16384)
 218       begin
 219         io.write(buf)
 220         $stdout.write(buf) unless skip_tee
 221         $stdin.readpartial(16384, buf)
 222       end while true
 223     rescue EOFError
 224     end
 225     io.close
 226   when 'test'
 227     truth, ok = true, nil
 228     raise ArgumentError, "-e must be specified" unless (test.size == 1)
 229
 230     truth, key = case ARGV.size
 231     when 1
 232       [ true, ARGV[0] ]
 233     when 2
 234       if ARGV[0] != "!"
 235         raise ArgumentError, "#{ARGV[0]}: binary operator expected"
 236       end
 237       [ false, ARGV[1] ]
 238     else
 239       raise ArgumentError, "Too many arguments"
 240     end
 241
 242     test[:e] or raise ArgumentError, "Unknown flag: -#{test.keys.first}"
 243     ok = mg.exist?(key)
 244     truth or ok = ! ok
 245     exit ok ? 0 : 1
 246   else
 247     raise ArgumentError, "Unknown command: #{cmd}"
 248   end
 249 rescue ArgumentError => err
 250   warn "Usage: #{$0} #{cmd} #{err.message}"
 251   exit 1
 252 end
 253 exit 0