bin/mog

   1 #!/usr/bin/env ruby
   2 require 'mogilefs'
   3 require 'optparse'
   4
   5 trap('INT') { exit 130 }
   6 trap('PIPE') { exit 0 }
   7
   8 # this is to be compatible with config files used by the Perl tools
   9 def parse_config_file!(path, overwrite = false)
  10   dest = {}
  11   File.open(path).each_line do |line|
  12     line.strip!
  13     if /^(domain|class)\s*=\s*(\S+)/.match(line)
  14       dest[$1.to_sym] = $2
  15     elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
  16       dest[:hosts] = $1.split(/\s*,\s*/)
  17     elsif m = /^timeout\s*=\s*(.*)/.match(line)
  18       dest[:timeout] = m[1].to_f
  19     else
  20       STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
  21     end
  22   end
  23   dest
  24 end
  25
  26 # parse the default config file if one exists
  27 def_file = File.expand_path("~/.mogilefs-client.conf")
  28 def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
  29
  30 # parse the command-line first, these options take precedence over all else
  31 cli_cfg = {}
  32 config_file = nil
  33 ls_l = false
  34 ls_h = false
  35 test = {}
  36
  37 ARGV.options do |x|
  38   x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
  39   x.separator ''
  40
  41   x.on('-c', '--config=/path/to/config',
  42        'config file to load') { |file| config_file = file }
  43
  44   x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
  45        'hostnames/IP addresses of trackers') do |trackers|
  46     cli_cfg[:hosts] = trackers
  47   end
  48
  49   x.on('-e', 'True if key exists') { test[:e] = true }
  50
  51   x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
  52   x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
  53   x.on('-l', "long listing format (`ls' command)") { ls_l = true }
  54   x.on('-h', '--human-readable',
  55        "print sizes in human-readable format (`ls' command)") { ls_h = true }
  56
  57   x.separator ''
  58   x.on('--help', 'Show this help message.') { puts x; exit }
  59   x.parse!
  60 end
  61
  62 # parse the config file specified at the command-line
  63 file_cfg = config_file ? parse_config_file!(config_file, true) : {}
  64
  65 # read environment variables, too.  This Ruby API favors the term
  66 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
  67 # Favor the term more consistent with what the MogileFS inventors used.
  68 env_cfg = {}
  69 if ENV["MOG_TRACKERS"]
  70   env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
  71 end
  72 if ENV["MOG_HOSTS"] && env_cfg[:hosts].empty?
  73   env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
  74 end
  75 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
  76 env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
  77
  78 # merge the configs, favoring them in order specified:
  79 cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
  80
  81 # error-checking
  82 err = []
  83 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
  84 err << "domain must be specified" unless cfg[:domain]
  85 if err.any?
  86   STDERR.puts "Errors:\n  #{err.join("\n  ")}"
  87   STDERR.puts ARGV.options
  88   exit 1
  89 end
  90
  91 unless cmd = ARGV.shift
  92   STDERR.puts ARGV.options
  93   exit 1
  94 end
  95
  96 cfg[:timeout] ||= 30 # longer timeout for interactive use
  97 include MogileFS::Util
  98 mg = MogileFS::MogileFS.new(cfg)
  99
 100 def store_file_retry(mg, key, storage_class, filepath)
 101   tries = 0
 102   begin
 103     mg.store_file(key, storage_class, filepath)
 104   rescue MogileFS::UnreadableSocketError,
 105          MogileFS::Backend::NoDevicesError => err
 106     if ((tries += 1) < 10)
 107       STDERR.puts "Retrying on error: #{err}: #{err.message} tries: #{tries}"
 108       retry
 109     else
 110       STDERR.puts "FATAL: #{err}: #{err.message} tries: #{tries}"
 111     end
 112     exit 1
 113   end
 114 end
 115
 116 begin
 117   case cmd
 118   when 'cp'
 119     filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
 120     key = ARGV.shift or raise ArgumentError, '<filename> <key>'
 121     ARGV.shift and raise ArgumentError, '<filename> <key>'
 122     cfg[:class] or raise ArgumentError, 'E: --class must be specified'
 123     store_file_retry(mg, key, cfg[:class], filename)
 124   when 'cat'
 125     ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
 126     ARGV.each { |key| mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) } }
 127   when 'ls'
 128     prefixes = ARGV.empty? ? [ nil ] : ARGV
 129     prefixes.each do |prefix|
 130       mg.each_key(prefix) do |key|
 131         if ls_l
 132           path_nr = "% 2d" % mg.get_paths(key).size
 133           size = mg.size(key)
 134           if ls_h && size > 1024
 135             suff = ''
 136             %w(K M G).each do |s|
 137               size /= 1024.0
 138               suff = s
 139               break if size <= 1024
 140             end
 141             size = sprintf("%.1f%s", size, suff)
 142           else
 143             size = size.to_s
 144           end
 145           size = (' ' * (12 - size.length)) << size # right justify
 146           puts [ path_nr, size, key ].pack("A4 A16 A32")
 147         else
 148           puts key
 149         end
 150       end
 151     end
 152   when 'rm'
 153     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 154     ARGV.each { |key| mg.delete(key) }
 155   when 'mv'
 156     from = ARGV.shift or raise ArgumentError, '<from> <to>'
 157     to = ARGV.shift or raise ArgumentError, '<from> <to>'
 158     ARGV.shift and raise ArgumentError, '<from> <to>'
 159     mg.rename(from, to)
 160   when 'stat' # this outputs a RFC822-like format
 161     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
 162     ARGV.each_with_index do |key, i|
 163       if size = mg.size(key)
 164         puts "Key: #{key}"
 165         puts "Size: #{size}"
 166         mg.get_paths(key).each_with_index do |path,i|
 167           puts "URL-#{i}: #{path}"
 168         end
 169         puts ""
 170       else
 171         STDERR.puts "No such key: #{key}"
 172       end
 173     end
 174   when 'tee'
 175     require 'tempfile'
 176     key = ARGV.shift or raise ArgumentError, '<key>'
 177     ARGV.shift and raise ArgumentError, '<key>'
 178     cfg[:class] or raise ArgumentError, 'E: --class must be specified'
 179     buf = ''
 180     tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
 181     at_exit { tmp.unlink }
 182     begin
 183       sysrwloop(STDIN, tmp)
 184       store_file_retry(mg, key, cfg[:class], tmp.path)
 185     ensure
 186       tmp.close
 187     end
 188   when 'test'
 189     truth, ok = true, nil
 190     raise ArgumentError, "-e must be specified" unless (test.size == 1)
 191
 192     truth, key = case ARGV.size
 193     when 1
 194       [ true, ARGV[0] ]
 195     when 2
 196       if ARGV[0] != "!"
 197         raise ArgumentError, "#{ARGV[0]}: binary operator expected"
 198       end
 199       [ false, ARGV[1] ]
 200     else
 201       raise ArgumentError, "Too many arguments"
 202     end
 203
 204     paths = mg.get_paths(key)
 205     if test[:e]
 206       ok = !!(paths && paths.size > 0)
 207     else
 208       raise ArgumentError, "Unknown flag: -#{test.keys.first}"
 209     end
 210
 211     truth or ok = ! ok
 212     exit ok ? 0 : 1
 213   else
 214     raise ArgumentError, "Unknown command: #{cmd}"
 215   end
 216 rescue ArgumentError => err
 217   STDERR.puts "Usage: #{$0} #{cmd} #{err.message}"
 218   exit 1
 219 end
 220 exit 0