mogilefs-client 2.0.0
[ruby-mogilefs-client.git] / bin / mog
blobe8efa8f814d2729739c80ca82cde2c1f5a5b4df4
1 #!/usr/bin/env ruby
2 require 'mogilefs'
3 require 'optparse'
5 trap('INT') { exit 130 }
6 trap('PIPE') { exit 0 }
8 # this is to be compatible with config files used by the Perl tools
9 def parse_config_file!(path, overwrite = false)
10 dest = {}
11 File.open(path).each_line do |line|
12 line.strip!
13 if /^(domain|class)\s*=\s*(\S+)/.match(line)
14 dest[$1.to_sym] = $2
15 elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
16 dest[:hosts] = $1.split(/\s*,\s*/)
17 elsif m = /^timeout\s*=\s*(.*)/.match(line)
18 dest[:timeout] = m[1].to_f
19 else
20 STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
21 end
22 end
23 dest
24 end
26 # parse the default config file if one exists
27 def_file = File.expand_path("~/.mogilefs-client.conf")
28 def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
30 # parse the command-line first, these options take precedence over all else
31 cli_cfg = {}
32 config_file = nil
33 ls_l = false
34 ls_h = false
35 test = {}
36 cat = { :raw => false }
38 ARGV.options do |x|
39 x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
40 x.separator ''
42 x.on('-c', '--config=/path/to/config',
43 'config file to load') { |file| config_file = file }
45 x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
46 'hostnames/IP addresses of trackers') do |trackers|
47 cli_cfg[:hosts] = trackers
48 end
50 x.on('-e', 'True if key exists') { test[:e] = true }
51 x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
53 x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
54 x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
55 x.on('-l', "long listing format (`ls' command)") { ls_l = true }
56 x.on('-h', '--human-readable',
57 "print sizes in human-readable format (`ls' command)") { ls_h = true }
59 x.separator ''
60 x.on('--help', 'Show this help message.') { puts x; exit }
61 x.parse!
62 end
64 # parse the config file specified at the command-line
65 file_cfg = config_file ? parse_config_file!(config_file, true) : {}
67 # read environment variables, too. This Ruby API favors the term
68 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
69 # Favor the term more consistent with what the MogileFS inventors used.
70 env_cfg = {}
71 if ENV["MOG_TRACKERS"]
72 env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
73 end
74 if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
75 env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
76 end
77 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
78 env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
80 # merge the configs, favoring them in order specified:
81 cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
83 # error-checking
84 err = []
85 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
86 err << "domain must be specified" unless cfg[:domain]
87 if err.any?
88 STDERR.puts "Errors:\n #{err.join("\n ")}"
89 STDERR.puts ARGV.options
90 exit 1
91 end
93 unless cmd = ARGV.shift
94 STDERR.puts ARGV.options
95 exit 1
96 end
98 cfg[:timeout] ||= 30 # longer timeout for interactive use
99 include MogileFS::Util
100 mg = MogileFS::MogileFS.new(cfg)
102 def store_file_retry(mg, key, storage_class, filepath)
103 tries = 0
104 begin
105 mg.store_file(key, storage_class, filepath)
106 rescue MogileFS::UnreadableSocketError,
107 MogileFS::Backend::NoDevicesError => err
108 if ((tries += 1) < 10)
109 STDERR.puts "Retrying on error: #{err}: #{err.message} tries: #{tries}"
110 retry
111 else
112 STDERR.puts "FATAL: #{err}: #{err.message} tries: #{tries}"
114 exit 1
118 begin
119 case cmd
120 when 'cp'
121 filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
122 key = ARGV.shift or raise ArgumentError, '<filename> <key>'
123 ARGV.shift and raise ArgumentError, '<filename> <key>'
124 store_file_retry(mg, key, cfg[:class], filename)
125 when 'cat'
126 ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
127 ARGV.each do |key|
128 if (!cat[:raw] && key =~ /^_big_info:/)
129 mg.bigfile_write(key, STDOUT, {:verify => true})
130 else
131 mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) }
134 when 'ls'
135 prefixes = ARGV.empty? ? [ nil ] : ARGV
136 prefixes.each do |prefix|
137 mg.each_key(prefix) do |key|
138 if ls_l
139 path_nr = "% 2d" % mg.get_paths(key).size
140 size = mg.size(key)
141 if ls_h && size > 1024
142 suff = ''
143 %w(K M G).each do |s|
144 size /= 1024.0
145 suff = s
146 break if size <= 1024
148 size = sprintf("%.1f%s", size, suff)
149 else
150 size = size.to_s
152 size = (' ' * (12 - size.length)) << size # right justify
153 puts [ path_nr, size, key ].pack("A4 A16 A32")
154 else
155 puts key
159 when 'rm'
160 ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
161 ARGV.each { |key| mg.delete(key) }
162 when 'mv'
163 from = ARGV.shift or raise ArgumentError, '<from> <to>'
164 to = ARGV.shift or raise ArgumentError, '<from> <to>'
165 ARGV.shift and raise ArgumentError, '<from> <to>'
166 mg.rename(from, to)
167 when 'stat' # this outputs a RFC822-like format
168 ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
169 ARGV.each_with_index do |key, i|
170 if size = mg.size(key)
171 puts "Key: #{key}"
172 puts "Size: #{size}"
173 mg.get_paths(key).each_with_index do |path,i|
174 puts "URL-#{i}: #{path}"
176 puts ""
177 else
178 STDERR.puts "No such key: #{key}"
181 when 'tee'
182 require 'tempfile'
183 key = ARGV.shift or raise ArgumentError, '<key>'
184 ARGV.shift and raise ArgumentError, '<key>'
185 cfg[:class] or raise ArgumentError, 'E: --class must be specified'
186 buf = ''
187 tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
188 at_exit { tmp.unlink }
190 # if stdout is pointing to /dev/null, don't bother installing the filter.
191 STDOUT.sync = true
192 tee_filter = File.stat('/dev/null') == STDOUT.stat ?
193 nil : Proc.new { |buf| STDOUT.write(buf); buf }
194 begin
195 sysrwloop(STDIN, tmp, tee_filter)
196 store_file_retry(mg, key, cfg[:class], tmp.path)
197 ensure
198 tmp.close
200 when 'test'
201 truth, ok = true, nil
202 raise ArgumentError, "-e must be specified" unless (test.size == 1)
204 truth, key = case ARGV.size
205 when 1
206 [ true, ARGV[0] ]
207 when 2
208 if ARGV[0] != "!"
209 raise ArgumentError, "#{ARGV[0]}: binary operator expected"
211 [ false, ARGV[1] ]
212 else
213 raise ArgumentError, "Too many arguments"
216 paths = mg.get_paths(key)
217 if test[:e]
218 ok = !!(paths && paths.size > 0)
219 else
220 raise ArgumentError, "Unknown flag: -#{test.keys.first}"
223 truth or ok = ! ok
224 exit ok ? 0 : 1
225 else
226 raise ArgumentError, "Unknown command: #{cmd}"
228 rescue ArgumentError => err
229 STDERR.puts "Usage: #{$0} #{cmd} #{err.message}"
230 exit 1
232 exit 0