remove sysrwloop for copy_stream
[ruby-mogilefs-client.git] / bin / mog
blobc62d8188a148e13a327f5c95398d3ad2e358a7de
1 #!/usr/bin/env ruby
2 require 'mogilefs'
3 require 'optparse'
4 [ STDIN, STDOUT, STDERR].each { |io| io.binmode }
6 trap('INT') { exit 130 }
7 trap('PIPE') { exit 0 }
9 # this is to be compatible with config files used by the Perl tools
10 def parse_config_file!(path, overwrite = false)
11 dest = {}
12 File.open(path).each_line do |line|
13 line.strip!
14 if /^(domain|class)\s*=\s*(\S+)/.match(line)
15 dest[$1.to_sym] = $2
16 elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
17 dest[:hosts] = $1.split(/\s*,\s*/)
18 elsif m = /^timeout\s*=\s*(.*)/.match(line)
19 dest[:timeout] = m[1].to_f
20 else
21 STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
22 end
23 end
24 dest
25 end
27 # parse the default config file if one exists
28 def_file = File.expand_path("~/.mogilefs-client.conf")
29 def_cfg = File.exist?(def_file) ? parse_config_file!(def_file) : {}
31 # parse the command-line first, these options take precedence over all else
32 cli_cfg = {}
33 config_file = nil
34 ls_l = false
35 ls_h = false
36 test = {}
37 cat = { :raw => false }
39 ARGV.options do |x|
40 x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
41 x.separator ''
43 x.on('-c', '--config=/path/to/config',
44 'config file to load') { |file| config_file = file }
46 x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]', Array,
47 'hostnames/IP addresses of trackers') do |trackers|
48 cli_cfg[:hosts] = trackers
49 end
51 x.on('-e', 'True if key exists') { test[:e] = true }
52 x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
54 x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
55 x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
56 x.on('-l', "long listing format (`ls' command)") { ls_l = true }
57 x.on('-h', '--human-readable',
58 "print sizes in human-readable format (`ls' command)") { ls_h = true }
60 x.separator ''
61 x.on('--help', 'Show this help message.') { puts x; exit }
62 x.parse!
63 end
65 # parse the config file specified at the command-line
66 file_cfg = config_file ? parse_config_file!(config_file, true) : {}
68 # read environment variables, too. This Ruby API favors the term
69 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
70 # Favor the term more consistent with what the MogileFS inventors used.
71 env_cfg = {}
72 if ENV["MOG_TRACKERS"]
73 env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
74 end
75 if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
76 env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
77 end
78 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
79 env_cfg[:class] = ENV["MOG_CLASS"] if ENV["MOG_CLASS"]
81 # merge the configs, favoring them in order specified:
82 cfg = {}.merge(def_cfg).merge(env_cfg).merge(file_cfg).merge(cli_cfg)
84 # error-checking
85 err = []
86 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
87 err << "domain must be specified" unless cfg[:domain]
88 if err.any?
89 STDERR.puts "Errors:\n #{err.join("\n ")}"
90 STDERR.puts ARGV.options
91 exit 1
92 end
94 unless cmd = ARGV.shift
95 STDERR.puts ARGV.options
96 exit 1
97 end
99 cfg[:timeout] ||= 30 # longer timeout for interactive use
100 include MogileFS::Util
101 mg = MogileFS::MogileFS.new(cfg)
103 def store_file_retry(mg, key, storage_class, filepath)
104 tries = 0
105 begin
106 mg.store_file(key, storage_class, filepath)
107 rescue MogileFS::UnreadableSocketError,
108 MogileFS::Backend::NoDevicesError => err
109 if ((tries += 1) < 10)
110 STDERR.puts "Retrying on error: #{err}: #{err.message} tries: #{tries}"
111 retry
112 else
113 STDERR.puts "FATAL: #{err}: #{err.message} tries: #{tries}"
115 exit 1
119 begin
120 case cmd
121 when 'cp'
122 filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
123 key = ARGV.shift or raise ArgumentError, '<filename> <key>'
124 ARGV.shift and raise ArgumentError, '<filename> <key>'
125 store_file_retry(mg, key, cfg[:class], filename)
126 when 'cat'
127 ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
128 ARGV.each do |key|
129 if (!cat[:raw] && key =~ /^_big_info:/)
130 mg.bigfile_write(key, STDOUT, {:verify => true})
131 else
132 mg.get_file_data(key) { |fp| copy_stream(fp, STDOUT) }
135 when 'ls'
136 prefixes = ARGV.empty? ? [ nil ] : ARGV
137 prefixes.each do |prefix|
138 mg.each_key(prefix) do |key|
139 if ls_l
140 path_nr = "% 2d" % mg.get_paths(key).size
141 size = mg.size(key)
142 if ls_h && size > 1024
143 suff = ''
144 %w(K M G).each do |s|
145 size /= 1024.0
146 suff = s
147 break if size <= 1024
149 size = sprintf("%.1f%s", size, suff)
150 else
151 size = size.to_s
153 size = (' ' * (12 - size.length)) << size # right justify
154 puts [ path_nr, size, key ].pack("A4 A16 A32")
155 else
156 puts key
160 when 'rm'
161 ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
162 ARGV.each { |key| mg.delete(key) }
163 when 'mv'
164 from = ARGV.shift or raise ArgumentError, '<from> <to>'
165 to = ARGV.shift or raise ArgumentError, '<from> <to>'
166 ARGV.shift and raise ArgumentError, '<from> <to>'
167 mg.rename(from, to)
168 when 'stat' # this outputs a RFC822-like format
169 ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
170 ARGV.each_with_index do |key, i|
171 if size = mg.size(key)
172 puts "Key: #{key}"
173 puts "Size: #{size}"
174 mg.get_paths(key).each_with_index do |path,i|
175 puts "URL-#{i}: #{path}"
177 puts ""
178 else
179 STDERR.puts "No such key: #{key}"
182 when 'tee'
183 require 'tempfile'
184 key = ARGV.shift or raise ArgumentError, '<key>'
185 ARGV.shift and raise ArgumentError, '<key>'
186 cfg[:class] or raise ArgumentError, 'E: --class must be specified'
187 buf = ''
188 tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
190 # if stdout is pointing to /dev/null, don't bother installing the filter.
191 STDOUT.sync = true
192 tee_obj = tmp
193 if File.stat('/dev/null') != STDOUT.stat
194 tee_obj = lambda do |buf|
195 rv = nil
196 [ STDOUT, tmp ].each { |io| rv = io.write(buf) }
199 def tee_obj.write(buf)
200 self[buf]
203 begin
204 copy_stream(STDIN, tee_obj)
205 store_file_retry(mg, key, cfg[:class], tmp.path)
206 ensure
207 tmp.close!
209 when 'test'
210 truth, ok = true, nil
211 raise ArgumentError, "-e must be specified" unless (test.size == 1)
213 truth, key = case ARGV.size
214 when 1
215 [ true, ARGV[0] ]
216 when 2
217 if ARGV[0] != "!"
218 raise ArgumentError, "#{ARGV[0]}: binary operator expected"
220 [ false, ARGV[1] ]
221 else
222 raise ArgumentError, "Too many arguments"
225 paths = mg.get_paths(key)
226 if test[:e]
227 ok = !!(paths && paths.size > 0)
228 else
229 raise ArgumentError, "Unknown flag: -#{test.keys.first}"
232 truth or ok = ! ok
233 exit ok ? 0 : 1
234 else
235 raise ArgumentError, "Unknown command: #{cmd}"
237 rescue ArgumentError => err
238 STDERR.puts "Usage: #{$0} #{cmd} #{err.message}"
239 exit 1
241 exit 0