3 # * net-http-persistent RubyGem
5 # * upstream MogileFS::Server 2.45 or later
6 $stdout.sync = $stderr.sync = true
8 Usage: #$0 -t TRACKERS"
10 The output of this script can be piped to awk + curl to DELETE the files:
11 #$0 -t TRACKERS | awk '{system("curl -XDELETE "$3)}'
17 require 'net/http/persistent'
18 Thread.abort_on_exception = true
19 MogileFS::VERSION <= "3.0.0" and
20 abort "Upgrade mogilefs-client (to a version that distributes this script)" \
21 "MogileFS::Admin#each_fid is probably broken in this version"
25 x.banner = usage.strip
27 x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
28 Array, 'hostnames/IP addresses of trackers') do |args|
32 x.on('-h', '--help', 'Show this help message.') { puts x; exit }
36 adm = MogileFS::Admin.new(:hosts => trackers)
37 NHP = Net::HTTP::Persistent.new(File.basename($0))
38 client = MogileFS::MogileFS.new(:hosts => trackers, :domain => "none")
40 def start_perdev_thread(pfx)
44 while fid_path = todo.shift
45 path = "#{pfx}#{fid_path}"
48 resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
49 done << [ path, resp ]
58 def setup_devices(dev_map, adm)
60 adm.get_hosts.each do |host|
61 hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
64 adm.get_devices.each do |device|
65 pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
66 todo, done = start_perdev_thread(pfx)
71 def check(bad, curfid, rv)
74 when Net::HTTPNotFound # good
76 bad << "#{curfid} #{resp.content_length} #{path}\n"
78 warn "E: #{resp.inspect} (#{resp.class}) #{path}"
83 setup_devices(dev_map, adm)
89 (next_fid..(fidid - 1)).each do |curfid|
90 nfid = sprintf("%010u", curfid)
91 /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
92 fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
94 dev_map.each_key { |todo| todo << fid_path }
95 dev_map.each_value { |done| check(bad, curfid, done.shift) }
99 info = client.file_debug(curfid)
100 abort "BUG: #{info.inspect} found!" if info["fid_dkey"]
101 rescue MogileFS::Backend::UnknownFidError