examples/stale_fid_checker.rb

   1 #!/usr/bin/env ruby
   2 # This requires:
   3 # * net-http-persistent RubyGem
   4 # * Ruby 1.9.2+
   5 # * upstream MogileFS::Server 2.45 or later
   6 $stdout.sync = $stderr.sync = true
   7 require 'uri'
   8 require 'optparse'
   9 require 'mogilefs'
  10 require 'net/http/persistent'
  11 Thread.abort_on_exception = true
  12
  13 trackers = []
  14 ARGV.options do |x|
  15   x.banner = "Usage: #$0 -t TRACKERS"
  16   x.separator ''
  17   x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
  18        Array, 'hostnames/IP addresses of trackers') do |args|
  19     trackers = args
  20   end
  21   x.parse!
  22 end
  23
  24 adm = MogileFS::Admin.new(:hosts => trackers)
  25 NHP = Net::HTTP::Persistent.new(File.basename($0))
  26
  27 def start_perdev_thread(pfx)
  28   todo = Queue.new
  29   done = Queue.new
  30   Thread.new do
  31     while fid_path = todo.shift
  32       path = "#{pfx}#{fid_path}"
  33       uri = URI.parse(path)
  34       begin
  35         resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
  36         done << [ path, resp ]
  37       rescue => err
  38         done << [ path, err ]
  39       end
  40     end
  41   end
  42   [ todo, done ]
  43 end
  44
  45 def setup_devices(dev_map, adm)
  46   hosts = {}
  47   adm.get_hosts.each do |host|
  48     hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
  49   end
  50
  51   adm.get_devices.each do |device|
  52     pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
  53     todo, done = start_perdev_thread(pfx)
  54     dev_map[todo] = done
  55   end
  56 end
  57
  58 def check(bad, curfid, rv)
  59   path, resp = rv
  60   case resp
  61   when Net::HTTPNotFound # good
  62   when Net::HTTPOK
  63     bad << "#{curfid} #{resp.content_length} #{path}\n"
  64   else
  65     warn "E: #{resp.inspect} (#{resp.class}) #{path}"
  66   end
  67 end
  68
  69 dev_map = {}
  70 setup_devices(dev_map, adm)
  71 next_fid = 0
  72 adm.each_fid do |fid|
  73   fidid = fid["fid"]
  74
  75   if fidid != next_fid
  76     (next_fid..(fidid - 1)).each do |curfid|
  77       nfid = sprintf("%010u", curfid)
  78       /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
  79       fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
  80       bad = []
  81       dev_map.each_key { |todo| todo << fid_path }
  82       dev_map.each_value { |done| check(bad, curfid, done.shift) }
  83       next if bad.empty?
  84
  85       puts bad.join
  86     end
  87   end
  88   next_fid = fidid + 1
  89 end