mogfiledebug

   1 #!/usr/bin/perl
   2
   3 =head1 NAME
   4
   5 mogfiledebug -- Dump gobs of information about a FID
   6
   7 =head1 SYNOPSIS
   8
   9     $ mogfiledebug --trackers=host --domain=foo --key=bar
  10     $ mogfiledebug --trackers=host --fid=1234
  11
  12 =head1 DESCRIPTION
  13
  14 Utility for troubleshooting problemic files in a mogilefs cluster. Also useful
  15 for verification or testing new setups.
  16
  17 Finds as much information about a file as it can. All of the paths, any queues
  18 it might be sitting in, etc. Will then test all of the paths, MD5 hash their
  19 contents, and check the file lengths. If you see errors about a FID in
  20 mogilefsd's logs plugging it through mogfiledebug should illuminate most of
  21 the potential issues.
  22
  23 This is also useful information for posting to the mailing list, along with
  24 the error you had.
  25
  26 =head1 OPTIONS
  27
  28 =over
  29
  30 =item --trackers=host1:7001,host2:7001
  31
  32 Use these MogileFS trackers to negotiate with.
  33
  34 =item --domain=<domain>
  35
  36 Set the MogileFS domain to use.
  37
  38 =item --key="<key>"
  39
  40 The key to inspect. Can be an arbitrary string.
  41
  42 =item --fid=<fid>
  43
  44 A numeric fid to inspect. Provide this as an alternative to a domain/key
  45 combination.
  46
  47 =back
  48
  49 =head1 AUTHOR
  50
  51 Dormando E<lt>L<dormando@rydia.net>E<gt>
  52
  53 =head1 BUGS
  54
  55 None known. Could use more helpful prints, or a longer troubleshooting manual.
  56
  57 =head1 LICENSE
  58
  59 Licensed for use and redistribution under the same terms as Perl itself.
  60
  61 =cut
  62
  63 use strict;
  64 use warnings;
  65
  66 use lib './lib';
  67 use MogileFS::Utils;
  68 use Digest::MD5;
  69 use LWP::UserAgent;
  70
  71 my $util = MogileFS::Utils->new;
  72 my $usage = qq{--trackers=host --domain=foo --key='/hello.jpg'
  73 If FID is known, but domain/key are not known:
  74 --trackers=host --fid=123456};
  75 # FIXME: add "nofetch" mode that just prints paths?
  76 my $c = $util->getopts($usage, qw/key=s fid=i/);
  77
  78 my $arg;
  79 if ($c->{fid}) {
  80         $c->{domain} ||= 'mogfiledebug-unset';
  81         $arg = 'fid';
  82 } else {
  83         $arg = 'key';
  84 }
  85
  86 my $mogc = $util->client;
  87 my $details = $mogc->file_debug($arg => $c->{$arg});
  88 if ($mogc->errcode) {
  89     die "Error fetching fid info: " . $mogc->errstr;
  90 }
  91
  92 my %parts = ();
  93 my @paths = grep { $_ =~ m/^devpath_/ } keys %$details;
  94 while (my ($k, $v) = each %$details) {
  95     next if $k =~ m/^devpath_/;
  96     if ($k =~ s/^(\w+)_//) {
  97         $parts{$1}->{$k} = $v;
  98     }
  99 }
 100
 101 # If no paths, print something about that.
 102 if (@paths) {
 103     my @results;
 104     # For each actual path, fetch and calculate the MD5SUM.
 105     print "Fetching and summing paths...\n";
 106     for my $key (@paths) {
 107         my $path = $details->{$key};
 108         push(@results, fetch_path($path));
 109     }
 110     my $hash; # detect if hashes don't match
 111     my $len = $parts{fid}->{length};
 112     print "No length, cannot verify content length" unless defined $len;
 113     # No I don't have a good excuse for why this isn't one loop.
 114     for my $res (@results) {
 115         print "\nResults for path: ", $res->{path}, "\n";
 116         if ($res->{res} =~ /404/) {
 117             print " - ERROR: File copy is missing: ", $res->{res}, "\n";
 118             next;
 119         }
 120         $hash = $res->{hash} unless $hash;
 121         if ($hash ne $res->{hash}) {
 122             print " - ERROR: Hash does not match first path!\n";
 123         }
 124         if (defined $len && defined $res->{length} && $len != $res->{length}) {
 125             print " - ERROR: Length does not match file row!\n";
 126         }
 127         print " - MD5 Hash: ", $res->{hash}, "\n";
 128         print " - Length: ", $res->{length}, "\n" if defined $res->{length};
 129         print " - Last-Modified: ", $res->{mtime}, "\n" if defined $res->{mtime};
 130         print " - HTTP result: ", $res->{res}, "\n";
 131     }
 132 } else {
 133     print "No valid-ish paths found\n";
 134 }
 135
 136 # print info from all of the queues. Raw is fine? failcount/etc.
 137 print "\nTempfile and/or queue rows...\n";
 138 my $found = 0;
 139 for my $type (qw/tempfile replqueue delqueue rebqueue fsckqueue/) {
 140     my $part = $parts{$type};
 141     next unless (defined $part);
 142     $found++;
 143     printf("- %12s\n", $type);
 144     while (my ($k, $v) = each %$part) {
 145         printf("  %20s: %20s\n", $k, $v);
 146     }
 147 }
 148 print "none.\n" unless $found;
 149
 150 # Print rest of file info like file_info
 151 if (my $fid = $parts{fid}) {
 152     print "\n- File Row:\n";
 153     for my $item (sort keys %$fid) {
 154         printf("  %8s: %20s\n", $item, $fid->{$item});
 155     }
 156 } else {
 157     print qq{- ERROR: No file row was found!
 158 File may have been deleted or never closed.
 159 See above for any matching rows from tempfile or delqueue.
 160 };
 161 }
 162
 163 if (my $devids = $details->{devids}) {
 164     print "\n- Raw devids: ", $devids, "\n";
 165 }
 166
 167 if (my $hash = $details->{checksum}) {
 168     print "\n- Stored checksum: ", $hash, "\n";
 169 }
 170
 171 sub fetch_path {
 172     my $path = shift;
 173     my $ua   = LWP::UserAgent->new;
 174     my $ctx  = Digest::MD5->new;
 175     $ua->timeout(10);
 176     my %toret = (length => 0);
 177
 178     my $sum_up = sub {
 179         $toret{length} += length($_[0]);
 180         $ctx->add($_[0]);
 181     };
 182     my $res = $ua->get($path, ':content_cb' => $sum_up,
 183         ':read_size_hint' => 32768);
 184
 185     $toret{hash}  = $ctx->hexdigest;
 186     $toret{res}   = $res->status_line;
 187     $toret{mtime} = $res->header("Last-Modified");
 188     $toret{path}  = $path;
 189     return \%toret;
 190 }