moglistfids

   1 #!/usr/bin/perl
   2
   3 =head1 NAME
   4
   5 moglistfids -- Iterate fid/key data from a MogileFS installation
   6
   7 =head1 DESCRIPTION
   8
   9 Example utility for pulling all file data out of a MogileFS installation.
  10 Utilities like this can be built on for creating backup systems,
  11 Mogile<->Mogile syncronization systems, or Mogile->S3 syncronization.
  12
  13 This method is only a way of pulling new files which have existed since the
  14 last time it was checked, as there's no logging of deleted files.
  15
  16 =head1 OPTIONS
  17
  18 =over
  19
  20 =item --trackers=host1:7001,host2:7001
  21
  22 Use these MogileFS trackers to negotiate with.
  23
  24 =item --fromfid=<fid>
  25
  26 The highest numbered fid fetched the last time this utility was run.
  27
  28 =item --count=<count>
  29
  30 Numer of fids to inspect and return.
  31
  32 =back
  33
  34 =head1 AUTHOR
  35
  36 Dormando E<lt>L<dormando@rydia.net>E<gt>
  37
  38 =head1 BUGS
  39
  40 None known.
  41
  42 =head1 LICENSE
  43
  44 Licensed for use and redistribution under the same terms as Perl itself.
  45
  46 =cut
  47
  48 use strict;
  49 use warnings;
  50
  51 use MogileFS::Admin;
  52 use lib './lib';
  53 use MogileFS::Utils;
  54
  55 my $util = MogileFS::Utils->new;
  56 my $usage = "--trackers=host --fromfid=123 --count=5000";
  57 my $c = $util->getopts($usage, qw/fromfid=i count=i/);
  58
  59 my $moga = MogileFS::Admin->new(hosts => $c->{trackers});
  60
  61 my $fromfid = $c->{fromfid} || 0;
  62 my $count   = $c->{count} || 100;
  63
  64 while ($count) {
  65     # Try to fetch the max, but we will likely get less.
  66     my $fids_chunk = $moga->list_fids($fromfid, $count);
  67     if ($moga->errcode) {
  68         die "Error listing fids: ", $moga->errstr, "\n";
  69     }
  70     my @fids = sort { $a <=> $b } keys %$fids_chunk;
  71     last unless @fids;
  72     $fromfid = $fids[-1];
  73     $count  -= @fids;
  74     for my $fid (@fids) {
  75         my $file = $fids_chunk->{$fid};
  76         print "fid ", $fid, "\n";
  77         for my $key (sort keys %$file) {
  78             my $val = $file->{$key};
  79             $val = _escape_url_string($val) if $key eq 'dkey';
  80             print $key, " ", $val, "\n";
  81         }
  82         print "\n";
  83     }
  84 }
  85
  86 sub _escape_url_string {
  87     my $str = shift;
  88     $str =~ s/([^a-zA-Z0-9_\,\-.\/\\\: ])/uc sprintf("%%%02x",ord($1))/eg;
  89     $str =~ tr/ /+/;
  90     return $str;
  91 }