From 436f57af99152d62c8c56fda394cbb2159747fe9 Mon Sep 17 00:00:00 2001
From: Eric Wong <normalperson@yhbt.net>
Date: Mon, 8 Oct 2012 22:44:03 +0000
Subject: [PATCH] client: add each_file_info iterator

This allows fast listing of keys and metadata (length,
checksum, devcount, class).
---
 lib/mogilefs/mogilefs.rb                    | 51 +++++++++++++++++++++++++++++
 test/test_mogilefs_integration_list_keys.rb | 16 +++++++++
 2 files changed, 67 insertions(+)

diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb
index eccca78..426217e 100644
--- a/lib/mogilefs/mogilefs.rb
+++ b/lib/mogilefs/mogilefs.rb
@@ -88,6 +88,57 @@ class MogileFS::MogileFS < MogileFS::Client
     nil
   end
 
+  # Enumerates keys and yields a +file_info+ hash for each key matched by
+  # +prefix+
+  def each_file_info(prefix = "", args = nil)
+    # FIXME: there's a lot of duplicate code from list_keys_verbose here...
+    raise ArgumentError, "need block" unless block_given?
+    ordered = ready = nil
+    on_file_info = lambda do |info|
+      Hash === info or raise info
+      file_info_cleanup(info)
+
+      # deal with trackers with multiple queryworkers responding out-of-order
+      ready[info["key"]] = info
+      while info = ready.delete(ordered[-1])
+        ordered.pop
+        yield info
+      end
+    end
+
+    nr = 0
+    opts = { :domain => @domain }
+    opts[:devices] = 1 if args && args[:devices]
+    after = args ? args[:after] : nil
+    limit = args ? args[:limit] : nil
+
+    begin
+      keys, after = list_keys(prefix, after, limit || 1000)
+      return nr unless keys && keys[0]
+      ordered = keys.reverse
+      ready = {}
+      nr += keys.size
+      limit -= keys.size if limit
+
+      keys.each do |key|
+        opts[:key] = key
+        @backend.pipeline_dispatch(:file_info, opts, &on_file_info)
+      end
+      @backend.pipeline_wait
+    rescue MogileFS::PipelineError, SystemCallError,
+           MogileFS::RequestTruncatedError,
+           MogileFS::UnreadableSocketError,
+           MogileFS::InvalidResponseError, # truncated response
+           MogileFS::Timeout
+      @backend.shutdown
+      keys = ordered - ready.keys
+      retry
+    end while limit == nil || limit > 0
+  rescue
+    @backend.shutdown
+    raise
+  end
+
   # Retrieves the contents of +key+.  If +dst+ is specified, +dst+
   # should be an IO-like object capable of receiving the +write+ method
   # or a path name.  +copy_length+ may be specified to limit the number of
diff --git a/test/test_mogilefs_integration_list_keys.rb b/test/test_mogilefs_integration_list_keys.rb
index 6c62e6b..1de6cda 100644
--- a/test/test_mogilefs_integration_list_keys.rb
+++ b/test/test_mogilefs_integration_list_keys.rb
@@ -36,5 +36,21 @@ class TestMogileFSIntegrationListKeys < TestMogIntegration
       assert_equal "ek_#{n.to_s}", key
       n += 1
     end
+    assert_equal 9, n
+  end
+
+  def test_each_file_info
+    9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
+    n = 0
+    @client.each_file_info do |info|
+      assert_equal @client.domain, info["domain"]
+      assert_equal n.to_s.size, info["length"]
+      assert_kind_of Integer, info["fid"]
+      assert_kind_of Integer, info["devcount"]
+      assert_equal "default", info["class"]
+      assert_equal "ek_#{n}", info["key"]
+      n += 1
+    end
+    assert_equal 9, n
   end
 end
-- 
2.11.4.GIT