src/java/voldemort/tools/ExportBDBToTextDump.java

   1 /**
   2  * Copyright 2013 LinkedIn, Inc
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
   5  * use this file except in compliance with the License. You may obtain a copy of
   6  * the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13  * License for the specific language governing permissions and limitations under
  14  * the License.
  15  */
  16 package voldemort.tools;
  17
  18 import joptsimple.OptionParser;
  19 import joptsimple.OptionSet;
  20 import voldemort.server.VoldemortConfig;
  21 import voldemort.store.StorageEngine;
  22 import voldemort.store.StoreDefinition;
  23 import voldemort.store.bdb.BdbStorageConfiguration;
  24 import voldemort.utils.ByteArray;
  25 import voldemort.utils.ByteUtils;
  26 import voldemort.utils.ClosableIterator;
  27 import voldemort.utils.Pair;
  28 import voldemort.versioning.VectorClock;
  29 import voldemort.versioning.Versioned;
  30
  31 import java.io.*;
  32 import java.util.Arrays;
  33 import java.util.Properties;
  34 import java.util.concurrent.Callable;
  35
  36 public class ExportBDBToTextDump {
  37     static long SPLIT_SIZE = 1000000L;
  38     static Integer WRITER_BUFFER_SIZE = 16777216;
  39
  40     public static OptionParser getParser() {
  41         OptionParser parser = new OptionParser();
  42         parser.acceptsAll(Arrays.asList("bdb"), "Store level BDB folder")
  43                 .withRequiredArg()
  44                 .ofType(String.class)
  45                 .describedAs("input-file-or-folder");
  46         parser.acceptsAll(Arrays.asList("o", "output"), "Output folder of text dump")
  47                 .withRequiredArg()
  48                 .ofType(String.class)
  49                 .describedAs("output-folder");
  50
  51         return parser;
  52     }
  53
  54     public static void validateOptions(OptionSet options) throws IOException {
  55         Integer exitStatus = null;
  56         if(options.has("help")) {
  57             exitStatus = 0;
  58             System.out.println("This programs loads a data dump to a Voldemort Store.");
  59             System.out.println("Supported data dump should be a file or a folder containing files with lines of data.");
  60             System.out.println("Each line should be in the format of the following (all in Hex Decimal format)");
  61             System.out.println("\n\tKEY_BINARY VECTOR_CLOCK_BINARY VALUE_BINARY\n");
  62         }
  63         else if(!options.has("input")) {
  64             System.err.println("Option \"input\" is required");
  65             exitStatus = 1;
  66         }
  67         else if(!new File((String)options.valueOf("input")).isDirectory()) {
  68             System.err.println("Not a directory: " + options.valueOf("input") );
  69             exitStatus = 1;
  70         }
  71         else if(!options.has("output")) {
  72             System.err.println("Option \"output\" is required");
  73             exitStatus = 1;
  74         }
  75         else if(!new File((String)options.valueOf("output")).isDirectory()) {
  76             System.err.println("Not a directory: " + options.valueOf("output") );
  77             exitStatus = 1;
  78         }
  79         if(exitStatus != null) {
  80             if(exitStatus == 0)
  81                 getParser().printHelpOn(System.out);
  82             else
  83                 getParser().printHelpOn(System.err);
  84             System.exit(exitStatus);
  85         }
  86     }
  87     public static void main(String[] argv) throws Exception {
  88         OptionParser parser = getParser();
  89         OptionSet options = parser.parse(argv);
  90         validateOptions(options);
  91
  92         // bdb_folder output_folder
  93         String storeBdbFolderPath = (String) options.valueOf("bdb");
  94         String outputFolderPath = (String) options.valueOf("output");
  95
  96         File storeBdbFolder = new File(storeBdbFolderPath);
  97         File outputFolder = new File(outputFolderPath);
  98         final String storeName = storeBdbFolder.getName();
  99
 100         Properties properties = new Properties();
 101         properties.put("node.id","0");
 102         properties.put("voldemort.home", storeBdbFolder.getParent());
 103         VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
 104         voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
 105         voldemortConfig.setEnableJmx(false);
 106         voldemortConfig.setBdbOneEnvPerStore(true);
 107         BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
 108         class MockStoreDefinition extends StoreDefinition {
 109             public MockStoreDefinition() {
 110                 super(storeName,null,null,null,null,null,null,null,0,null,0,null,0,null,null,null,null,null,null,null,null,null,null,null,null,0);
 111             }
 112             @Override
 113             public boolean hasMemoryFootprint() {
 114                 return false;
 115             }
 116         }
 117         StoreDefinition storeDef = new MockStoreDefinition();
 118         StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(storeDef, null);
 119         long reportIntervalMs = 10000L;
 120         long lastCount = 0;
 121         Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
 122
 123         long count = 0;
 124         BufferedWriter splitFileWriter = null;
 125         ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries = engine.entries();
 126         while(entries.hasNext()) {
 127             if(splitFileWriter == null) {
 128                 long splitId = count / SPLIT_SIZE;
 129                 File splitFile = new File(outputFolder, makeSplitFileName(splitId));
 130                 splitFileWriter = new BufferedWriter(new FileWriter(splitFile), WRITER_BUFFER_SIZE);
 131             }
 132             Pair<ByteArray, Versioned<byte[]>> pair = entries.next();
 133             String line = makeLine(pair);
 134             splitFileWriter.write(line);
 135
 136             if((count + 1) % SPLIT_SIZE == 0) {
 137                 splitFileWriter.close();
 138                 splitFileWriter = null;
 139             }
 140             count++;
 141             final Long countObject = count;
 142             Boolean reported = rp.tryReport(new Callable<Boolean>() {
 143                 @Override
 144                 public Boolean call() throws Exception {
 145                     System.out.print(String.format("Exported %15d entries", countObject));
 146                     return true;
 147                 }
 148             });
 149             if(reported != null) {
 150                 System.out.println(String.format("; Speed: %8d/s", (count - lastCount)/ (reportIntervalMs / 1000)));
 151                 lastCount = count;
 152             }
 153         }
 154         entries.close();
 155         if(splitFileWriter != null) {
 156             splitFileWriter.close();
 157         }
 158         System.out.println(String.format("Finished exporting %d entries", count));
 159     }
 160
 161     static private class Reporter<T> {
 162         Long intervalMs;
 163         Long lastReport;
 164         Reporter(long intervalMs) {
 165             this.intervalMs = intervalMs;
 166         }
 167
 168         public T tryReport(Callable<T> callable) throws Exception {
 169             if(lastReport == null) {
 170                 lastReport = System.currentTimeMillis();
 171                 return null;
 172             } else {
 173                 if(lastReport + intervalMs < System.currentTimeMillis()) {
 174                     T result = callable.call();
 175                     lastReport = System.currentTimeMillis();
 176                     return result;
 177                 }
 178             }
 179             return null;
 180         }
 181     }
 182
 183     public static String makeLine(Pair<ByteArray, Versioned<byte[]>> pair) {
 184         Versioned<byte[]> versioned = pair.getSecond();
 185         byte[] keyBytes = pair.getFirst().get();
 186         byte[] versionBytes = ((VectorClock)versioned.getVersion()).toBytes();
 187         byte[] valueBytes = pair.getSecond().getValue();
 188         return String.format("%s %s %s\n", ByteUtils.toHexString(keyBytes),ByteUtils.toHexString(versionBytes),ByteUtils.toHexString(valueBytes));
 189     }
 190
 191     public static String makeSplitFileName(long splitId) {
 192         return String.format("x%010d.split", splitId);
 193     }
 194 }