contrib/directory-archive/fetch-all-v3

   1 #!/bin/bash
   2
   3 # Download all current v3 directory status votes and the consensus document,
   4 # then download the descriptors and extra info documents.
   5
   6 # Copyright (c) 2005, 2006, 2007, 2008 Peter Palfrader
   7 #
   8 # Permission is hereby granted, free of charge, to any person obtaining a copy
   9 # of this software and associated documentation files (the "Software"), to deal
  10 # in the Software without restriction, including without limitation the rights
  11 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 # copies of the Software, and to permit persons to whom the Software is
  13 # furnished to do so, subject to the following conditions:
  14 #
  15 # The above copyright notice and this permission notice shall be included in
  16 # all copies or substantial portions of the Software.
  17 #
  18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 # SOFTWARE.
  25
  26 TZ=UTC
  27 export TZ
  28
  29 DIRSERVERS=""
  30 DIRSERVERS="$DIRSERVERS 86.59.21.38:80"         # tor26
  31 DIRSERVERS="$DIRSERVERS 128.31.0.34:9031"       # moria1
  32 DIRSERVERS="$DIRSERVERS 216.224.124.114:9030"   # ides
  33 DIRSERVERS="$DIRSERVERS 88.198.7.215:80"        # gabelmoo
  34 #DIRSERVERS="$DIRSERVERS 140.247.60.64:80"      # lefkada
  35 DIRSERVERS="$DIRSERVERS 194.109.206.212:80"     # dizum
  36 DIRSERVERS="$DIRSERVERS 128.31.0.34:9032"       # moria2
  37 TIME=$(date "+%Y%m%d-%H%M%S")
  38
  39 . fetch-all-functions
  40
  41 consensus=""
  42 tmpdir="consensus/tmp"
  43 [ -d "$tmpdir" ] || mkdir -p "$tmpdir"
  44 for dirserver in $DIRSERVERS; do
  45         wget -q -O "$tmpdir/$TIME-consensus" http://$dirserver/tor/status-vote/current/consensus
  46         if [ "$?" != 0 ]; then
  47                 rm -f "$tmpdir/$TIME-consensus"
  48                 continue
  49         fi
  50
  51         freshconsensus="$tmpdir/$TIME-consensus"
  52
  53         timestamp=$(awk '$1=="valid-after" {printf "%s-%s", $2, $3}' < "$freshconsensus")
  54         datedir=$(awk '$1=="valid-after" {printf "%s", $2}' < "$freshconsensus" | tr '-' '/')
  55         dir="consensus/$datedir"
  56         [ -d "$dir" ] || mkdir -p "$dir"
  57
  58
  59         consensus="$dir/$timestamp-consensus.bz2"
  60         if ! [ -e "$consensus" ]; then
  61                 # the consensus is new, or at least we don't have it yet
  62                 bzip2 "$freshconsensus"
  63                 mv "$freshconsensus.bz2" "$consensus"
  64                 break
  65         fi
  66
  67         rm -f "$freshconsensus"
  68         echo "Consensus from $timestamp (gotten from $dirserver) already exists!" >&2
  69         # maybe there is a newer one on a different authority, so try again.
  70 done
  71
  72 if [ "$consensus" = "" ]; then
  73         echo "No consensus available" 2>&1
  74         exit 1
  75 fi
  76
  77
  78 votes=$(bzcat $consensus | awk '$1 == "vote-digest" {print $2}')
  79 for vote in $votes; do
  80         for dirserver in $DIRSERVERS; do
  81                 wget -q -O "$dir/$TIME-vote-$vote" http://$dirserver/tor/status-vote/current/d/$vote
  82                 if [ "$?" != 0 ]; then
  83                         rm -f "$dir/$TIME-vote-$vote"
  84                         continue
  85                 fi
  86                 break
  87         done
  88         if [ -e "$dir/$TIME-vote-$vote" ]; then
  89                 voteridentity=$(awk '$1=="fingerprint" {print $2}' < "$dir/$TIME-vote-$vote")
  90                 if [ -e "$dir/$timestamp-vote-$voteridentity-$vote.bz2" ]; then
  91                         echo "Vote $vote from $voteridentity already exists!" >&2
  92                         rm -f "$dir/$TIME-vote-$vote"
  93                         continue;
  94                 fi
  95                 mv "$dir/$TIME-vote-$vote"  "$dir/$timestamp-vote-$voteridentity-$vote"
  96                 bzip2 "$dir/$timestamp-vote-$voteridentity-$vote"
  97         else
  98                 echo "Failed to get vote $vote!" >&2
  99         fi
 100 done
 101
 102 digests=$( for i in ` bzcat $consensus | awk '$1 == "r" {printf "%s===\n", $4}' | sort -u `; do
 103                 echo $i | \
 104                 base64-decode | \
 105                 perl -e 'undef $/; $a=<>; print unpack("H\*", $a),"\n";';
 106         done )
 107 for digest in $digests; do
 108         fetch_digest "$digest" "server-descriptor"
 109 done