src/fetcher

   1 #!/bin/sh
   2 # fetcher:
   3 # get text contents from sources, return creation/modification time
   4 # and source description together with text through stdout
   5 # (non-zero exit code in case of error)
   6
   7 . `dirname $0`/constants.sh
   8
   9 # parse protocol and source from argument
  10 proto=${1%%://*}
  11 src="${1#*://}"
  12
  13 # genesis time, less than 0 means failure
  14 gen=-1
  15
  16 # temporary and log file
  17 tmpf=${TMP:-/tmp}/fetcher-$proto-$$.tmp
  18 logf=${TMP:-/tmp}/fetcher-log-`date +%y%U`.log
  19 #echo ::$tmpf::
  20
  21 # timeout for total script is 3 times that for fetch operation
  22 timeout=`expr $FETCHTIMEOUT '*' 3`
  23
  24 logit () { echo : "$@" >>$logf ; }
  25
  26 cat <<EOT >>$logf
  27
  28 # fetcher at `date`
  29
  30 EOT
  31
  32 case $proto in
  33 file)
  34  statstat=`stat --help 2>&1|head -n 1`
  35  case $statstat in
  36  Usage*) gen=`stat -c %Y "$src"` ;;
  37  "stat: unknown option"*) gen=`stat -f %a "$src"` ;;
  38  *) gen=`date +%s` ;;
  39  esac
  40  cat "$src" >$tmpf
  41  logit getting file: $src ...
  42  ;;
  43 http|https|ftp)
  44  logit wget ...
  45  wget -a $logf -O $tmpf -T $FETCHTIMEOUT "$proto://$src" && gen=`date +%s`
  46  ;;
  47 gopher)
  48  ;;
  49 rss)
  50  ;;
  51 *) gen=-1 ;;
  52 esac
  53
  54 echo SOURCE $proto://$src
  55 echo GENESIS $gen
  56 echo
  57 if test $gen -gt 0
  58 then
  59  cat $tmpf
  60  gen=0
  61  logit ok
  62 else
  63  gen=1
  64  logit FAILED
  65 fi
  66
  67 rm -f $tmpf
  68 exit $gen