Tools/scripts/ftpmirror.py

   1 #! /usr/bin/env python
   2
   3 """Mirror a remote ftp subtree into a local directory tree.
   4
   5 usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
   6                  [-l username [-p passwd [-a account]]]
   7                  hostname[:port] [remotedir [localdir]]
   8 -v: verbose
   9 -q: quiet
  10 -i: interactive mode
  11 -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
  12 -n: don't log in
  13 -r: remove local files/directories no longer pertinent
  14 -l username [-p passwd [-a account]]: login info (default .netrc or anonymous)
  15 -s pat: skip files matching pattern
  16 hostname: remote host w/ optional port separated by ':'
  17 remotedir: remote directory (default initial)
  18 localdir: local directory (default current)
  19 """
  20
  21 import os
  22 import sys
  23 import time
  24 import getopt
  25 import ftplib
  26 import netrc
  27 from fnmatch import fnmatch
  28
  29 # Print usage message and exit
  30 def usage(*args):
  31     sys.stdout = sys.stderr
  32     for msg in args: print(msg)
  33     print(__doc__)
  34     sys.exit(2)
  35
  36 verbose = 1 # 0 for -q, 2 for -v
  37 interactive = 0
  38 mac = 0
  39 rmok = 0
  40 nologin = 0
  41 skippats = ['.', '..', '.mirrorinfo']
  42
  43 # Main program: parse command line and start processing
  44 def main():
  45     global verbose, interactive, mac, rmok, nologin
  46     try:
  47         opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
  48     except getopt.error as msg:
  49         usage(msg)
  50     login = ''
  51     passwd = ''
  52     account = ''
  53     if not args: usage('hostname missing')
  54     host = args[0]
  55     port = 0
  56     if ':' in host:
  57         host, port = host.split(':', 1)
  58         port = int(port)
  59     try:
  60         auth = netrc.netrc().authenticators(host)
  61         if auth is not None:
  62             login, account, passwd = auth
  63     except (netrc.NetrcParseError, IOError):
  64         pass
  65     for o, a in opts:
  66         if o == '-l': login = a
  67         if o == '-p': passwd = a
  68         if o == '-a': account = a
  69         if o == '-v': verbose = verbose + 1
  70         if o == '-q': verbose = 0
  71         if o == '-i': interactive = 1
  72         if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
  73         if o == '-n': nologin = 1
  74         if o == '-r': rmok = 1
  75         if o == '-s': skippats.append(a)
  76     remotedir = ''
  77     localdir = ''
  78     if args[1:]:
  79         remotedir = args[1]
  80         if args[2:]:
  81             localdir = args[2]
  82             if args[3:]: usage('too many arguments')
  83     #
  84     f = ftplib.FTP()
  85     if verbose: print("Connecting to '%s%s'..." % (host,
  86                                                    (port and ":%d"%port or "")))
  87     f.connect(host,port)
  88     if not nologin:
  89         if verbose:
  90             print('Logging in as %r...' % (login or 'anonymous'))
  91         f.login(login, passwd, account)
  92     if verbose: print('OK.')
  93     pwd = f.pwd()
  94     if verbose > 1: print('PWD =', repr(pwd))
  95     if remotedir:
  96         if verbose > 1: print('cwd(%s)' % repr(remotedir))
  97         f.cwd(remotedir)
  98         if verbose > 1: print('OK.')
  99         pwd = f.pwd()
 100         if verbose > 1: print('PWD =', repr(pwd))
 101     #
 102     mirrorsubdir(f, localdir)
 103
 104 # Core logic: mirror one subdirectory (recursively)
 105 def mirrorsubdir(f, localdir):
 106     pwd = f.pwd()
 107     if localdir and not os.path.isdir(localdir):
 108         if verbose: print('Creating local directory', repr(localdir))
 109         try:
 110             makedir(localdir)
 111         except os.error as msg:
 112             print("Failed to establish local directory", repr(localdir))
 113             return
 114     infofilename = os.path.join(localdir, '.mirrorinfo')
 115     try:
 116         text = open(infofilename, 'r').read()
 117     except IOError as msg:
 118         text = '{}'
 119     try:
 120         info = eval(text)
 121     except (SyntaxError, NameError):
 122         print('Bad mirror info in', repr(infofilename))
 123         info = {}
 124     subdirs = []
 125     listing = []
 126     if verbose: print('Listing remote directory %r...' % (pwd,))
 127     f.retrlines('LIST', listing.append)
 128     filesfound = []
 129     for line in listing:
 130         if verbose > 1: print('-->', repr(line))
 131         if mac:
 132             # Mac listing has just filenames;
 133             # trailing / means subdirectory
 134             filename = line.strip()
 135             mode = '-'
 136             if filename[-1:] == '/':
 137                 filename = filename[:-1]
 138                 mode = 'd'
 139             infostuff = ''
 140         else:
 141             # Parse, assuming a UNIX listing
 142             words = line.split(None, 8)
 143             if len(words) < 6:
 144                 if verbose > 1: print('Skipping short line')
 145                 continue
 146             filename = words[-1].lstrip()
 147             i = filename.find(" -> ")
 148             if i >= 0:
 149                 # words[0] had better start with 'l'...
 150                 if verbose > 1:
 151                     print('Found symbolic link %r' % (filename,))
 152                 linkto = filename[i+4:]
 153                 filename = filename[:i]
 154             infostuff = words[-5:-1]
 155             mode = words[0]
 156         skip = 0
 157         for pat in skippats:
 158             if fnmatch(filename, pat):
 159                 if verbose > 1:
 160                     print('Skip pattern', repr(pat), end=' ')
 161                     print('matches', repr(filename))
 162                 skip = 1
 163                 break
 164         if skip:
 165             continue
 166         if mode[0] == 'd':
 167             if verbose > 1:
 168                 print('Remembering subdirectory', repr(filename))
 169             subdirs.append(filename)
 170             continue
 171         filesfound.append(filename)
 172         if filename in info and info[filename] == infostuff:
 173             if verbose > 1:
 174                 print('Already have this version of',repr(filename))
 175             continue
 176         fullname = os.path.join(localdir, filename)
 177         tempname = os.path.join(localdir, '@'+filename)
 178         if interactive:
 179             doit = askabout('file', filename, pwd)
 180             if not doit:
 181                 if filename not in info:
 182                     info[filename] = 'Not retrieved'
 183                 continue
 184         try:
 185             os.unlink(tempname)
 186         except os.error:
 187             pass
 188         if mode[0] == 'l':
 189             if verbose:
 190                 print("Creating symlink %r -> %r" % (filename, linkto))
 191             try:
 192                 os.symlink(linkto, tempname)
 193             except IOError as msg:
 194                 print("Can't create %r: %s" % (tempname, msg))
 195                 continue
 196         else:
 197             try:
 198                 fp = open(tempname, 'wb')
 199             except IOError as msg:
 200                 print("Can't create %r: %s" % (tempname, msg))
 201                 continue
 202             if verbose:
 203                 print('Retrieving %r from %r as %r...' % (filename, pwd, fullname))
 204             if verbose:
 205                 fp1 = LoggingFile(fp, 1024, sys.stdout)
 206             else:
 207                 fp1 = fp
 208             t0 = time.time()
 209             try:
 210                 f.retrbinary('RETR ' + filename,
 211                              fp1.write, 8*1024)
 212             except ftplib.error_perm as msg:
 213                 print(msg)
 214             t1 = time.time()
 215             bytes = fp.tell()
 216             fp.close()
 217             if fp1 != fp:
 218                 fp1.close()
 219         try:
 220             os.unlink(fullname)
 221         except os.error:
 222             pass            # Ignore the error
 223         try:
 224             os.rename(tempname, fullname)
 225         except os.error as msg:
 226             print("Can't rename %r to %r: %s" % (tempname, fullname, msg))
 227             continue
 228         info[filename] = infostuff
 229         writedict(info, infofilename)
 230         if verbose and mode[0] != 'l':
 231             dt = t1 - t0
 232             kbytes = bytes / 1024.0
 233             print(int(round(kbytes)), end=' ')
 234             print('Kbytes in', end=' ')
 235             print(int(round(dt)), end=' ')
 236             print('seconds', end=' ')
 237             if t1 > t0:
 238                 print('(~%d Kbytes/sec)' % \
 239                           int(round(kbytes/dt),))
 240             print()
 241     #
 242     # Remove files from info that are no longer remote
 243     deletions = 0
 244     for filename in list(info.keys()):
 245         if filename not in filesfound:
 246             if verbose:
 247                 print("Removing obsolete info entry for", end=' ')
 248                 print(repr(filename), "in", repr(localdir or "."))
 249             del info[filename]
 250             deletions = deletions + 1
 251     if deletions:
 252         writedict(info, infofilename)
 253     #
 254     # Remove local files that are no longer in the remote directory
 255     try:
 256         if not localdir: names = os.listdir(os.curdir)
 257         else: names = os.listdir(localdir)
 258     except os.error:
 259         names = []
 260     for name in names:
 261         if name[0] == '.' or name in info or name in subdirs:
 262             continue
 263         skip = 0
 264         for pat in skippats:
 265             if fnmatch(name, pat):
 266                 if verbose > 1:
 267                     print('Skip pattern', repr(pat), end=' ')
 268                     print('matches', repr(name))
 269                 skip = 1
 270                 break
 271         if skip:
 272             continue
 273         fullname = os.path.join(localdir, name)
 274         if not rmok:
 275             if verbose:
 276                 print('Local file', repr(fullname), end=' ')
 277                 print('is no longer pertinent')
 278             continue
 279         if verbose: print('Removing local file/dir', repr(fullname))
 280         remove(fullname)
 281     #
 282     # Recursively mirror subdirectories
 283     for subdir in subdirs:
 284         if interactive:
 285             doit = askabout('subdirectory', subdir, pwd)
 286             if not doit: continue
 287         if verbose: print('Processing subdirectory', repr(subdir))
 288         localsubdir = os.path.join(localdir, subdir)
 289         pwd = f.pwd()
 290         if verbose > 1:
 291             print('Remote directory now:', repr(pwd))
 292             print('Remote cwd', repr(subdir))
 293         try:
 294             f.cwd(subdir)
 295         except ftplib.error_perm as msg:
 296             print("Can't chdir to", repr(subdir), ":", repr(msg))
 297         else:
 298             if verbose: print('Mirroring as', repr(localsubdir))
 299             mirrorsubdir(f, localsubdir)
 300             if verbose > 1: print('Remote cwd ..')
 301             f.cwd('..')
 302         newpwd = f.pwd()
 303         if newpwd != pwd:
 304             print('Ended up in wrong directory after cd + cd ..')
 305             print('Giving up now.')
 306             break
 307         else:
 308             if verbose > 1: print('OK.')
 309
 310 # Helper to remove a file or directory tree
 311 def remove(fullname):
 312     if os.path.isdir(fullname) and not os.path.islink(fullname):
 313         try:
 314             names = os.listdir(fullname)
 315         except os.error:
 316             names = []
 317         ok = 1
 318         for name in names:
 319             if not remove(os.path.join(fullname, name)):
 320                 ok = 0
 321         if not ok:
 322             return 0
 323         try:
 324             os.rmdir(fullname)
 325         except os.error as msg:
 326             print("Can't remove local directory %r: %s" % (fullname, msg))
 327             return 0
 328     else:
 329         try:
 330             os.unlink(fullname)
 331         except os.error as msg:
 332             print("Can't remove local file %r: %s" % (fullname, msg))
 333             return 0
 334     return 1
 335
 336 # Wrapper around a file for writing to write a hash sign every block.
 337 class LoggingFile:
 338     def __init__(self, fp, blocksize, outfp):
 339         self.fp = fp
 340         self.bytes = 0
 341         self.hashes = 0
 342         self.blocksize = blocksize
 343         self.outfp = outfp
 344     def write(self, data):
 345         self.bytes = self.bytes + len(data)
 346         hashes = int(self.bytes) / self.blocksize
 347         while hashes > self.hashes:
 348             self.outfp.write('#')
 349             self.outfp.flush()
 350             self.hashes = self.hashes + 1
 351         self.fp.write(data)
 352     def close(self):
 353         self.outfp.write('\n')
 354
 355 def raw_input(prompt):
 356     sys.stdout.write(prompt)
 357     sys.stdout.flush()
 358     return sys.stdin.readline()
 359
 360 # Ask permission to download a file.
 361 def askabout(filetype, filename, pwd):
 362     prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
 363     while 1:
 364         reply = raw_input(prompt).strip().lower()
 365         if reply in ['y', 'ye', 'yes']:
 366             return 1
 367         if reply in ['', 'n', 'no', 'nop', 'nope']:
 368             return 0
 369         print('Please answer yes or no.')
 370
 371 # Create a directory if it doesn't exist.  Recursively create the
 372 # parent directory as well if needed.
 373 def makedir(pathname):
 374     if os.path.isdir(pathname):
 375         return
 376     dirname = os.path.dirname(pathname)
 377     if dirname: makedir(dirname)
 378     os.mkdir(pathname, 0o777)
 379
 380 # Write a dictionary to a file in a way that can be read back using
 381 # rval() but is still somewhat readable (i.e. not a single long line).
 382 # Also creates a backup file.
 383 def writedict(dict, filename):
 384     dir, fname = os.path.split(filename)
 385     tempname = os.path.join(dir, '@' + fname)
 386     backup = os.path.join(dir, fname + '~')
 387     try:
 388         os.unlink(backup)
 389     except os.error:
 390         pass
 391     fp = open(tempname, 'w')
 392     fp.write('{\n')
 393     for key, value in dict.items():
 394         fp.write('%r: %r,\n' % (key, value))
 395     fp.write('}\n')
 396     fp.close()
 397     try:
 398         os.rename(filename, backup)
 399     except os.error:
 400         pass
 401     os.rename(tempname, filename)
 402
 403
 404 if __name__ == '__main__':
 405     main()