dput/dput.py

   1 #! /usr/bin/python2
   2 # -*- coding: utf-8; -*-
   3 #
   4 # dput/dput.py
   5 # Part of ‘dput’, a Debian package upload toolkit.
   6 #
   7 # Copyright © 2015 Ben Finney <ben+debian@benfinney.id.au>
   8 # Copyright © 2008–2013 Y Giridhar Appaji Nag <appaji@debian.org>
   9 # Copyright © 2006–2008 Thomas Viehmann <tv@beamnet.de>
  10 # Copyright © 2000–2005 Christian Kurz <shorty@debian.org>
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16 #
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU General Public License along
  23 # with this program; if not, write to the Free Software Foundation, Inc.,
  24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  25
  26 """ dput — Debian package upload tool. """
  27
  28 import os
  29 import os.path
  30 import sys
  31 import string
  32 import textwrap
  33 import re
  34 import signal
  35 import subprocess
  36 import pwd
  37 import stat
  38 import email.parser
  39 from hashlib import md5, sha1
  40 import importlib
  41 import pkgutil
  42
  43 # Now import our modules
  44 import ConfigParser
  45
  46 app_library_path = os.path.dirname(__file__)
  47
  48 from .helper import dputhelper
  49
  50
  51 debug = 0
  52
  53
  54 def import_upload_functions():
  55     """ Import upload method modules and make them available. """
  56     upload_methods = {}
  57
  58     package_name = "methods"
  59     modules_path = os.path.join(app_library_path, package_name)
  60     modules_found = [
  61             name for (__, name, ispkg) in
  62             pkgutil.iter_modules([modules_path])
  63             if not ispkg]
  64     if debug:
  65         sys.stdout.write("D: modules_found: %r\n" % modules_found)
  66     for module_name in modules_found:
  67         module = importlib.import_module("{package}.{module}".format(
  68                 package=".".join(["dput", package_name]),
  69                 module=module_name))
  70         if debug:
  71             sys.stdout.write("D: Module: %s (%r)\n" % (module_name, module))
  72         method_name = module_name
  73         if debug:
  74             sys.stdout.write("D: Method name: %s\n" % method_name)
  75
  76         upload_methods[method_name] = module.upload
  77
  78     return upload_methods
  79
  80
  81 def parse_changes(chg_fd):
  82     """ Parse the changes file. """
  83     check = chg_fd.read(5)
  84     if check != '-----':
  85         chg_fd.seek(0)
  86     else:
  87         # found a PGP header, gonna ditch the next 3 lines
  88         # eat the rest of the line
  89         chg_fd.readline()
  90         # Hash: SHA1
  91         chg_fd.readline()
  92         # empty line
  93         chg_fd.readline()
  94     if not chg_fd.readline().find('Format') != -1:
  95         chg_fd.readline()
  96     changes_text = chg_fd.read()
  97     changes = email.parser.HeaderParser().parsestr(changes_text)
  98     if 'files' not in changes:
  99         raise KeyError("No Files field in upload control file")
 100     for a in changes['files'].strip().split('\n'):
 101         if len(a.split()) != 5:
 102             sys.stderr.write("Invalid Files line in .changes:\n  %s\n" % a)
 103             sys.exit(1)
 104     return changes
 105
 106
 107 def read_configs(extra_config, debug):
 108     """ Read configuration settings from config files.
 109
 110         :param extra_config: Filesystem path of config file to read.
 111         :param debug: If true, enable debugging output.
 112         :return: The resulting `ConfigParser` instance.
 113
 114         Read config files in this order:
 115         * If specified on the command line, only read `extra_config`.
 116         * Otherwise, read ‘/etc/dput.cf’ then ‘~/.dput.cf’.
 117         The config parser will layer values.
 118
 119         """
 120     config = ConfigParser.ConfigParser()
 121
 122     config.set('DEFAULT', 'login', 'username')
 123     config.set('DEFAULT', 'method', 'scp')
 124     config.set('DEFAULT', 'hash', 'md5')
 125     config.set('DEFAULT', 'allow_unsigned_uploads', '0')
 126     config.set('DEFAULT', 'allow_dcut', '0')
 127     config.set('DEFAULT', 'distributions', '')
 128     config.set('DEFAULT', 'allowed_distributions', '')
 129     config.set('DEFAULT', 'run_lintian', '0')
 130     config.set('DEFAULT', 'run_dinstall', '0')
 131     config.set('DEFAULT', 'check_version', '0')
 132     config.set('DEFAULT', 'scp_compress', '0')
 133     config.set('DEFAULT', 'default_host_main', '')
 134     config.set('DEFAULT', 'post_upload_command', '')
 135     config.set('DEFAULT', 'pre_upload_command', '')
 136     config.set('DEFAULT', 'ssh_config_options', '')
 137     config.set('DEFAULT', 'passive_ftp', '1')
 138     config.set('DEFAULT', 'progress_indicator', '0')
 139     config.set('DEFAULT', 'delayed', '')
 140
 141     if extra_config:
 142         config_files = (extra_config,)
 143     else:
 144         config_files = ('/etc/dput.cf', os.path.expanduser("~/.dput.cf"))
 145     fd = None
 146     for config_file in config_files:
 147         try:
 148             fd = open(config_file)
 149         except IOError as e:
 150             if debug:
 151                 sys.stderr.write(
 152                         "%s: %s, skipping\n" % (e.strerror, config_file))
 153             continue
 154         if debug:
 155             sys.stdout.write(
 156                     "D: Parsing Configuration File %s\n" % config_file)
 157         try:
 158             config.readfp(fd)
 159         except ConfigParser.ParsingError as e:
 160             sys.stderr.write("Error parsing config file:\n%s\n" % str(e))
 161             sys.exit(1)
 162         fd.close()
 163     if fd is None:
 164         sys.stderr.write(
 165                 "Error: Could not open any configfile, tried %s\n"
 166                 % (', '.join(config_files)))
 167         sys.exit(1)
 168     # only check for fqdn and incoming dir, rest have reasonable defaults
 169     error = 0
 170     for section in config.sections():
 171         if config.get(section, 'method') == 'local':
 172             config.set(section, 'fqdn', 'localhost')
 173         if (
 174                 not config.has_option(section, 'fqdn') and
 175                 config.get(section, 'method') != 'local'):
 176             sys.stderr.write(
 177                     "Config error: %s must have a fqdn set\n" % section)
 178             error = 1
 179         if not config.has_option(section, 'incoming'):
 180             sys.stderr.write(
 181                     "Config error: %s must have an incoming directory set\n"
 182                     % section)
 183             error = 1
 184     if error:
 185         sys.exit(1)
 186
 187     return config
 188
 189
 190 hexStr = string.hexdigits
 191
 192
 193 def hexify_string(string):
 194     """ Convert a string of bytes to hexadecimal text representation. """
 195     char = ''
 196     ord_func = ord if isinstance(string, str) else int
 197     for c in string:
 198         char += hexStr[(ord_func(c) >> 4) & 0xF] + hexStr[ord_func(c) & 0xF]
 199     return char
 200
 201
 202 def checksum_test(filename, hash):
 203     """ Generate a checksum for a file.
 204
 205         Currently supports md5, sha1. ripemd may come in the future.
 206
 207         """
 208     try:
 209         file_to_test = open(filename, 'rb')
 210     except IOError:
 211         sys.stdout.write("Can't open %s\n" % filename)
 212         sys.exit(1)
 213
 214     if hash == 'md5':
 215         hash_type = md5
 216     else:
 217         hash_type = sha1
 218
 219     check_obj = hash_type()
 220
 221     while 1:
 222         data = file_to_test.read(65536)
 223         if len(data) == 0:
 224             break
 225         check_obj.update(data)
 226
 227     file_to_test.close()
 228     checksum = hexify_string(check_obj.digest())
 229
 230     return checksum
 231
 232
 233 def check_signature(filename):
 234     """ Verify the GnuPG signature on a file. """
 235     gnupg_path = "/usr/bin/gpg"
 236     if os.access(filename, os.R_OK):
 237         if os.access(gnupg_path, os.X_OK):
 238             gnupg_argv = [
 239                     gnupg_path,
 240                     "--status-fd", "1",
 241                     "--verify", "--batch", filename]
 242             gnupg_subprocess = subprocess.Popen(
 243                     gnupg_argv, stdout=subprocess.PIPE)
 244             gnupg_output = gnupg_subprocess.stdout.read()
 245             if gnupg_output.count('[GNUPG:] GOODSIG'):
 246                 sys.stdout.write("Good signature on %s.\n" % filename)
 247             elif gnupg_output.count('[GNUPG:] BADSIG'):
 248                 sys.stdout.write("Bad signature on %s.\n" % filename)
 249                 sys.exit(1)
 250             elif gnupg_output.count('[GNUPG:] ERRSIG'):
 251                 sys.stdout.write(
 252                         "Error verifying signature on %s.\n" % filename)
 253                 sys.exit(1)
 254             elif gnupg_output.count('[GNUPG:] NODATA'):
 255                 sys.stdout.write("No signature on %s.\n" % filename)
 256                 sys.exit(1)
 257             else:
 258                 sys.stdout.write(
 259                         "Error in finding signature verification status.\n")
 260         else:
 261             sys.stdout.write(
 262                     "Can't verify signature on %s without GnuPG\n" % filename)
 263             sys.stdout.write(
 264                     "If you are still using PGP, please read this:\n"
 265                     "http://www.gnupg.org/gph/en/pgp2x.html\n")
 266             sys.exit(1)
 267     else:
 268         sys.stdout.write("Can't read %s\n" % filename)
 269         sys.exit(1)
 270
 271
 272 def check_upload_variant(changes, debug):
 273     """ Check if this is a binary_upload only or not. """
 274     binary_upload = 0
 275     if 'architecture' in changes:
 276         arch = changes['architecture']
 277         if debug:
 278             sys.stdout.write("D: Architecture: %s\n" % arch)
 279         if arch.find('source') < 0:
 280             if debug:
 281                 sys.stdout.write("D: Doing a binary upload only.\n")
 282             binary_upload = 1
 283     return binary_upload
 284
 285
 286 def verify_signature(
 287         host, changes_file, dsc_file,
 288         config, check_only, unsigned_upload, binary_upload, debug):
 289     """ Check the signature on the two files given via function call.
 290
 291         :param host: Configuration host name.
 292         :param changes_file: Filesystem path of upload control file.
 293         :param dsc_file: Filesystem path of source control file.
 294         :param config: `ConfigParser` instance for this application.
 295         :param check_only: If true, no upload is requested.
 296         :param unsigned_upload: If true, allow an unsigned upload.
 297         :param binary_upload: If true, this upload excludes source.
 298         :param debug: If true, enable debugging output.
 299         :return: ``None``.
 300
 301         """
 302     if debug:
 303         sys.stdout.write("D: .changes-File: %s\n" % changes_file)
 304         sys.stdout.write("D: .dsc-File: %s\n" % dsc_file)
 305     if ((check_only or config.getboolean(host, 'allow_unsigned_uploads') == 0)
 306             and not unsigned_upload):
 307         sys.stdout.write("Checking signature on .changes\n")
 308         check_signature(changes_file)
 309         if not binary_upload:
 310             sys.stdout.write("Checking signature on .dsc\n")
 311             check_signature(dsc_file)
 312
 313
 314 def source_check(changes, debug):
 315     """ Check if a source tarball has to be included in the package or not. """
 316     include_orig = include_tar = 0
 317     if 'version' in changes:
 318         version = changes['version']
 319         if debug:
 320             sys.stdout.write("D: Package Version: %s\n" % version)
 321         # versions with a dash in them are for non-native only
 322         if version.find('-') == -1:
 323             # debian native
 324             include_tar = 1
 325         else:
 326             if version.find(':') > 0:
 327                 if debug:
 328                     sys.stdout.write("D: Epoch found\n")
 329                 epoch, version = version.split(':', 1)
 330             pos = version.rfind('-')
 331             upstream_version = version[0:pos]
 332             debian_version = version[pos + 1:]
 333             if debug:
 334                 sys.stdout.write(
 335                         "D: Upstream Version: %s\n" % upstream_version)
 336                 sys.stdout.write("D: Debian Version: %s\n" % debian_version)
 337             if (
 338                     debian_version == '0.1' or debian_version == '1'
 339                     or debian_version == '1.1'):
 340                 include_orig = 1
 341             else:
 342                 include_tar = 1
 343     return (include_orig, include_tar)
 344
 345
 346 def verify_files(
 347         path, filename, host,
 348         config, check_only, check_version, unsigned_upload, debug):
 349     """ Run some tests on the files to verify that they are in good shape.
 350
 351         :param path: Directory path of the upload control file.
 352         :param filename: Filename of the upload control file.
 353         :param host: Configuration host name.
 354         :param config: `ConfigParser` instance for this application.
 355         :param check_only: If true, no upload is requested.
 356         :param check_version: If true, check the package version
 357             before upload.
 358         :param unsigned_upload: If true, allow an unsigned upload.
 359         :param debug: If true, enable debugging output.
 360         :return: A collection of filesystem paths of all files to upload.
 361
 362         """
 363     file_seen = include_orig_tar_gz = include_tar_gz = binary_only = 0
 364     files_to_upload = []
 365
 366     name_of_file = filename
 367
 368     change_file = os.path.join(path, name_of_file)
 369
 370     if debug:
 371         sys.stdout.write(
 372                 "D: Validating contents of changes file %s\n" % change_file)
 373     try:
 374         chg_fd = open(change_file, 'r')
 375     except IOError:
 376         sys.stdout.write("Can't open %s\n" % change_file)
 377         sys.exit(1)
 378     changes = parse_changes(chg_fd)
 379     chg_fd.close
 380
 381     # Find out if it's a binary only upload or not
 382     binary_upload = check_upload_variant(changes, debug)
 383
 384     if binary_upload:
 385         dsc_file = ''
 386     else:
 387         dsc_file = None
 388         for file in changes['files'].strip().split('\n'):
 389             # filename only
 390             filename = file.split()[4]
 391             if filename.find('.dsc') != -1:
 392                 if debug:
 393                     sys.stdout.write("D: dsc-File: %s\n" % filename)
 394                 dsc_file = os.path.join(path, filename)
 395         if not dsc_file:
 396             sys.stderr.write("Error: no dsc file found in sourceful upload\n")
 397             sys.exit(1)
 398
 399     # Run the check to verify that the package has been tested.
 400     try:
 401         if config.getboolean(host, 'check_version') == 1 or check_version:
 402             version_check(path, changes, debug)
 403     except ConfigParser.NoSectionError as e:
 404         sys.stderr.write("Error in config file:\n%s\n" % str(e))
 405         sys.exit(1)
 406
 407     # Verify the signature of the maintainer
 408     verify_signature(
 409             host, change_file, dsc_file,
 410             config, check_only, unsigned_upload, binary_upload, debug)
 411
 412     # Check the sources
 413     (include_orig_tar_gz, include_tar_gz) = source_check(changes, debug)
 414
 415     # Check md5sum and the size
 416     file_list = changes['files'].strip().split('\n')
 417     hash_to_use = config.get('DEFAULT', 'hash')
 418     for line in file_list:
 419         (check_sum, size, section, priority, file) = line.split()
 420         file_to_upload = os.path.join(path, file)
 421         if debug:
 422             sys.stdout.write("D: File to upload: %s\n" % file_to_upload)
 423         if checksum_test(file_to_upload, hash_to_use) != check_sum:
 424             if debug:
 425                 sys.stdout.write(
 426                         "D: Checksum from .changes: %s\n" % check_sum)
 427                 sys.stdout.write(
 428                         "D: Generated Checksum: %s\n" %
 429                         checksum_test(file_to_upload, hash_to_use))
 430             sys.stdout.write(
 431                     "Checksum doesn't match for %s\n" % file_to_upload)
 432             sys.exit(1)
 433         else:
 434             if debug:
 435                 sys.stdout.write(
 436                         "D: Checksum for %s is fine\n" % file_to_upload)
 437         if os.stat(file_to_upload)[stat.ST_SIZE] != int(size):
 438             if debug:
 439                 sys.stdout.write("D: size from .changes: %s\n" % size)
 440                 sys.stdout.write(
 441                         "D: calculated size: %s\n"
 442                         % os.stat(file_to_upload)[stat.ST_SIZE])
 443             sys.stdout.write(
 444                     "size doesn't match for %s\n" % file_to_upload)
 445
 446         files_to_upload.append(file_to_upload)
 447
 448     # Check filenames
 449     for file in files_to_upload:
 450         if file[-12:] == '.orig.tar.gz' and not include_orig_tar_gz:
 451             if debug:
 452                 sys.stdout.write("D: Filename: %s\n" % file)
 453                 sys.stdout.write("D: Suffix: %s\n\n" % file[-12:])
 454             sys.stdout.write(
 455                     "Package includes an .orig.tar.gz file although"
 456                     " the debian revision suggests\n"
 457                     "that it might not be required."
 458                     " Multiple uploads of the .orig.tar.gz may be\n"
 459                     "rejected by the upload queue management software.\n")
 460         elif (
 461                 file[-7:] == '.tar.gz' and not include_tar_gz
 462                 and not include_orig_tar_gz):
 463             if debug:
 464                 sys.stdout.write("D: Filename: %s\n" % file)
 465                 sys.stdout.write("D: Suffix: %s\n" % file[-7:])
 466             sys.stdout.write(
 467                     "Package includes a .tar.gz file although"
 468                     " the version suggests that it might\n"
 469                     "not be required."
 470                     " Multiple uploads of the .tar.gz may be rejected by the\n"
 471                     "upload queue management software.\n")
 472
 473     distribution = changes.get('distribution')
 474     allowed_distributions = config.get(host, 'allowed_distributions')
 475     if distribution and allowed_distributions:
 476         if debug:
 477             sys.stdout.write(
 478                     "D: Checking: distribution %s matches %s\n"
 479                     % (distribution, allowed_distributions))
 480         if not re.match(allowed_distributions, distribution):
 481             raise dputhelper.DputUploadFatalException(
 482                     "Error: uploading files for distribution %s to %s"
 483                     " not allowed."
 484                     % (distribution, host))
 485
 486     if debug:
 487         sys.stdout.write("D: File to upload: %s\n" % change_file)
 488     files_to_upload.append(change_file)
 489
 490     return files_to_upload
 491
 492
 493 def print_config(config, debug):
 494     """ Print the configuration and exit. """
 495     sys.stdout.write("\n")
 496     config.write(sys.stdout)
 497     sys.stdout.write("\n")
 498
 499
 500 def create_upload_file(package, host, fqdn, path, files_to_upload, debug):
 501     """ Write the log file for the upload.
 502
 503         :param package: File name of package to upload.
 504         :param host: Configuration host name.
 505         :param fqdn: Fully-qualified domain name of the remote host.
 506         :param path: Filesystem path of the upload control file.
 507         :param debug: If true, enable debugging output.
 508         :return: ``None``.
 509
 510         The upload log file is named ‘basename.hostname.upload’, where
 511         “basename” is the package file name without suffix, and
 512         “hostname” is the name of the host as specified in the
 513         configuration file.
 514
 515         For example, uploading ‘foo_1.2.3-1_xyz.deb’ to host ‘bar’
 516         will be logged to ‘foo_1.2.3-1_xyz.bar.upload’.
 517
 518         The upload log file is written to the
 519         directory containing the upload control file.
 520
 521         """
 522     # only need first part
 523     base = os.path.splitext(package)[0]
 524     logfile_name = os.path.join(path, base + '.' + host + '.upload')
 525     if debug:
 526         sys.stdout.write("D: Writing logfile: %s\n" % logfile_name)
 527     try:
 528         if os.access(logfile_name, os.R_OK):
 529             logfile_fd = open(logfile_name, 'a')
 530         else:
 531             logfile_fd = open(logfile_name, 'w')
 532     except IOError:
 533         sys.stderr.write("Could not write %s\n" % logfile_name)
 534         sys.exit(1)
 535
 536     for file in files_to_upload:
 537         entry_for_logfile = (
 538                 'Successfully uploaded ' + os.path.basename(file) +
 539                 ' to ' + fqdn + ' for ' + host + '.\n')
 540         logfile_fd.write(entry_for_logfile)
 541     logfile_fd.close()
 542
 543
 544 def run_lintian_test(changes_file):
 545     """ Run lintian on the changes file and stop if it finds errors. """
 546
 547     if os.access(changes_file, os.R_OK):
 548         if os.access("/usr/bin/lintian", os.R_OK):
 549             old_signal = signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 550             sys.stdout.write("Package is now being checked with lintian.\n")
 551             if dputhelper.check_call(
 552                     ['lintian', '-i', changes_file]
 553                     ) != dputhelper.EXIT_STATUS_SUCCESS:
 554                 sys.stdout.write(
 555                         "\n"
 556                         "Lintian says this package is not compliant"
 557                         " with the current policy.\n"
 558                         "Please check the current policy and your package.\n"
 559                         "Also see lintian documentation about overrides.\n")
 560                 sys.exit(1)
 561             else:
 562                 signal.signal(signal.SIGPIPE, old_signal)
 563                 return 0
 564         else:
 565             sys.stdout.write(
 566                     "lintian is not installed, skipping package test.\n")
 567     else:
 568         sys.stdout.write("Can't read %s\n" % changes_file)
 569         sys.exit(1)
 570
 571
 572 def guess_upload_host(path, filename, config):
 573     """ Guess the host where the package should be uploaded to.
 574
 575         :param path: Directory path of the upload control file.
 576         :param filename: Filename of the upload control file.
 577         :param config: `ConfigParser` instance for this application.
 578         :return: The hostname determined for this upload.
 579
 580         This is based on information from the upload control
 581         (‘*.changes’) file.
 582
 583         """
 584     non_us = 0
 585     distribution = ""
 586     dist_re = re.compile(r'^Distribution: (.*)')
 587
 588     name_of_file = filename
 589     changes_file = os.path.join(path, name_of_file)
 590
 591     try:
 592         changes_file_fd = open(changes_file, 'r')
 593     except IOError:
 594         sys.stdout.write("Can't open %s\n" % changes_file)
 595         sys.exit(1)
 596     lines = changes_file_fd.readlines()
 597     for line in lines:
 598         match = dist_re.search(line)
 599         if match:
 600             distribution = match.group(1)
 601
 602     # Try to guess a host based on the Distribution: field
 603     if distribution:
 604         for section in config.sections():
 605             host_dists = config.get(section, 'distributions')
 606             if not host_dists:
 607                 continue
 608             for host_dist in host_dists.split(','):
 609                 if distribution == host_dist.strip():
 610                     if debug:
 611                         sys.stdout.write(
 612                                 "D: guessing host %s"
 613                                 " based on distribution %s\n"
 614                                 % (section, host_dist))
 615                     return section
 616
 617     if len(config.get('DEFAULT', 'default_host_main')) != 0:
 618         sys.stdout.write(
 619                 "Trying to upload package to %s\n"
 620                 % config.get('DEFAULT', 'default_host_main'))
 621         return config.get('DEFAULT', 'default_host_main')
 622     else:
 623         sys.stdout.write(
 624                 "Trying to upload package to ftp-master"
 625                 " (ftp.upload.debian.org)\n")
 626         return "ftp-master"
 627
 628
 629 def dinstall_caller(filename, host, fqdn, login, incoming, debug):
 630     """ Run ‘dinstall’ for the package on the remote host.
 631
 632         :param filename: Debian package filename to install.
 633         :param host: Configuration host name.
 634         :param fqdn: Fully-qualified domain name of the remote host.
 635         :param login: Username for login to the remote host.
 636         :param incoming: Filesystem path on remote host for incoming
 637             packages.
 638         :param debug: If true, enable debugging output.
 639         :return: ``None``.
 640
 641         Run ‘dinstall’ on the remote host in test mode, and present
 642         the output to the user.
 643
 644         This is so the user can see if the package would be installed
 645         or not.
 646
 647         """
 648     command = [
 649             'ssh', '%s@%s' % (login, fqdn),
 650             'cd', '%s' % incoming,
 651             ';', 'dinstall', '-n', '%s' % filename]
 652     if debug:
 653         sys.stdout.write(
 654                 "D: Logging into %s@%s:%s\n" % (login, host, incoming))
 655         sys.stdout.write("D: dinstall -n %s\n" % filename)
 656     if dputhelper.check_call(command) != dputhelper.EXIT_STATUS_SUCCESS:
 657         sys.stdout.write(
 658                 "Error occured while trying to connect, or while"
 659                 " attempting to run dinstall.\n")
 660         sys.exit(1)
 661
 662
 663 def version_check(path, changes, debug):
 664     """ Check if the caller has installed the package also on his system.
 665
 666         This is for testing purposes before uploading it. If not, we
 667         reject the upload.
 668
 669         """
 670     files_to_check = []
 671
 672     # Get arch
 673     dpkg_proc = subprocess.Popen(
 674             'dpkg --print-architecture',
 675             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 676             shell=True, close_fds=True)
 677     (dpkg_stdout, dpkg_stderr) = (dpkg_proc.stdout, dpkg_proc.stderr)
 678     dpkg_architecture = dpkg_stdout.read().strip()
 679     dpkg_stdout.close()
 680     dpkg_stderr_output = dpkg_stderr.read()
 681     dpkg_stderr.close()
 682     if debug and dpkg_stderr_output:
 683         sys.stdout.write(
 684                 "D: dpkg-architecture stderr output:"
 685                 " %r\n" % dpkg_stderr_output)
 686     if debug:
 687         sys.stdout.write(
 688                 "D: detected architecture: '%s'\n" % dpkg_architecture)
 689
 690     # Get filenames of deb files:
 691     for file in changes['files'].strip().split('\n'):
 692         filename = os.path.join(path, file.split()[4])
 693         if filename.endswith('.deb'):
 694             if debug:
 695                 sys.stdout.write("D: Debian Package: %s\n" % filename)
 696             dpkg_proc = subprocess.Popen(
 697                     'dpkg --field %s' % filename,
 698                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 699                     shell=True, close_fds=True)
 700             (dpkg_stdout, dpkg_stderr) = (dpkg_proc.stdout, dpkg_proc.stderr)
 701             dpkg_output = dpkg_stdout.read()
 702             dpkg_stdout.close()
 703             dpkg_fields = email.parser.HeaderParser().parsestr(dpkg_output)
 704             dpkg_stderr_output = dpkg_stderr.read()
 705             dpkg_stderr.close()
 706             if debug and dpkg_stderr_output:
 707                 sys.stdout.write(
 708                         "D: dpkg stderr output:"
 709                         " %r\n" % dpkg_stderr_output)
 710             if (
 711                     dpkg_architecture
 712                     and dpkg_fields['architecture'] not in [
 713                         'all', dpkg_architecture]):
 714                 if debug:
 715                     sys.stdout.write(
 716                             "D: not install-checking %s due to arch mismatch\n"
 717                             % filename)
 718             else:
 719                 package_name = dpkg_fields['package']
 720                 version_number = dpkg_fields['version']
 721                 if debug:
 722                     sys.stdout.write(
 723                             "D: Package to Check: %s\n" % package_name)
 724                 if debug:
 725                     sys.stdout.write(
 726                             "D: Version to Check: %s\n" % version_number)
 727                 files_to_check.append((package_name, version_number))
 728
 729     for file, version_to_check in files_to_check:
 730         if debug:
 731             sys.stdout.write("D: Name of Package: %s\n" % file)
 732         dpkg_proc = subprocess.Popen(
 733                 'dpkg -s %s' % file,
 734                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 735                 shell=True, close_fds=True)
 736         (dpkg_stdout, dpkg_stderr) = (dpkg_proc.stdout, dpkg_proc.stderr)
 737         dpkg_output = dpkg_stdout.read()
 738         dpkg_stdout.close()
 739         dpkg_fields = email.parser.HeaderParser().parsestr(dpkg_output)
 740         dpkg_stderr_output = dpkg_stderr.read()
 741         dpkg_stderr.close()
 742         if debug and dpkg_stderr_output:
 743             sys.stdout.write(
 744                     "D: dpkg stderr output:"
 745                     " %r\n" % dpkg_stderr_output)
 746         if 'version' in dpkg_fields:
 747             installed_version = dpkg_fields['version']
 748             if debug:
 749                 sys.stdout.write(
 750                         "D: Installed-Version: %s\n" % installed_version)
 751             if debug:
 752                 sys.stdout.write(
 753                         "D: Check-Version: %s\n" % version_to_check)
 754             if installed_version != version_to_check:
 755                 sys.stdout.write(
 756                         "Package to upload is not installed, but it appears"
 757                         " you have an older version installed.\n")
 758         else:
 759             sys.stdout.write(
 760                     "Uninstalled Package. Test it before uploading it.\n")
 761             sys.exit(1)
 762
 763
 764 def execute_command(command, type, debug=False):
 765     """ Run a command that the user-defined in the config_file.
 766
 767         :param command: Command line to execute.
 768         :param type: String specifying which type of command: 'pre' or 'post'.
 769         :param debug: If true, enable debugging output.
 770         :return: ``None``.
 771
 772         """
 773     if debug:
 774         sys.stdout.write("D: Command: %s\n" % command)
 775     if subprocess.call(command, shell=True):
 776         raise dputhelper.DputUploadFatalException(
 777                 "Error: %s upload command failed." % type)
 778
 779
 780 def check_upload_logfile(
 781         changes_file, host, fqdn,
 782         check_only, call_lintian, force_upload, debug):
 783     """ Check if the user already put this package on the specified host.
 784
 785         :param changes_file: Filesystem path of upload control file.
 786         :param host: Configuration host name.
 787         :param fqdn: Fully-qualified domain name of the remote host.
 788         :param check_only: If true, no upload is requested.
 789         :param call_lintian: If true, a Lintian invocation is requested.
 790         :param force_upload: If true, don't check the upload log file.
 791         :param debug: If true, enable debugging output.
 792         :return: ``None``.
 793
 794         """
 795     uploaded = 0
 796     upload_logfile = changes_file[:-8] + '.' + host + '.upload'
 797     if not check_only and not force_upload:
 798         if not os.path.exists(upload_logfile):
 799             return
 800         try:
 801             fd_logfile = open(upload_logfile)
 802         except IOError:
 803             sys.stdout.write("Couldn't open %s\n" % upload_logfile)
 804             sys.exit(1)
 805         for line in fd_logfile.readlines():
 806             if line.find(fqdn) != -1:
 807                 uploaded = 1
 808         if uploaded:
 809             sys.stdout.write(
 810                     "Package has already been uploaded to %s on %s\n"
 811                     % (host, fqdn))
 812             sys.stdout.write("Nothing more to do for %s\n" % changes_file)
 813             sys.exit(0)
 814
 815
 816 def make_usage_message():
 817     """ Make the program usage help message. """
 818     text = textwrap.dedent("""\
 819         Usage: dput [options] [host] <package(s).changes>
 820          Supported options (see man page for long forms):
 821            -c: Config file to parse.
 822            -d: Enable debug messages.
 823            -D: Run dinstall after upload.
 824            -e: Upload to a delayed queue. Takes an argument from 0 to 15.
 825            -f: Force an upload.
 826            -h: Display this help message.
 827            -H: Display a list of hosts from the config file.
 828            -l: Run lintian before upload.
 829            -U: Do not write a .upload file after uploading.
 830            -o: Only check the package.
 831            -p: Print the configuration.
 832            -P: Use passive mode for ftp uploads.
 833            -s: Simulate the upload only.
 834            -u: Don't check GnuPG signature.
 835            -v: Display version information.
 836            -V: Check the package version and then upload it.
 837         """)
 838     return text
 839
 840
 841 def main():
 842     """ Main function, no further comment needed. :) """
 843
 844     global debug
 845
 846     check_version = config_print = force_upload = 0
 847     call_lintian = no_upload_log = config_host_list = 0
 848     ftp_passive_mode = 0
 849     preferred_host = ''
 850     config_file = ''
 851     dinstall = False
 852     check_only = False
 853     unsigned_upload = False
 854     delay_upload = None
 855     simulate = False
 856
 857     progname = dputhelper.get_progname()
 858     version = dputhelper.get_distribution_version()
 859
 860     # Parse Command Line Options.
 861     (opts, args) = dputhelper.getopt(
 862             sys.argv[1:],
 863             'c:dDe:fhHlUopPsuvV', [
 864                 'debug', 'dinstall', 'check-only',
 865                 'check-version', 'config=', 'force', 'help',
 866                 'host-list', 'lintian', 'no-upload-log',
 867                 'passive', 'print', 'simulate', 'unchecked',
 868                 'delayed=', 'version'])
 869     for option, arg in opts:
 870         if option in ('-h', '--help'):
 871             sys.stdout.write(make_usage_message())
 872             return
 873         elif option in ('-v', '--version'):
 874             sys.stdout.write("{progname} {version}\n".format(
 875                     progname=progname, version=version))
 876             return
 877         elif option in ('-d', '--debug'):
 878             debug = 1
 879         elif option in ('-D', '--dinstall'):
 880             dinstall = True
 881         elif option in ('-c', '--config'):
 882             config_file = arg
 883         elif option in ('-f', '--force'):
 884             force_upload = 1
 885         elif option in ('-H', '--host-list'):
 886             config_host_list = 1
 887         elif option in ('-l', '--lintian'):
 888             call_lintian = 1
 889         elif option in ('-U', '--no-upload-log'):
 890             no_upload_log = 1
 891         elif option in ('-o', '--check-only'):
 892             check_only = True
 893         elif option in ('-p', '--print'):
 894             config_print = 1
 895         elif option in ('-P', '--passive'):
 896             ftp_passive_mode = 1
 897         elif option in ('-s', '--simulate'):
 898             simulate = True
 899         elif option in ('-u', '--unchecked'):
 900             unsigned_upload = True
 901         elif option in ('-e', '--delayed'):
 902             if arg in map(str, range(16)):
 903                 delay_upload = arg
 904             else:
 905                 sys.stdout.write(
 906                         "Incorrect delayed argument,"
 907                         " dput only understands 0 to 15.\n")
 908                 sys.exit(1)
 909         elif option in ('-V', '--check_version'):
 910             check_version = 1
 911
 912     # Always print the version number in the debug output
 913     # so that in case of bugreports, we know which version
 914     # the user has installed
 915     if debug:
 916         sys.stdout.write(
 917                 "D: {progname} {version}\n".format(
 918                     progname=progname, version=version))
 919
 920     # Try to get the login from the enviroment
 921     if 'USER' in os.environ:
 922         login = os.environ['USER']
 923         if debug:
 924             sys.stdout.write("D: Login: %s\n" % login)
 925     else:
 926         sys.stdout.write("$USER not set, will use login information.\n")
 927         # Else use the current username
 928         login = pwd.getpwuid(os.getuid())[0]
 929         if debug:
 930             sys.stdout.write("D: User-ID: %s\n" % os.getuid())
 931             sys.stdout.write("D: Login: %s\n" % login)
 932
 933     # Start Config File Parsing.
 934     config = read_configs(config_file, debug)
 935
 936     if config_print:
 937         print_config(config, debug)
 938         sys.exit(0)
 939
 940     if config_host_list:
 941         sys.stdout.write(
 942                 "\n"
 943                 "Default Method: %s\n"
 944                 "\n" % config.get('DEFAULT', 'method'))
 945         for section in config.sections():
 946             distributions = ""
 947             if config.get(section, 'distributions'):
 948                 distributions = (
 949                         ", distributions: %s" %
 950                         config.get(section, 'distributions'))
 951             sys.stdout.write(
 952                     "%s => %s  (Upload method: %s%s)\n" % (
 953                         section,
 954                         config.get(section, 'fqdn'),
 955                         config.get(section, 'method'),
 956                         distributions))
 957         sys.stdout.write("\n")
 958         sys.exit(0)
 959
 960     # Process further command line options.
 961     if len(args) == 0:
 962         sys.stdout.write(
 963                 "No package or host has been provided, see dput -h\n")
 964         sys.exit(0)
 965     elif len(args) == 1 and not check_only:
 966         package_to_upload = args[0:]
 967     else:
 968         if not check_only:
 969             if debug:
 970                 sys.stdout.write(
 971                         "D: Checking if a host was named"
 972                         " on the command line.\n")
 973             if config.has_section(args[0]):
 974                 if debug:
 975                     sys.stdout.write("D: Host %s found in config\n" % args[0])
 976                 # Host was also named, so only the rest will be a list
 977                 # of packages to upload.
 978                 preferred_host = args[0]
 979                 package_to_upload = args[1:]
 980             elif (
 981                     not config.has_section(args[0])
 982                     and not args[0].endswith('.changes')):
 983                 sys.stderr.write("No host %s found in config\n" % args[0])
 984                 if args[0] == 'gluck_delayed':
 985                     sys.stderr.write("""
 986     The delayed upload queue has been moved back to
 987     ftp-master (aka ftp.upload.debian.org).
 988     """)
 989                 sys.exit(1)
 990             else:
 991                 if debug:
 992                     sys.stdout.write("D: No host named on command line.\n")
 993                 # Only packages have been named on the command line.
 994                 preferred_host = ''
 995                 package_to_upload = args[0:]
 996         else:
 997             if debug:
 998                 sys.stdout.write("D: Checking for the package name.\n")
 999             if config.has_section(args[0]):
1000                 sys.stdout.write("D: Host %s found in config.\n" % args[0])
1001                 preferred_host = args[0]
1002                 package_to_upload = args[1:]
1003             elif not config.has_section(args[0]):
1004                 sys.stdout.write("D: No host %s found in config\n" % args[0])
1005                 package_to_upload = args[0:]
1006
1007     upload_methods = import_upload_functions()
1008
1009     # Run the same checks for all packages that have been given on
1010     # the command line
1011     for package_name in package_to_upload:
1012         # Check that a .changes file was given on the command line
1013         # and no matching .upload file exists.
1014         if package_name[-8:] != '.changes':
1015             sys.stdout.write(
1016                     "Not a .changes file.\n"
1017                     "Please select a .changes file to upload.\n")
1018             sys.stdout.write("Tried to upload: %s\n" % package_name)
1019             sys.exit(1)
1020
1021         # Construct the package name for further usage.
1022         path, name_of_package = os.path.split(package_name)
1023         if path == '':
1024             path = os.getcwd()
1025
1026         # Define the host to upload to.
1027         if preferred_host == '':
1028             host = guess_upload_host(path, name_of_package, config)
1029         else:
1030             host = preferred_host
1031         if config.get(host, 'method') == 'local':
1032             fqdn = 'localhost'
1033         else:
1034             fqdn = config.get(host, 'fqdn')
1035
1036         # Check if we already did this upload or not
1037         check_upload_logfile(
1038                 package_name, host, fqdn,
1039                 check_only, call_lintian, force_upload, debug)
1040
1041         # Run the change file tests.
1042         files_to_upload = verify_files(
1043                 path, name_of_package, host,
1044                 config, check_only, check_version, unsigned_upload, debug)
1045
1046         # Run the lintian test if the user asked us to do so.
1047         if (
1048                 call_lintian or
1049                 config.getboolean(host, 'run_lintian') == 1):
1050             run_lintian_test(os.path.join(path, name_of_package))
1051         elif check_only:
1052             sys.stdout.write(
1053                     "Warning: The option -o does not automatically include \n"
1054                     "a lintian run any more. Please use the option -ol if \n"
1055                     "you want to include running lintian in your checking.\n")
1056
1057         # don't upload, skip to the next item
1058         if check_only:
1059             sys.stdout.write("Package checked by dput.\n")
1060             continue
1061
1062         # Pre-Upload Commands
1063         if len(config.get(host, 'pre_upload_command')) != 0:
1064             type = 'pre'
1065             command = config.get(host, 'pre_upload_command')
1066             execute_command(command, type, debug)
1067
1068         # Check the upload methods that we have as default and per host
1069         if debug:
1070             sys.stdout.write(
1071                     "D: Default Method: %s\n"
1072                     % config.get('DEFAULT', 'method'))
1073         if config.get('DEFAULT', 'method') not in upload_methods:
1074             sys.stdout.write(
1075                     "Unknown upload method: %s\n"
1076                     % config.get('DEFAULT', 'method'))
1077             sys.exit(1)
1078         if debug:
1079             sys.stdout.write(
1080                     "D: Host Method: %s\n" % config.get(host, 'method'))
1081         if config.get(host, 'method') not in upload_methods:
1082             sys.stdout.write(
1083                     "Unknown upload method: %s\n"
1084                     % config.get(host, 'method'))
1085             sys.exit(1)
1086
1087         # Inspect the Config and set appropriate upload method
1088         if not config.get(host, 'method'):
1089             method = config.get('DEFAULT', 'method')
1090         else:
1091             method = config.get(host, 'method')
1092
1093         # Check now the login and redefine it if needed
1094         if (
1095                 len(config.get(host, 'login')) != 0 and
1096                 config.get(host, 'login') != 'username'):
1097             login = config.get(host, 'login')
1098             if debug:
1099                 sys.stdout.write(
1100                         "D: Login %s from section %s used\n" % (login, host))
1101         elif (
1102                 len(config.get('DEFAULT', 'login')) != 0 and
1103                 config.get('DEFAULT', 'login') != 'username'):
1104             login = config.get('DEFAULT', 'login')
1105             if debug:
1106                 sys.stdout.write("D: Default login %s used\n" % login)
1107         else:
1108             if debug:
1109                 sys.stdout.write(
1110                         "D: Neither host %s nor default login used. Using %s\n"
1111                         % (host, login))
1112
1113         incoming = config.get(host, 'incoming')
1114
1115         # if delay_upload wasn't passed via -e/--delayed
1116         if delay_upload is None:
1117             delay_upload = config.get(host, 'delayed')
1118             if not delay_upload:
1119                 delay_upload = config.get('DEFAULT', 'delayed')
1120
1121         if delay_upload:
1122             if int(delay_upload) == 0:
1123                 sys.stdout.write("Uploading to DELAYED/0-day.\n")
1124             if incoming[-1] == '/':
1125                 first_char = ''
1126             else:
1127                 first_char = '/'
1128             incoming += first_char + 'DELAYED/' + delay_upload + '-day'
1129             delayed = ' [DELAYED/' + delay_upload + ']'
1130         else:
1131             delayed = ''
1132
1133         # Do the actual upload
1134         if not simulate:
1135             sys.stdout.write(
1136                     "Uploading to %s%s (via %s to %s):\n"
1137                     % (host, delayed, method, fqdn))
1138             if debug:
1139                 sys.stdout.write("D: FQDN: %s\n" % fqdn)
1140                 sys.stdout.write("D: Login: %s\n" % login)
1141                 sys.stdout.write("D: Incoming: %s\n" % incoming)
1142             progress = config.getint(host, 'progress_indicator')
1143             if not os.isatty(1):
1144                 progress = 0
1145             if method == 'ftp':
1146                 if ':' in fqdn:
1147                     fqdn, port = fqdn.rsplit(":", 1)
1148                 else:
1149                     port = 21
1150                 ftp_mode = config.getboolean(host, 'passive_ftp')
1151                 if ftp_passive_mode == 1:
1152                     ftp_mode = 1
1153                 if debug:
1154                     sys.stdout.write("D: FTP port: %s\n" % port)
1155                     if ftp_mode == 1:
1156                         sys.stdout.write("D: Using passive ftp\n")
1157                     else:
1158                         sys.stdout.write("D: Using active ftp\n")
1159                 upload_methods[method](
1160                         fqdn, login, incoming,
1161                         files_to_upload, debug, ftp_mode,
1162                         progress=progress, port=port)
1163             elif method == 'scp':
1164                 if debug and config.getboolean(host, 'scp_compress'):
1165                     sys.stdout.write("D: Setting compression for scp\n")
1166                 scp_compress = config.getboolean(host, 'scp_compress')
1167                 ssh_config_options = [
1168                         y for y in (
1169                             x.strip() for x in
1170                             config.get(host, 'ssh_config_options').split('\n'))
1171                         if y]
1172                 if debug:
1173                     sys.stdout.write(
1174                             "D: ssh config options:"
1175                             "\n  "
1176                             + "\n  ".join(ssh_config_options)
1177                             + "\n")
1178                 upload_methods[method](
1179                         fqdn, login, incoming,
1180                         files_to_upload, debug, scp_compress,
1181                         ssh_config_options)
1182             else:
1183                 upload_methods[method](
1184                         fqdn, login, incoming,
1185                         files_to_upload, debug, 0, progress=progress)
1186         # Or just simulate it.
1187         else:
1188             for file in files_to_upload:
1189                 sys.stdout.write(
1190                         "Uploading with %s: %s to %s:%s\n"
1191                         % (method, file, fqdn, incoming))
1192
1193         # Create the logfile after the package has
1194         # been put into the archive.
1195         if not simulate:
1196             if not no_upload_log:
1197                 create_upload_file(
1198                         name_of_package, host, fqdn, path,
1199                         files_to_upload, debug)
1200             sys.stdout.write("Successfully uploaded packages.\n")
1201         else:
1202             sys.stdout.write("Simulated upload.\n")
1203
1204         # Run dinstall if the user asked us to do so.
1205         if debug:
1206             sys.stdout.write("D: dinstall: %s\n" % dinstall)
1207             sys.stdout.write(
1208                     "D: Host Config: %s\n"
1209                     % config.getboolean(host, 'run_dinstall'))
1210         if config.getboolean(host, 'run_dinstall') == 1 or dinstall:
1211             if not simulate:
1212                 dinstall_caller(
1213                         name_of_package, host, fqdn, login, incoming, debug)
1214             else:
1215                 sys.stdout.write("Will run dinstall now.\n")
1216
1217         # Post-Upload Command
1218         if len(config.get(host, 'post_upload_command')) != 0:
1219             type = 'post'
1220             command = config.get(host, 'post_upload_command')
1221             execute_command(command, type, debug)
1222
1223     return
1224
1225
1226 if __name__ == '__main__':
1227     try:
1228         main()
1229     except KeyboardInterrupt:
1230         sys.stdout.write("Exiting due to user interrupt.\n")
1231         sys.exit(1)
1232     except dputhelper.DputException as e:
1233         sys.stderr.write("%s\n" % e)
1234         sys.exit(1)
1235
1236 \f
1237 # Local variables:
1238 # coding: utf-8
1239 # mode: python
1240 # End:
1241 # vim: fileencoding=utf-8 filetype=python :