dput/dput.py

   1 #! /usr/bin/python2
   2 # -*- coding: utf-8; -*-
   3 #
   4 # dput/dput.py
   5 # Part of ‘dput’, a Debian package upload toolkit.
   6 #
   7 # Copyright © 2015–2016 Ben Finney <bignose@debian.org>
   8 # Copyright © 2008–2013 Y Giridhar Appaji Nag <appaji@debian.org>
   9 # Copyright © 2006–2008 Thomas Viehmann <tv@beamnet.de>
  10 # Copyright © 2000–2005 Christian Kurz <shorty@debian.org>
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16 #
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU General Public License along
  23 # with this program; if not, write to the Free Software Foundation, Inc.,
  24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  25
  26 """ dput — Debian package upload tool. """
  27
  28 import os
  29 import os.path
  30 import sys
  31 import string
  32 import textwrap
  33 import re
  34 import signal
  35 import subprocess
  36 import pwd
  37 import stat
  38 import email.parser
  39 from hashlib import md5, sha1
  40 import importlib
  41 import pkgutil
  42
  43 # Now import our modules
  44 import ConfigParser
  45
  46 app_library_path = os.path.dirname(__file__)
  47
  48 from . import crypto
  49 from .helper import dputhelper
  50
  51
  52 debug = 0
  53
  54
  55 def import_upload_functions():
  56     """ Import upload method modules and make them available. """
  57     upload_methods = {}
  58
  59     package_name = "methods"
  60     modules_path = os.path.join(app_library_path, package_name)
  61     modules_found = [
  62             name for (__, name, ispkg) in
  63             pkgutil.iter_modules([modules_path])
  64             if not ispkg]
  65     if debug:
  66         sys.stdout.write("D: modules_found: %r\n" % modules_found)
  67     for module_name in modules_found:
  68         module = importlib.import_module("{package}.{module}".format(
  69                 package=".".join(["dput", package_name]),
  70                 module=module_name))
  71         if debug:
  72             sys.stdout.write("D: Module: %s (%r)\n" % (module_name, module))
  73         method_name = module_name
  74         if debug:
  75             sys.stdout.write("D: Method name: %s\n" % method_name)
  76
  77         upload_methods[method_name] = module.upload
  78
  79     return upload_methods
  80
  81
  82 def parse_changes(chg_fd):
  83     """ Parse the changes file. """
  84     check = chg_fd.read(5)
  85     if check != '-----':
  86         chg_fd.seek(0)
  87     else:
  88         # found a PGP header, gonna ditch the next 3 lines
  89         # eat the rest of the line
  90         chg_fd.readline()
  91         # Hash: SHA1
  92         chg_fd.readline()
  93         # empty line
  94         chg_fd.readline()
  95     if not chg_fd.readline().find('Format') != -1:
  96         chg_fd.readline()
  97     changes_text = chg_fd.read()
  98     changes = email.parser.HeaderParser().parsestr(changes_text)
  99     if 'files' not in changes:
 100         raise KeyError("No Files field in upload control file")
 101     for a in changes['files'].strip().split('\n'):
 102         if len(a.split()) != 5:
 103             sys.stderr.write("Invalid Files line in .changes:\n  %s\n" % a)
 104             sys.exit(1)
 105     return changes
 106
 107
 108 def read_configs(extra_config, debug):
 109     """ Read configuration settings from config files.
 110
 111         :param extra_config: Filesystem path of config file to read.
 112         :param debug: If true, enable debugging output.
 113         :return: The resulting `ConfigParser` instance.
 114
 115         Read config files in this order:
 116         * If specified on the command line, only read `extra_config`.
 117         * Otherwise, read ‘/etc/dput.cf’ then ‘~/.dput.cf’.
 118         The config parser will layer values.
 119
 120         """
 121     config = ConfigParser.ConfigParser()
 122
 123     config.set('DEFAULT', 'login', 'username')
 124     config.set('DEFAULT', 'method', 'scp')
 125     config.set('DEFAULT', 'hash', 'md5')
 126     config.set('DEFAULT', 'allow_unsigned_uploads', '0')
 127     config.set('DEFAULT', 'allow_dcut', '0')
 128     config.set('DEFAULT', 'distributions', '')
 129     config.set('DEFAULT', 'allowed_distributions', '')
 130     config.set('DEFAULT', 'run_lintian', '0')
 131     config.set('DEFAULT', 'run_dinstall', '0')
 132     config.set('DEFAULT', 'check_version', '0')
 133     config.set('DEFAULT', 'scp_compress', '0')
 134     config.set('DEFAULT', 'default_host_main', '')
 135     config.set('DEFAULT', 'post_upload_command', '')
 136     config.set('DEFAULT', 'pre_upload_command', '')
 137     config.set('DEFAULT', 'ssh_config_options', '')
 138     config.set('DEFAULT', 'passive_ftp', '1')
 139     config.set('DEFAULT', 'progress_indicator', '0')
 140     config.set('DEFAULT', 'delayed', '')
 141
 142     if extra_config:
 143         config_files = (extra_config,)
 144     else:
 145         config_files = ('/etc/dput.cf', os.path.expanduser("~/.dput.cf"))
 146     fd = None
 147     for config_file in config_files:
 148         try:
 149             fd = open(config_file)
 150         except IOError as e:
 151             if debug:
 152                 sys.stderr.write(
 153                         "%s: %s, skipping\n" % (e.strerror, config_file))
 154             continue
 155         if debug:
 156             sys.stdout.write(
 157                     "D: Parsing Configuration File %s\n" % config_file)
 158         try:
 159             config.readfp(fd)
 160         except ConfigParser.ParsingError as e:
 161             sys.stderr.write("Error parsing config file:\n%s\n" % str(e))
 162             sys.exit(1)
 163         fd.close()
 164     if fd is None:
 165         sys.stderr.write(
 166                 "Error: Could not open any configfile, tried %s\n"
 167                 % (', '.join(config_files)))
 168         sys.exit(1)
 169     # only check for fqdn and incoming dir, rest have reasonable defaults
 170     error = 0
 171     for section in config.sections():
 172         if config.get(section, 'method') == 'local':
 173             config.set(section, 'fqdn', 'localhost')
 174         if (
 175                 not config.has_option(section, 'fqdn') and
 176                 config.get(section, 'method') != 'local'):
 177             sys.stderr.write(
 178                     "Config error: %s must have a fqdn set\n" % section)
 179             error = 1
 180         if not config.has_option(section, 'incoming'):
 181             sys.stderr.write(
 182                     "Config error: %s must have an incoming directory set\n"
 183                     % section)
 184             error = 1
 185     if error:
 186         sys.exit(1)
 187
 188     return config
 189
 190
 191 hexStr = string.hexdigits
 192
 193
 194 def hexify_string(string):
 195     """ Convert a string of bytes to hexadecimal text representation. """
 196     char = ''
 197     ord_func = ord if isinstance(string, str) else int
 198     for c in string:
 199         char += hexStr[(ord_func(c) >> 4) & 0xF] + hexStr[ord_func(c) & 0xF]
 200     return char
 201
 202
 203 def checksum_test(filename, hash_name):
 204     """ Get the hex string for the hash of a file's content.
 205
 206         :param filename: Path to the file to read.
 207         :param hash_name: Name of the hash to use.
 208         :return: The computed hash value, as hexadecimal text.
 209
 210         Currently supports md5, sha1. ripemd may come in the future.
 211
 212         """
 213     try:
 214         file_to_test = open(filename, 'rb')
 215     except IOError:
 216         sys.stdout.write("Can't open %s\n" % filename)
 217         sys.exit(1)
 218
 219     if hash_name == 'md5':
 220         hash_type = md5
 221     else:
 222         hash_type = sha1
 223
 224     check_obj = hash_type()
 225
 226     while 1:
 227         data = file_to_test.read(65536)
 228         if len(data) == 0:
 229             break
 230         check_obj.update(data)
 231
 232     file_to_test.close()
 233     checksum = hexify_string(check_obj.digest())
 234
 235     return checksum
 236
 237
 238 def check_upload_variant(changes, debug):
 239     """ Check if this is a binary_upload only or not. """
 240     binary_upload = 0
 241     if 'architecture' in changes:
 242         arch = changes['architecture']
 243         if debug:
 244             sys.stdout.write("D: Architecture: %s\n" % arch)
 245         if arch.find('source') < 0:
 246             if debug:
 247                 sys.stdout.write("D: Doing a binary upload only.\n")
 248             binary_upload = 1
 249     return binary_upload
 250
 251
 252 def verify_signature(
 253         host, changes_file_path, dsc_file_path,
 254         config, check_only, unsigned_upload, binary_upload, debug):
 255     """ Check the signature on the two files given via function call.
 256
 257         :param host: Configuration host name.
 258         :param changes_file_path: Filesystem path of upload control file.
 259         :param dsc_file_path: Filesystem path of source control file.
 260         :param config: `ConfigParser` instance for this application.
 261         :param check_only: If true, no upload is requested.
 262         :param unsigned_upload: If true, allow an unsigned upload.
 263         :param binary_upload: If true, this upload excludes source.
 264         :param debug: If true, enable debugging output.
 265         :return: ``None``.
 266
 267         """
 268
 269     def assert_good_signature_or_exit(path):
 270         """ Assert the signature on the file at `path` is good. """
 271         try:
 272             with open(path) as infile:
 273                 crypto.check_file_signature(infile)
 274         except Exception as exc:
 275             if isinstance(exc, crypto.gpgme.GpgmeError):
 276                 sys.stdout.write("{}\n".format(exc))
 277                 sys.exit(1)
 278             else:
 279                 raise
 280
 281     if debug:
 282         sys.stdout.write(
 283                 "D: upload control file: {}\n".format(changes_file_path))
 284         sys.stdout.write(
 285                 "D: source control file: {}\n".format(dsc_file_path))
 286     if ((check_only or config.getboolean(host, 'allow_unsigned_uploads') == 0)
 287             and not unsigned_upload):
 288         sys.stdout.write("Checking signature on .changes\n")
 289         assert_good_signature_or_exit(changes_file_path)
 290         if not binary_upload:
 291             sys.stdout.write("Checking signature on .dsc\n")
 292             assert_good_signature_or_exit(dsc_file_path)
 293
 294
 295 def source_check(changes, debug):
 296     """ Check if a source tarball has to be included in the package or not. """
 297     include_orig = include_tar = 0
 298     if 'version' in changes:
 299         version = changes['version']
 300         if debug:
 301             sys.stdout.write("D: Package Version: %s\n" % version)
 302         # versions with a dash in them are for non-native only
 303         if version.find('-') == -1:
 304             # debian native
 305             include_tar = 1
 306         else:
 307             if version.find(':') > 0:
 308                 if debug:
 309                     sys.stdout.write("D: Epoch found\n")
 310                 epoch, version = version.split(':', 1)
 311             pos = version.rfind('-')
 312             upstream_version = version[0:pos]
 313             debian_version = version[pos + 1:]
 314             if debug:
 315                 sys.stdout.write(
 316                         "D: Upstream Version: %s\n" % upstream_version)
 317                 sys.stdout.write("D: Debian Version: %s\n" % debian_version)
 318             if (
 319                     debian_version == '0.1' or debian_version == '1'
 320                     or debian_version == '1.1'):
 321                 include_orig = 1
 322             else:
 323                 include_tar = 1
 324     return (include_orig, include_tar)
 325
 326
 327 def verify_files(
 328         path, filename, host,
 329         config, check_only, check_version, unsigned_upload, debug):
 330     """ Run some tests on the files to verify that they are in good shape.
 331
 332         :param path: Directory path of the upload control file.
 333         :param filename: Filename of the upload control file.
 334         :param host: Configuration host name.
 335         :param config: `ConfigParser` instance for this application.
 336         :param check_only: If true, no upload is requested.
 337         :param check_version: If true, check the package version
 338             before upload.
 339         :param unsigned_upload: If true, allow an unsigned upload.
 340         :param debug: If true, enable debugging output.
 341         :return: A collection of filesystem paths of all files to upload.
 342
 343         """
 344     file_seen = include_orig_tar_gz = include_tar_gz = binary_only = 0
 345     files_to_upload = []
 346
 347     name_of_file = filename
 348
 349     change_file = os.path.join(path, name_of_file)
 350
 351     if debug:
 352         sys.stdout.write(
 353                 "D: Validating contents of changes file %s\n" % change_file)
 354     try:
 355         chg_fd = open(change_file, 'r')
 356     except IOError:
 357         sys.stdout.write("Can't open %s\n" % change_file)
 358         sys.exit(1)
 359     changes = parse_changes(chg_fd)
 360     chg_fd.close
 361
 362     # Find out if it's a binary only upload or not
 363     binary_upload = check_upload_variant(changes, debug)
 364
 365     if binary_upload:
 366         dsc_file = ''
 367     else:
 368         dsc_file = None
 369         for file in changes['files'].strip().split('\n'):
 370             # filename only
 371             filename = file.split()[4]
 372             if filename.find('.dsc') != -1:
 373                 if debug:
 374                     sys.stdout.write("D: dsc-File: %s\n" % filename)
 375                 dsc_file = os.path.join(path, filename)
 376         if not dsc_file:
 377             sys.stderr.write("Error: no dsc file found in sourceful upload\n")
 378             sys.exit(1)
 379
 380     # Run the check to verify that the package has been tested.
 381     try:
 382         if config.getboolean(host, 'check_version') == 1 or check_version:
 383             version_check(path, changes, debug)
 384     except ConfigParser.NoSectionError as e:
 385         sys.stderr.write("Error in config file:\n%s\n" % str(e))
 386         sys.exit(1)
 387
 388     # Verify the signature of the maintainer
 389     verify_signature(
 390             host, change_file, dsc_file,
 391             config, check_only, unsigned_upload, binary_upload, debug)
 392
 393     # Check the sources
 394     (include_orig_tar_gz, include_tar_gz) = source_check(changes, debug)
 395
 396     # Check md5sum and the size
 397     file_list = changes['files'].strip().split('\n')
 398     hash_name = config.get('DEFAULT', 'hash')
 399     for line in file_list:
 400         (check_sum, size, section, priority, file) = line.split()
 401         file_to_upload = os.path.join(path, file)
 402         if debug:
 403             sys.stdout.write("D: File to upload: %s\n" % file_to_upload)
 404         if checksum_test(file_to_upload, hash_name) != check_sum:
 405             if debug:
 406                 sys.stdout.write(
 407                         "D: Checksum from .changes: %s\n" % check_sum)
 408                 sys.stdout.write(
 409                         "D: Generated Checksum: %s\n" %
 410                         checksum_test(file_to_upload, hash_name))
 411             sys.stdout.write(
 412                     "Checksum doesn't match for %s\n" % file_to_upload)
 413             sys.exit(1)
 414         else:
 415             if debug:
 416                 sys.stdout.write(
 417                         "D: Checksum for %s is fine\n" % file_to_upload)
 418         if os.stat(file_to_upload)[stat.ST_SIZE] != int(size):
 419             if debug:
 420                 sys.stdout.write("D: size from .changes: %s\n" % size)
 421                 sys.stdout.write(
 422                         "D: calculated size: %s\n"
 423                         % os.stat(file_to_upload)[stat.ST_SIZE])
 424             sys.stdout.write(
 425                     "size doesn't match for %s\n" % file_to_upload)
 426
 427         files_to_upload.append(file_to_upload)
 428
 429     # Check filenames
 430     for file in files_to_upload:
 431         if file[-12:] == '.orig.tar.gz' and not include_orig_tar_gz:
 432             if debug:
 433                 sys.stdout.write("D: Filename: %s\n" % file)
 434                 sys.stdout.write("D: Suffix: %s\n\n" % file[-12:])
 435             sys.stdout.write(
 436                     "Package includes an .orig.tar.gz file although"
 437                     " the debian revision suggests\n"
 438                     "that it might not be required."
 439                     " Multiple uploads of the .orig.tar.gz may be\n"
 440                     "rejected by the upload queue management software.\n")
 441         elif (
 442                 file[-7:] == '.tar.gz' and not include_tar_gz
 443                 and not include_orig_tar_gz):
 444             if debug:
 445                 sys.stdout.write("D: Filename: %s\n" % file)
 446                 sys.stdout.write("D: Suffix: %s\n" % file[-7:])
 447             sys.stdout.write(
 448                     "Package includes a .tar.gz file although"
 449                     " the version suggests that it might\n"
 450                     "not be required."
 451                     " Multiple uploads of the .tar.gz may be rejected by the\n"
 452                     "upload queue management software.\n")
 453
 454     distribution = changes.get('distribution')
 455     allowed_distributions = config.get(host, 'allowed_distributions')
 456     if distribution and allowed_distributions:
 457         if debug:
 458             sys.stdout.write(
 459                     "D: Checking: distribution %s matches %s\n"
 460                     % (distribution, allowed_distributions))
 461         if not re.match(allowed_distributions, distribution):
 462             raise dputhelper.DputUploadFatalException(
 463                     "Error: uploading files for distribution %s to %s"
 464                     " not allowed."
 465                     % (distribution, host))
 466
 467     if debug:
 468         sys.stdout.write("D: File to upload: %s\n" % change_file)
 469     files_to_upload.append(change_file)
 470
 471     return files_to_upload
 472
 473
 474 def print_config(config, debug):
 475     """ Print the configuration and exit. """
 476     sys.stdout.write("\n")
 477     config.write(sys.stdout)
 478     sys.stdout.write("\n")
 479
 480
 481 def create_upload_file(package, host, fqdn, path, files_to_upload, debug):
 482     """ Write the log file for the upload.
 483
 484         :param package: File name of package to upload.
 485         :param host: Configuration host name.
 486         :param fqdn: Fully-qualified domain name of the remote host.
 487         :param path: Filesystem path of the upload control file.
 488         :param debug: If true, enable debugging output.
 489         :return: ``None``.
 490
 491         The upload log file is named ‘basename.hostname.upload’, where
 492         “basename” is the package file name without suffix, and
 493         “hostname” is the name of the host as specified in the
 494         configuration file.
 495
 496         For example, uploading ‘foo_1.2.3-1_xyz.deb’ to host ‘bar’
 497         will be logged to ‘foo_1.2.3-1_xyz.bar.upload’.
 498
 499         The upload log file is written to the
 500         directory containing the upload control file.
 501
 502         """
 503     # only need first part
 504     base = os.path.splitext(package)[0]
 505     logfile_name = os.path.join(path, base + '.' + host + '.upload')
 506     if debug:
 507         sys.stdout.write("D: Writing logfile: %s\n" % logfile_name)
 508     try:
 509         if os.access(logfile_name, os.R_OK):
 510             logfile_fd = open(logfile_name, 'a')
 511         else:
 512             logfile_fd = open(logfile_name, 'w')
 513     except IOError:
 514         sys.stderr.write("Could not write %s\n" % logfile_name)
 515         sys.exit(1)
 516
 517     for file in files_to_upload:
 518         entry_for_logfile = (
 519                 'Successfully uploaded ' + os.path.basename(file) +
 520                 ' to ' + fqdn + ' for ' + host + '.\n')
 521         logfile_fd.write(entry_for_logfile)
 522     logfile_fd.close()
 523
 524
 525 def run_lintian_test(changes_file):
 526     """ Run lintian on the changes file and stop if it finds errors. """
 527
 528     if os.access(changes_file, os.R_OK):
 529         if os.access("/usr/bin/lintian", os.R_OK):
 530             old_signal = signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 531             sys.stdout.write("Package is now being checked with lintian.\n")
 532             if dputhelper.check_call(
 533                     ['lintian', '-i', changes_file]
 534                     ) != dputhelper.EXIT_STATUS_SUCCESS:
 535                 sys.stdout.write(
 536                         "\n"
 537                         "Lintian says this package is not compliant"
 538                         " with the current policy.\n"
 539                         "Please check the current policy and your package.\n"
 540                         "Also see lintian documentation about overrides.\n")
 541                 sys.exit(1)
 542             else:
 543                 signal.signal(signal.SIGPIPE, old_signal)
 544                 return 0
 545         else:
 546             sys.stdout.write(
 547                     "lintian is not installed, skipping package test.\n")
 548     else:
 549         sys.stdout.write("Can't read %s\n" % changes_file)
 550         sys.exit(1)
 551
 552
 553 def guess_upload_host(path, filename, config):
 554     """ Guess the host where the package should be uploaded to.
 555
 556         :param path: Directory path of the upload control file.
 557         :param filename: Filename of the upload control file.
 558         :param config: `ConfigParser` instance for this application.
 559         :return: The hostname determined for this upload.
 560
 561         This is based on information from the upload control
 562         (‘*.changes’) file.
 563
 564         """
 565     non_us = 0
 566     distribution = ""
 567     dist_re = re.compile(r'^Distribution: (.*)')
 568
 569     name_of_file = filename
 570     changes_file = os.path.join(path, name_of_file)
 571
 572     try:
 573         changes_file_fd = open(changes_file, 'r')
 574     except IOError:
 575         sys.stdout.write("Can't open %s\n" % changes_file)
 576         sys.exit(1)
 577     lines = changes_file_fd.readlines()
 578     for line in lines:
 579         match = dist_re.search(line)
 580         if match:
 581             distribution = match.group(1)
 582
 583     # Try to guess a host based on the Distribution: field
 584     if distribution:
 585         for section in config.sections():
 586             host_dists = config.get(section, 'distributions')
 587             if not host_dists:
 588                 continue
 589             for host_dist in host_dists.split(','):
 590                 if distribution == host_dist.strip():
 591                     if debug:
 592                         sys.stdout.write(
 593                                 "D: guessing host %s"
 594                                 " based on distribution %s\n"
 595                                 % (section, host_dist))
 596                     return section
 597
 598     if len(config.get('DEFAULT', 'default_host_main')) != 0:
 599         sys.stdout.write(
 600                 "Trying to upload package to %s\n"
 601                 % config.get('DEFAULT', 'default_host_main'))
 602         return config.get('DEFAULT', 'default_host_main')
 603     else:
 604         sys.stdout.write(
 605                 "Trying to upload package to ftp-master"
 606                 " (ftp.upload.debian.org)\n")
 607         return "ftp-master"
 608
 609
 610 def dinstall_caller(filename, host, fqdn, login, incoming, debug):
 611     """ Run ‘dinstall’ for the package on the remote host.
 612
 613         :param filename: Debian package filename to install.
 614         :param host: Configuration host name.
 615         :param fqdn: Fully-qualified domain name of the remote host.
 616         :param login: Username for login to the remote host.
 617         :param incoming: Filesystem path on remote host for incoming
 618             packages.
 619         :param debug: If true, enable debugging output.
 620         :return: ``None``.
 621
 622         Run ‘dinstall’ on the remote host in test mode, and present
 623         the output to the user.
 624
 625         This is so the user can see if the package would be installed
 626         or not.
 627
 628         """
 629     command = [
 630             'ssh', '%s@%s' % (login, fqdn),
 631             'cd', '%s' % incoming,
 632             ';', 'dinstall', '-n', '%s' % filename]
 633     if debug:
 634         sys.stdout.write(
 635                 "D: Logging into %s@%s:%s\n" % (login, host, incoming))
 636         sys.stdout.write("D: dinstall -n %s\n" % filename)
 637     if dputhelper.check_call(command) != dputhelper.EXIT_STATUS_SUCCESS:
 638         sys.stdout.write(
 639                 "Error occured while trying to connect, or while"
 640                 " attempting to run dinstall.\n")
 641         sys.exit(1)
 642
 643
 644 def version_check(path, changes, debug):
 645     """ Check if the caller has installed the package also on his system.
 646
 647         This is for testing purposes before uploading it. If not, we
 648         reject the upload.
 649
 650         """
 651     files_to_check = []
 652
 653     # Get arch
 654     dpkg_proc = subprocess.Popen(
 655             'dpkg --print-architecture',
 656             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 657             shell=True, close_fds=True)
 658     dpkg_stdout = dputhelper.make_text_stream(dpkg_proc.stdout)
 659     dpkg_stderr = dputhelper.make_text_stream(dpkg_proc.stderr)
 660     dpkg_output = dpkg_stdout.read()
 661     dpkg_architecture = dpkg_output.strip()
 662     dpkg_stdout.close()
 663     dpkg_stderr_output = dpkg_stderr.read()
 664     dpkg_stderr.close()
 665     if debug and dpkg_stderr_output:
 666         sys.stdout.write(
 667                 "D: dpkg-architecture stderr output:"
 668                 " %r\n" % dpkg_stderr_output)
 669     if debug:
 670         sys.stdout.write(
 671                 "D: detected architecture: '%s'\n" % dpkg_architecture)
 672
 673     # Get filenames of deb files:
 674     for file in changes['files'].strip().split('\n'):
 675         filename = os.path.join(path, file.split()[4])
 676         if filename.endswith('.deb'):
 677             if debug:
 678                 sys.stdout.write("D: Debian Package: %s\n" % filename)
 679             dpkg_proc = subprocess.Popen(
 680                     'dpkg --field %s' % filename,
 681                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 682                     shell=True, close_fds=True)
 683             dpkg_stdout = dputhelper.make_text_stream(dpkg_proc.stdout)
 684             dpkg_stderr = dputhelper.make_text_stream(dpkg_proc.stderr)
 685             dpkg_output = dpkg_stdout.read()
 686             dpkg_stdout.close()
 687             dpkg_fields = email.parser.HeaderParser().parsestr(dpkg_output)
 688             dpkg_stderr_output = dpkg_stderr.read()
 689             dpkg_stderr.close()
 690             if debug and dpkg_stderr_output:
 691                 sys.stdout.write(
 692                         "D: dpkg stderr output:"
 693                         " %r\n" % dpkg_stderr_output)
 694             if (
 695                     dpkg_architecture
 696                     and dpkg_fields['architecture'] not in [
 697                         'all', dpkg_architecture]):
 698                 if debug:
 699                     sys.stdout.write(
 700                             "D: not install-checking %s due to arch mismatch\n"
 701                             % filename)
 702             else:
 703                 package_name = dpkg_fields['package']
 704                 version_number = dpkg_fields['version']
 705                 if debug:
 706                     sys.stdout.write(
 707                             "D: Package to Check: %s\n" % package_name)
 708                 if debug:
 709                     sys.stdout.write(
 710                             "D: Version to Check: %s\n" % version_number)
 711                 files_to_check.append((package_name, version_number))
 712
 713     for file, version_to_check in files_to_check:
 714         if debug:
 715             sys.stdout.write("D: Name of Package: %s\n" % file)
 716         dpkg_proc = subprocess.Popen(
 717                 'dpkg -s %s' % file,
 718                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 719                 shell=True, close_fds=True)
 720         dpkg_stdout = dputhelper.make_text_stream(dpkg_proc.stdout)
 721         dpkg_stderr = dputhelper.make_text_stream(dpkg_proc.stderr)
 722         dpkg_output = dpkg_stdout.read()
 723         dpkg_stdout.close()
 724         dpkg_fields = email.parser.HeaderParser().parsestr(dpkg_output)
 725         dpkg_stderr_output = dpkg_stderr.read()
 726         dpkg_stderr.close()
 727         if debug and dpkg_stderr_output:
 728             sys.stdout.write(
 729                     "D: dpkg stderr output:"
 730                     " %r\n" % dpkg_stderr_output)
 731         if 'version' in dpkg_fields:
 732             installed_version = dpkg_fields['version']
 733             if debug:
 734                 sys.stdout.write(
 735                         "D: Installed-Version: %s\n" % installed_version)
 736             if debug:
 737                 sys.stdout.write(
 738                         "D: Check-Version: %s\n" % version_to_check)
 739             if installed_version != version_to_check:
 740                 sys.stdout.write(
 741                         "Package to upload is not installed, but it appears"
 742                         " you have an older version installed.\n")
 743         else:
 744             sys.stdout.write(
 745                     "Uninstalled Package. Test it before uploading it.\n")
 746             sys.exit(1)
 747
 748
 749 def execute_command(command, position, debug=False):
 750     """ Run a command that the user-defined in the config_file.
 751
 752         :param command: Command line to execute.
 753         :param position: Position of the command: 'pre' or 'post'.
 754         :param debug: If true, enable debugging output.
 755         :return: ``None``.
 756
 757         """
 758     if debug:
 759         sys.stdout.write("D: Command: %s\n" % command)
 760     if subprocess.call(command, shell=True):
 761         raise dputhelper.DputUploadFatalException(
 762                 "Error: %s upload command failed." % position)
 763
 764
 765 def check_upload_logfile(
 766         changes_file, host, fqdn,
 767         check_only, call_lintian, force_upload, debug):
 768     """ Check if the user already put this package on the specified host.
 769
 770         :param changes_file: Filesystem path of upload control file.
 771         :param host: Configuration host name.
 772         :param fqdn: Fully-qualified domain name of the remote host.
 773         :param check_only: If true, no upload is requested.
 774         :param call_lintian: If true, a Lintian invocation is requested.
 775         :param force_upload: If true, don't check the upload log file.
 776         :param debug: If true, enable debugging output.
 777         :return: ``None``.
 778
 779         """
 780     uploaded = 0
 781     upload_logfile = changes_file[:-8] + '.' + host + '.upload'
 782     if not check_only and not force_upload:
 783         if not os.path.exists(upload_logfile):
 784             return
 785         try:
 786             fd_logfile = open(upload_logfile)
 787         except IOError:
 788             sys.stdout.write("Couldn't open %s\n" % upload_logfile)
 789             sys.exit(1)
 790         for line in fd_logfile.readlines():
 791             if line.find(fqdn) != -1:
 792                 uploaded = 1
 793         if uploaded:
 794             sys.stdout.write(
 795                     "Package has already been uploaded to %s on %s\n"
 796                     % (host, fqdn))
 797             sys.stdout.write("Nothing more to do for %s\n" % changes_file)
 798             sys.exit(0)
 799
 800
 801 def make_usage_message():
 802     """ Make the program usage help message. """
 803     text = textwrap.dedent("""\
 804         Usage: dput [options] [host] <package(s).changes>
 805          Supported options (see man page for long forms):
 806            -c: Config file to parse.
 807            -d: Enable debug messages.
 808            -D: Run dinstall after upload.
 809            -e: Upload to a delayed queue. Takes an argument from 0 to 15.
 810            -f: Force an upload.
 811            -h: Display this help message.
 812            -H: Display a list of hosts from the config file.
 813            -l: Run lintian before upload.
 814            -U: Do not write a .upload file after uploading.
 815            -o: Only check the package.
 816            -p: Print the configuration.
 817            -P: Use passive mode for ftp uploads.
 818            -s: Simulate the upload only.
 819            -u: Don't check GnuPG signature.
 820            -v: Display version information.
 821            -V: Check the package version and then upload it.
 822         """)
 823     return text
 824
 825
 826 def main():
 827     """ Main function, no further comment needed. :) """
 828
 829     global debug
 830
 831     check_version = config_print = force_upload = 0
 832     call_lintian = no_upload_log = config_host_list = 0
 833     ftp_passive_mode = 0
 834     preferred_host = ''
 835     config_file = ''
 836     dinstall = False
 837     check_only = False
 838     unsigned_upload = False
 839     delay_upload = None
 840     simulate = False
 841
 842     progname = dputhelper.get_progname()
 843     version = dputhelper.get_distribution_version()
 844
 845     # Parse Command Line Options.
 846     (opts, args) = dputhelper.getopt(
 847             sys.argv[1:],
 848             'c:dDe:fhHlUopPsuvV', [
 849                 'debug', 'dinstall', 'check-only',
 850                 'check-version', 'config=', 'force', 'help',
 851                 'host-list', 'lintian', 'no-upload-log',
 852                 'passive', 'print', 'simulate', 'unchecked',
 853                 'delayed=', 'version'])
 854     for option, arg in opts:
 855         if option in ('-h', '--help'):
 856             sys.stdout.write(make_usage_message())
 857             return
 858         elif option in ('-v', '--version'):
 859             sys.stdout.write("{progname} {version}\n".format(
 860                     progname=progname, version=version))
 861             return
 862         elif option in ('-d', '--debug'):
 863             debug = 1
 864         elif option in ('-D', '--dinstall'):
 865             dinstall = True
 866         elif option in ('-c', '--config'):
 867             config_file = arg
 868         elif option in ('-f', '--force'):
 869             force_upload = 1
 870         elif option in ('-H', '--host-list'):
 871             config_host_list = 1
 872         elif option in ('-l', '--lintian'):
 873             call_lintian = 1
 874         elif option in ('-U', '--no-upload-log'):
 875             no_upload_log = 1
 876         elif option in ('-o', '--check-only'):
 877             check_only = True
 878         elif option in ('-p', '--print'):
 879             config_print = 1
 880         elif option in ('-P', '--passive'):
 881             ftp_passive_mode = 1
 882         elif option in ('-s', '--simulate'):
 883             simulate = True
 884         elif option in ('-u', '--unchecked'):
 885             unsigned_upload = True
 886         elif option in ('-e', '--delayed'):
 887             if arg in map(str, range(16)):
 888                 delay_upload = arg
 889             else:
 890                 sys.stdout.write(
 891                         "Incorrect delayed argument,"
 892                         " dput only understands 0 to 15.\n")
 893                 sys.exit(1)
 894         elif option in ('-V', '--check_version'):
 895             check_version = 1
 896
 897     # Always print the version number in the debug output
 898     # so that in case of bugreports, we know which version
 899     # the user has installed
 900     if debug:
 901         sys.stdout.write(
 902                 "D: {progname} {version}\n".format(
 903                     progname=progname, version=version))
 904
 905     # Try to get the login from the enviroment
 906     if 'USER' in os.environ:
 907         login = os.environ['USER']
 908         if debug:
 909             sys.stdout.write("D: Login: %s\n" % login)
 910     else:
 911         sys.stdout.write("$USER not set, will use login information.\n")
 912         # Else use the current username
 913         login = pwd.getpwuid(os.getuid())[0]
 914         if debug:
 915             sys.stdout.write("D: User-ID: %s\n" % os.getuid())
 916             sys.stdout.write("D: Login: %s\n" % login)
 917
 918     # Start Config File Parsing.
 919     config = read_configs(config_file, debug)
 920
 921     if config_print:
 922         print_config(config, debug)
 923         sys.exit(0)
 924
 925     if config_host_list:
 926         sys.stdout.write(
 927                 "\n"
 928                 "Default Method: %s\n"
 929                 "\n" % config.get('DEFAULT', 'method'))
 930         for section in config.sections():
 931             distributions = ""
 932             if config.get(section, 'distributions'):
 933                 distributions = (
 934                         ", distributions: %s" %
 935                         config.get(section, 'distributions'))
 936             sys.stdout.write(
 937                     "%s => %s  (Upload method: %s%s)\n" % (
 938                         section,
 939                         config.get(section, 'fqdn'),
 940                         config.get(section, 'method'),
 941                         distributions))
 942         sys.stdout.write("\n")
 943         sys.exit(0)
 944
 945     # Process further command line options.
 946     if len(args) == 0:
 947         sys.stdout.write(
 948                 "No package or host has been provided, see dput -h\n")
 949         sys.exit(0)
 950     elif len(args) == 1 and not check_only:
 951         package_to_upload = args[0:]
 952     else:
 953         if not check_only:
 954             if debug:
 955                 sys.stdout.write(
 956                         "D: Checking if a host was named"
 957                         " on the command line.\n")
 958             if config.has_section(args[0]):
 959                 if debug:
 960                     sys.stdout.write("D: Host %s found in config\n" % args[0])
 961                 # Host was also named, so only the rest will be a list
 962                 # of packages to upload.
 963                 preferred_host = args[0]
 964                 package_to_upload = args[1:]
 965             elif (
 966                     not config.has_section(args[0])
 967                     and not args[0].endswith('.changes')):
 968                 sys.stderr.write("No host %s found in config\n" % args[0])
 969                 if args[0] == 'gluck_delayed':
 970                     sys.stderr.write("""
 971     The delayed upload queue has been moved back to
 972     ftp-master (aka ftp.upload.debian.org).
 973     """)
 974                 sys.exit(1)
 975             else:
 976                 if debug:
 977                     sys.stdout.write("D: No host named on command line.\n")
 978                 # Only packages have been named on the command line.
 979                 preferred_host = ''
 980                 package_to_upload = args[0:]
 981         else:
 982             if debug:
 983                 sys.stdout.write("D: Checking for the package name.\n")
 984             if config.has_section(args[0]):
 985                 sys.stdout.write("D: Host %s found in config.\n" % args[0])
 986                 preferred_host = args[0]
 987                 package_to_upload = args[1:]
 988             elif not config.has_section(args[0]):
 989                 sys.stdout.write("D: No host %s found in config\n" % args[0])
 990                 package_to_upload = args[0:]
 991
 992     upload_methods = import_upload_functions()
 993
 994     # Run the same checks for all packages that have been given on
 995     # the command line
 996     for package_name in package_to_upload:
 997         # Check that a .changes file was given on the command line
 998         # and no matching .upload file exists.
 999         if package_name[-8:] != '.changes':
1000             sys.stdout.write(
1001                     "Not a .changes file.\n"
1002                     "Please select a .changes file to upload.\n")
1003             sys.stdout.write("Tried to upload: %s\n" % package_name)
1004             sys.exit(1)
1005
1006         # Construct the package name for further usage.
1007         path, name_of_package = os.path.split(package_name)
1008         if path == '':
1009             path = os.getcwd()
1010
1011         # Define the host to upload to.
1012         if preferred_host == '':
1013             host = guess_upload_host(path, name_of_package, config)
1014         else:
1015             host = preferred_host
1016         if config.get(host, 'method') == 'local':
1017             fqdn = 'localhost'
1018         else:
1019             fqdn = config.get(host, 'fqdn')
1020
1021         # Check if we already did this upload or not
1022         check_upload_logfile(
1023                 package_name, host, fqdn,
1024                 check_only, call_lintian, force_upload, debug)
1025
1026         # Run the change file tests.
1027         files_to_upload = verify_files(
1028                 path, name_of_package, host,
1029                 config, check_only, check_version, unsigned_upload, debug)
1030
1031         # Run the lintian test if the user asked us to do so.
1032         if (
1033                 call_lintian or
1034                 config.getboolean(host, 'run_lintian') == 1):
1035             run_lintian_test(os.path.join(path, name_of_package))
1036         elif check_only:
1037             sys.stdout.write(
1038                     "Warning: The option -o does not automatically include \n"
1039                     "a lintian run any more. Please use the option -ol if \n"
1040                     "you want to include running lintian in your checking.\n")
1041
1042         # don't upload, skip to the next item
1043         if check_only:
1044             sys.stdout.write("Package checked by dput.\n")
1045             continue
1046
1047         # Pre-Upload Commands
1048         if len(config.get(host, 'pre_upload_command')) != 0:
1049             position = 'pre'
1050             command = config.get(host, 'pre_upload_command')
1051             execute_command(command, position, debug)
1052
1053         # Check the upload methods that we have as default and per host
1054         if debug:
1055             sys.stdout.write(
1056                     "D: Default Method: %s\n"
1057                     % config.get('DEFAULT', 'method'))
1058         if config.get('DEFAULT', 'method') not in upload_methods:
1059             sys.stdout.write(
1060                     "Unknown upload method: %s\n"
1061                     % config.get('DEFAULT', 'method'))
1062             sys.exit(1)
1063         if debug:
1064             sys.stdout.write(
1065                     "D: Host Method: %s\n" % config.get(host, 'method'))
1066         if config.get(host, 'method') not in upload_methods:
1067             sys.stdout.write(
1068                     "Unknown upload method: %s\n"
1069                     % config.get(host, 'method'))
1070             sys.exit(1)
1071
1072         # Inspect the Config and set appropriate upload method
1073         if not config.get(host, 'method'):
1074             method = config.get('DEFAULT', 'method')
1075         else:
1076             method = config.get(host, 'method')
1077
1078         # Check now the login and redefine it if needed
1079         if (
1080                 len(config.get(host, 'login')) != 0 and
1081                 config.get(host, 'login') != 'username'):
1082             login = config.get(host, 'login')
1083             if debug:
1084                 sys.stdout.write(
1085                         "D: Login %s from section %s used\n" % (login, host))
1086         elif (
1087                 len(config.get('DEFAULT', 'login')) != 0 and
1088                 config.get('DEFAULT', 'login') != 'username'):
1089             login = config.get('DEFAULT', 'login')
1090             if debug:
1091                 sys.stdout.write("D: Default login %s used\n" % login)
1092         else:
1093             if debug:
1094                 sys.stdout.write(
1095                         "D: Neither host %s nor default login used. Using %s\n"
1096                         % (host, login))
1097
1098         incoming = config.get(host, 'incoming')
1099
1100         # if delay_upload wasn't passed via -e/--delayed
1101         if delay_upload is None:
1102             delay_upload = config.get(host, 'delayed')
1103             if not delay_upload:
1104                 delay_upload = config.get('DEFAULT', 'delayed')
1105
1106         if delay_upload:
1107             if int(delay_upload) == 0:
1108                 sys.stdout.write("Uploading to DELAYED/0-day.\n")
1109             if incoming[-1] == '/':
1110                 first_char = ''
1111             else:
1112                 first_char = '/'
1113             incoming += first_char + 'DELAYED/' + delay_upload + '-day'
1114             delayed = ' [DELAYED/' + delay_upload + ']'
1115         else:
1116             delayed = ''
1117
1118         # Do the actual upload
1119         if not simulate:
1120             sys.stdout.write(
1121                     "Uploading to %s%s (via %s to %s):\n"
1122                     % (host, delayed, method, fqdn))
1123             if debug:
1124                 sys.stdout.write("D: FQDN: %s\n" % fqdn)
1125                 sys.stdout.write("D: Login: %s\n" % login)
1126                 sys.stdout.write("D: Incoming: %s\n" % incoming)
1127             progress = config.getint(host, 'progress_indicator')
1128             if not os.isatty(1):
1129                 progress = 0
1130             if method == 'ftp':
1131                 if ':' in fqdn:
1132                     fqdn, port = fqdn.rsplit(":", 1)
1133                 else:
1134                     port = 21
1135                 ftp_mode = config.getboolean(host, 'passive_ftp')
1136                 if ftp_passive_mode == 1:
1137                     ftp_mode = 1
1138                 if debug:
1139                     sys.stdout.write("D: FTP port: %s\n" % port)
1140                     if ftp_mode == 1:
1141                         sys.stdout.write("D: Using passive ftp\n")
1142                     else:
1143                         sys.stdout.write("D: Using active ftp\n")
1144                 upload_methods[method](
1145                         fqdn, login, incoming,
1146                         files_to_upload, debug, ftp_mode,
1147                         progress=progress, port=port)
1148             elif method == 'scp':
1149                 if debug and config.getboolean(host, 'scp_compress'):
1150                     sys.stdout.write("D: Setting compression for scp\n")
1151                 scp_compress = config.getboolean(host, 'scp_compress')
1152                 ssh_config_options = [
1153                         y for y in (
1154                             x.strip() for x in
1155                             config.get(host, 'ssh_config_options').split('\n'))
1156                         if y]
1157                 if debug:
1158                     sys.stdout.write(
1159                             "D: ssh config options:"
1160                             "\n  "
1161                             + "\n  ".join(ssh_config_options)
1162                             + "\n")
1163                 upload_methods[method](
1164                         fqdn, login, incoming,
1165                         files_to_upload, debug, scp_compress,
1166                         ssh_config_options)
1167             else:
1168                 upload_methods[method](
1169                         fqdn, login, incoming,
1170                         files_to_upload, debug, 0, progress=progress)
1171         # Or just simulate it.
1172         else:
1173             for file in files_to_upload:
1174                 sys.stdout.write(
1175                         "Uploading with %s: %s to %s:%s\n"
1176                         % (method, file, fqdn, incoming))
1177
1178         # Create the logfile after the package has
1179         # been put into the archive.
1180         if not simulate:
1181             if not no_upload_log:
1182                 create_upload_file(
1183                         name_of_package, host, fqdn, path,
1184                         files_to_upload, debug)
1185             sys.stdout.write("Successfully uploaded packages.\n")
1186         else:
1187             sys.stdout.write("Simulated upload.\n")
1188
1189         # Run dinstall if the user asked us to do so.
1190         if debug:
1191             sys.stdout.write("D: dinstall: %s\n" % dinstall)
1192             sys.stdout.write(
1193                     "D: Host Config: %s\n"
1194                     % config.getboolean(host, 'run_dinstall'))
1195         if config.getboolean(host, 'run_dinstall') == 1 or dinstall:
1196             if not simulate:
1197                 dinstall_caller(
1198                         name_of_package, host, fqdn, login, incoming, debug)
1199             else:
1200                 sys.stdout.write("Will run dinstall now.\n")
1201
1202         # Post-Upload Command
1203         if len(config.get(host, 'post_upload_command')) != 0:
1204             position = 'post'
1205             command = config.get(host, 'post_upload_command')
1206             execute_command(command, position, debug)
1207
1208     return
1209
1210
1211 if __name__ == '__main__':
1212     try:
1213         main()
1214     except KeyboardInterrupt:
1215         sys.stdout.write("Exiting due to user interrupt.\n")
1216         sys.exit(1)
1217     except dputhelper.DputException as e:
1218         sys.stderr.write("%s\n" % e)
1219         sys.exit(1)
1220
1221 \f
1222 # Local variables:
1223 # coding: utf-8
1224 # mode: python
1225 # End:
1226 # vim: fileencoding=utf-8 filetype=python :