dput/dput.py

   1 #! /usr/bin/python2
   2 # -*- coding: utf-8; -*-
   3 #
   4 # dput/dput.py
   5 # Part of ‘dput’, a Debian package upload toolkit.
   6 #
   7 # Copyright © 2015–2016 Ben Finney <bignose@debian.org>
   8 # Copyright © 2008–2013 Y Giridhar Appaji Nag <appaji@debian.org>
   9 # Copyright © 2006–2008 Thomas Viehmann <tv@beamnet.de>
  10 # Copyright © 2000–2005 Christian Kurz <shorty@debian.org>
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16 #
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU General Public License along
  23 # with this program; if not, write to the Free Software Foundation, Inc.,
  24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  25
  26 """ dput — Debian package upload tool. """
  27
  28 import os
  29 import os.path
  30 import sys
  31 import string
  32 import textwrap
  33 import re
  34 import signal
  35 import subprocess
  36 import pwd
  37 import stat
  38 import email.parser
  39 from hashlib import md5, sha1
  40 import importlib
  41 import pkgutil
  42
  43 # Now import our modules
  44 import ConfigParser
  45
  46 app_library_path = os.path.dirname(__file__)
  47
  48 from .helper import dputhelper
  49
  50
  51 debug = 0
  52
  53
  54 def import_upload_functions():
  55     """ Import upload method modules and make them available. """
  56     upload_methods = {}
  57
  58     package_name = "methods"
  59     modules_path = os.path.join(app_library_path, package_name)
  60     modules_found = [
  61             name for (__, name, ispkg) in
  62             pkgutil.iter_modules([modules_path])
  63             if not ispkg]
  64     if debug:
  65         sys.stdout.write("D: modules_found: %r\n" % modules_found)
  66     for module_name in modules_found:
  67         module = importlib.import_module("{package}.{module}".format(
  68                 package=".".join(["dput", package_name]),
  69                 module=module_name))
  70         if debug:
  71             sys.stdout.write("D: Module: %s (%r)\n" % (module_name, module))
  72         method_name = module_name
  73         if debug:
  74             sys.stdout.write("D: Method name: %s\n" % method_name)
  75
  76         upload_methods[method_name] = module.upload
  77
  78     return upload_methods
  79
  80
  81 def parse_changes(chg_fd):
  82     """ Parse the changes file. """
  83     check = chg_fd.read(5)
  84     if check != '-----':
  85         chg_fd.seek(0)
  86     else:
  87         # found a PGP header, gonna ditch the next 3 lines
  88         # eat the rest of the line
  89         chg_fd.readline()
  90         # Hash: SHA1
  91         chg_fd.readline()
  92         # empty line
  93         chg_fd.readline()
  94     if not chg_fd.readline().find('Format') != -1:
  95         chg_fd.readline()
  96     changes_text = chg_fd.read()
  97     changes = email.parser.HeaderParser().parsestr(changes_text)
  98     if 'files' not in changes:
  99         raise KeyError("No Files field in upload control file")
 100     for a in changes['files'].strip().split('\n'):
 101         if len(a.split()) != 5:
 102             sys.stderr.write("Invalid Files line in .changes:\n  %s\n" % a)
 103             sys.exit(1)
 104     return changes
 105
 106
 107 def read_configs(extra_config, debug):
 108     """ Read configuration settings from config files.
 109
 110         :param extra_config: Filesystem path of config file to read.
 111         :param debug: If true, enable debugging output.
 112         :return: The resulting `ConfigParser` instance.
 113
 114         Read config files in this order:
 115         * If specified on the command line, only read `extra_config`.
 116         * Otherwise, read ‘/etc/dput.cf’ then ‘~/.dput.cf’.
 117         The config parser will layer values.
 118
 119         """
 120     config = ConfigParser.ConfigParser()
 121
 122     config.set('DEFAULT', 'login', 'username')
 123     config.set('DEFAULT', 'method', 'scp')
 124     config.set('DEFAULT', 'hash', 'md5')
 125     config.set('DEFAULT', 'allow_unsigned_uploads', '0')
 126     config.set('DEFAULT', 'allow_dcut', '0')
 127     config.set('DEFAULT', 'distributions', '')
 128     config.set('DEFAULT', 'allowed_distributions', '')
 129     config.set('DEFAULT', 'run_lintian', '0')
 130     config.set('DEFAULT', 'run_dinstall', '0')
 131     config.set('DEFAULT', 'check_version', '0')
 132     config.set('DEFAULT', 'scp_compress', '0')
 133     config.set('DEFAULT', 'default_host_main', '')
 134     config.set('DEFAULT', 'post_upload_command', '')
 135     config.set('DEFAULT', 'pre_upload_command', '')
 136     config.set('DEFAULT', 'ssh_config_options', '')
 137     config.set('DEFAULT', 'passive_ftp', '1')
 138     config.set('DEFAULT', 'progress_indicator', '0')
 139     config.set('DEFAULT', 'delayed', '')
 140
 141     if extra_config:
 142         config_files = (extra_config,)
 143     else:
 144         config_files = ('/etc/dput.cf', os.path.expanduser("~/.dput.cf"))
 145     fd = None
 146     for config_file in config_files:
 147         try:
 148             fd = open(config_file)
 149         except IOError as e:
 150             if debug:
 151                 sys.stderr.write(
 152                         "%s: %s, skipping\n" % (e.strerror, config_file))
 153             continue
 154         if debug:
 155             sys.stdout.write(
 156                     "D: Parsing Configuration File %s\n" % config_file)
 157         try:
 158             config.readfp(fd)
 159         except ConfigParser.ParsingError as e:
 160             sys.stderr.write("Error parsing config file:\n%s\n" % str(e))
 161             sys.exit(1)
 162         fd.close()
 163     if fd is None:
 164         sys.stderr.write(
 165                 "Error: Could not open any configfile, tried %s\n"
 166                 % (', '.join(config_files)))
 167         sys.exit(1)
 168     # only check for fqdn and incoming dir, rest have reasonable defaults
 169     error = 0
 170     for section in config.sections():
 171         if config.get(section, 'method') == 'local':
 172             config.set(section, 'fqdn', 'localhost')
 173         if (
 174                 not config.has_option(section, 'fqdn') and
 175                 config.get(section, 'method') != 'local'):
 176             sys.stderr.write(
 177                     "Config error: %s must have a fqdn set\n" % section)
 178             error = 1
 179         if not config.has_option(section, 'incoming'):
 180             sys.stderr.write(
 181                     "Config error: %s must have an incoming directory set\n"
 182                     % section)
 183             error = 1
 184     if error:
 185         sys.exit(1)
 186
 187     return config
 188
 189
 190 hexStr = string.hexdigits
 191
 192
 193 def hexify_string(string):
 194     """ Convert a string of bytes to hexadecimal text representation. """
 195     char = ''
 196     ord_func = ord if isinstance(string, str) else int
 197     for c in string:
 198         char += hexStr[(ord_func(c) >> 4) & 0xF] + hexStr[ord_func(c) & 0xF]
 199     return char
 200
 201
 202 def checksum_test(filename, hash_name):
 203     """ Get the hex string for the hash of a file's content.
 204
 205         :param filename: Path to the file to read.
 206         :param hash_name: Name of the hash to use.
 207         :return: The computed hash value, as hexadecimal text.
 208
 209         Currently supports md5, sha1. ripemd may come in the future.
 210
 211         """
 212     try:
 213         file_to_test = open(filename, 'rb')
 214     except IOError:
 215         sys.stdout.write("Can't open %s\n" % filename)
 216         sys.exit(1)
 217
 218     if hash_name == 'md5':
 219         hash_type = md5
 220     else:
 221         hash_type = sha1
 222
 223     check_obj = hash_type()
 224
 225     while 1:
 226         data = file_to_test.read(65536)
 227         if len(data) == 0:
 228             break
 229         check_obj.update(data)
 230
 231     file_to_test.close()
 232     checksum = hexify_string(check_obj.digest())
 233
 234     return checksum
 235
 236
 237 def check_signature(filename):
 238     """ Verify the GnuPG signature on a file. """
 239     gnupg_path = "/usr/bin/gpg"
 240     if os.access(filename, os.R_OK):
 241         if os.access(gnupg_path, os.X_OK):
 242             gnupg_argv = [
 243                     gnupg_path,
 244                     "--status-fd", "1",
 245                     "--verify", "--batch", filename]
 246             gnupg_subprocess = subprocess.Popen(
 247                     gnupg_argv, stdout=subprocess.PIPE)
 248             gnupg_stdout = dputhelper.make_text_stream(gnupg_subprocess.stdout)
 249             gnupg_output = gnupg_stdout.read()
 250             if gnupg_output.count('[GNUPG:] GOODSIG'):
 251                 sys.stdout.write("Good signature on %s.\n" % filename)
 252             elif gnupg_output.count('[GNUPG:] BADSIG'):
 253                 sys.stdout.write("Bad signature on %s.\n" % filename)
 254                 sys.exit(1)
 255             elif gnupg_output.count('[GNUPG:] ERRSIG'):
 256                 sys.stdout.write(
 257                         "Error verifying signature on %s.\n" % filename)
 258                 sys.exit(1)
 259             elif gnupg_output.count('[GNUPG:] NODATA'):
 260                 sys.stdout.write("No signature on %s.\n" % filename)
 261                 sys.exit(1)
 262             else:
 263                 sys.stdout.write(
 264                         "Error in finding signature verification status.\n")
 265         else:
 266             sys.stdout.write(
 267                     "Can't verify signature on %s without GnuPG\n" % filename)
 268             sys.stdout.write(
 269                     "If you are still using PGP, please read this:\n"
 270                     "http://www.gnupg.org/gph/en/pgp2x.html\n")
 271             sys.exit(1)
 272     else:
 273         sys.stdout.write("Can't read %s\n" % filename)
 274         sys.exit(1)
 275
 276
 277 def check_upload_variant(changes, debug):
 278     """ Check if this is a binary_upload only or not. """
 279     binary_upload = 0
 280     if 'architecture' in changes:
 281         arch = changes['architecture']
 282         if debug:
 283             sys.stdout.write("D: Architecture: %s\n" % arch)
 284         if arch.find('source') < 0:
 285             if debug:
 286                 sys.stdout.write("D: Doing a binary upload only.\n")
 287             binary_upload = 1
 288     return binary_upload
 289
 290
 291 def verify_signature(
 292         host, changes_file, dsc_file,
 293         config, check_only, unsigned_upload, binary_upload, debug):
 294     """ Check the signature on the two files given via function call.
 295
 296         :param host: Configuration host name.
 297         :param changes_file: Filesystem path of upload control file.
 298         :param dsc_file: Filesystem path of source control file.
 299         :param config: `ConfigParser` instance for this application.
 300         :param check_only: If true, no upload is requested.
 301         :param unsigned_upload: If true, allow an unsigned upload.
 302         :param binary_upload: If true, this upload excludes source.
 303         :param debug: If true, enable debugging output.
 304         :return: ``None``.
 305
 306         """
 307     if debug:
 308         sys.stdout.write("D: .changes-File: %s\n" % changes_file)
 309         sys.stdout.write("D: .dsc-File: %s\n" % dsc_file)
 310     if ((check_only or config.getboolean(host, 'allow_unsigned_uploads') == 0)
 311             and not unsigned_upload):
 312         sys.stdout.write("Checking signature on .changes\n")
 313         check_signature(changes_file)
 314         if not binary_upload:
 315             sys.stdout.write("Checking signature on .dsc\n")
 316             check_signature(dsc_file)
 317
 318
 319 def source_check(changes, debug):
 320     """ Check if a source tarball has to be included in the package or not. """
 321     include_orig = include_tar = 0
 322     if 'version' in changes:
 323         version = changes['version']
 324         if debug:
 325             sys.stdout.write("D: Package Version: %s\n" % version)
 326         # versions with a dash in them are for non-native only
 327         if version.find('-') == -1:
 328             # debian native
 329             include_tar = 1
 330         else:
 331             if version.find(':') > 0:
 332                 if debug:
 333                     sys.stdout.write("D: Epoch found\n")
 334                 epoch, version = version.split(':', 1)
 335             pos = version.rfind('-')
 336             upstream_version = version[0:pos]
 337             debian_version = version[pos + 1:]
 338             if debug:
 339                 sys.stdout.write(
 340                         "D: Upstream Version: %s\n" % upstream_version)
 341                 sys.stdout.write("D: Debian Version: %s\n" % debian_version)
 342             if (
 343                     debian_version == '0.1' or debian_version == '1'
 344                     or debian_version == '1.1'):
 345                 include_orig = 1
 346             else:
 347                 include_tar = 1
 348     return (include_orig, include_tar)
 349
 350
 351 def verify_files(
 352         path, filename, host,
 353         config, check_only, check_version, unsigned_upload, debug):
 354     """ Run some tests on the files to verify that they are in good shape.
 355
 356         :param path: Directory path of the upload control file.
 357         :param filename: Filename of the upload control file.
 358         :param host: Configuration host name.
 359         :param config: `ConfigParser` instance for this application.
 360         :param check_only: If true, no upload is requested.
 361         :param check_version: If true, check the package version
 362             before upload.
 363         :param unsigned_upload: If true, allow an unsigned upload.
 364         :param debug: If true, enable debugging output.
 365         :return: A collection of filesystem paths of all files to upload.
 366
 367         """
 368     file_seen = include_orig_tar_gz = include_tar_gz = binary_only = 0
 369     files_to_upload = []
 370
 371     name_of_file = filename
 372
 373     change_file = os.path.join(path, name_of_file)
 374
 375     if debug:
 376         sys.stdout.write(
 377                 "D: Validating contents of changes file %s\n" % change_file)
 378     try:
 379         chg_fd = open(change_file, 'r')
 380     except IOError:
 381         sys.stdout.write("Can't open %s\n" % change_file)
 382         sys.exit(1)
 383     changes = parse_changes(chg_fd)
 384     chg_fd.close
 385
 386     # Find out if it's a binary only upload or not
 387     binary_upload = check_upload_variant(changes, debug)
 388
 389     if binary_upload:
 390         dsc_file = ''
 391     else:
 392         dsc_file = None
 393         for file in changes['files'].strip().split('\n'):
 394             # filename only
 395             filename = file.split()[4]
 396             if filename.find('.dsc') != -1:
 397                 if debug:
 398                     sys.stdout.write("D: dsc-File: %s\n" % filename)
 399                 dsc_file = os.path.join(path, filename)
 400         if not dsc_file:
 401             sys.stderr.write("Error: no dsc file found in sourceful upload\n")
 402             sys.exit(1)
 403
 404     # Run the check to verify that the package has been tested.
 405     try:
 406         if config.getboolean(host, 'check_version') == 1 or check_version:
 407             version_check(path, changes, debug)
 408     except ConfigParser.NoSectionError as e:
 409         sys.stderr.write("Error in config file:\n%s\n" % str(e))
 410         sys.exit(1)
 411
 412     # Verify the signature of the maintainer
 413     verify_signature(
 414             host, change_file, dsc_file,
 415             config, check_only, unsigned_upload, binary_upload, debug)
 416
 417     # Check the sources
 418     (include_orig_tar_gz, include_tar_gz) = source_check(changes, debug)
 419
 420     # Check md5sum and the size
 421     file_list = changes['files'].strip().split('\n')
 422     hash_name = config.get('DEFAULT', 'hash')
 423     for line in file_list:
 424         (check_sum, size, section, priority, file) = line.split()
 425         file_to_upload = os.path.join(path, file)
 426         if debug:
 427             sys.stdout.write("D: File to upload: %s\n" % file_to_upload)
 428         if checksum_test(file_to_upload, hash_name) != check_sum:
 429             if debug:
 430                 sys.stdout.write(
 431                         "D: Checksum from .changes: %s\n" % check_sum)
 432                 sys.stdout.write(
 433                         "D: Generated Checksum: %s\n" %
 434                         checksum_test(file_to_upload, hash_name))
 435             sys.stdout.write(
 436                     "Checksum doesn't match for %s\n" % file_to_upload)
 437             sys.exit(1)
 438         else:
 439             if debug:
 440                 sys.stdout.write(
 441                         "D: Checksum for %s is fine\n" % file_to_upload)
 442         if os.stat(file_to_upload)[stat.ST_SIZE] != int(size):
 443             if debug:
 444                 sys.stdout.write("D: size from .changes: %s\n" % size)
 445                 sys.stdout.write(
 446                         "D: calculated size: %s\n"
 447                         % os.stat(file_to_upload)[stat.ST_SIZE])
 448             sys.stdout.write(
 449                     "size doesn't match for %s\n" % file_to_upload)
 450
 451         files_to_upload.append(file_to_upload)
 452
 453     # Check filenames
 454     for file in files_to_upload:
 455         if file[-12:] == '.orig.tar.gz' and not include_orig_tar_gz:
 456             if debug:
 457                 sys.stdout.write("D: Filename: %s\n" % file)
 458                 sys.stdout.write("D: Suffix: %s\n\n" % file[-12:])
 459             sys.stdout.write(
 460                     "Package includes an .orig.tar.gz file although"
 461                     " the debian revision suggests\n"
 462                     "that it might not be required."
 463                     " Multiple uploads of the .orig.tar.gz may be\n"
 464                     "rejected by the upload queue management software.\n")
 465         elif (
 466                 file[-7:] == '.tar.gz' and not include_tar_gz
 467                 and not include_orig_tar_gz):
 468             if debug:
 469                 sys.stdout.write("D: Filename: %s\n" % file)
 470                 sys.stdout.write("D: Suffix: %s\n" % file[-7:])
 471             sys.stdout.write(
 472                     "Package includes a .tar.gz file although"
 473                     " the version suggests that it might\n"
 474                     "not be required."
 475                     " Multiple uploads of the .tar.gz may be rejected by the\n"
 476                     "upload queue management software.\n")
 477
 478     distribution = changes.get('distribution')
 479     allowed_distributions = config.get(host, 'allowed_distributions')
 480     if distribution and allowed_distributions:
 481         if debug:
 482             sys.stdout.write(
 483                     "D: Checking: distribution %s matches %s\n"
 484                     % (distribution, allowed_distributions))
 485         if not re.match(allowed_distributions, distribution):
 486             raise dputhelper.DputUploadFatalException(
 487                     "Error: uploading files for distribution %s to %s"
 488                     " not allowed."
 489                     % (distribution, host))
 490
 491     if debug:
 492         sys.stdout.write("D: File to upload: %s\n" % change_file)
 493     files_to_upload.append(change_file)
 494
 495     return files_to_upload
 496
 497
 498 def print_config(config, debug):
 499     """ Print the configuration and exit. """
 500     sys.stdout.write("\n")
 501     config.write(sys.stdout)
 502     sys.stdout.write("\n")
 503
 504
 505 def create_upload_file(package, host, fqdn, path, files_to_upload, debug):
 506     """ Write the log file for the upload.
 507
 508         :param package: File name of package to upload.
 509         :param host: Configuration host name.
 510         :param fqdn: Fully-qualified domain name of the remote host.
 511         :param path: Filesystem path of the upload control file.
 512         :param debug: If true, enable debugging output.
 513         :return: ``None``.
 514
 515         The upload log file is named ‘basename.hostname.upload’, where
 516         “basename” is the package file name without suffix, and
 517         “hostname” is the name of the host as specified in the
 518         configuration file.
 519
 520         For example, uploading ‘foo_1.2.3-1_xyz.deb’ to host ‘bar’
 521         will be logged to ‘foo_1.2.3-1_xyz.bar.upload’.
 522
 523         The upload log file is written to the
 524         directory containing the upload control file.
 525
 526         """
 527     # only need first part
 528     base = os.path.splitext(package)[0]
 529     logfile_name = os.path.join(path, base + '.' + host + '.upload')
 530     if debug:
 531         sys.stdout.write("D: Writing logfile: %s\n" % logfile_name)
 532     try:
 533         if os.access(logfile_name, os.R_OK):
 534             logfile_fd = open(logfile_name, 'a')
 535         else:
 536             logfile_fd = open(logfile_name, 'w')
 537     except IOError:
 538         sys.stderr.write("Could not write %s\n" % logfile_name)
 539         sys.exit(1)
 540
 541     for file in files_to_upload:
 542         entry_for_logfile = (
 543                 'Successfully uploaded ' + os.path.basename(file) +
 544                 ' to ' + fqdn + ' for ' + host + '.\n')
 545         logfile_fd.write(entry_for_logfile)
 546     logfile_fd.close()
 547
 548
 549 def run_lintian_test(changes_file):
 550     """ Run lintian on the changes file and stop if it finds errors. """
 551
 552     if os.access(changes_file, os.R_OK):
 553         if os.access("/usr/bin/lintian", os.R_OK):
 554             old_signal = signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 555             sys.stdout.write("Package is now being checked with lintian.\n")
 556             if dputhelper.check_call(
 557                     ['lintian', '-i', changes_file]
 558                     ) != dputhelper.EXIT_STATUS_SUCCESS:
 559                 sys.stdout.write(
 560                         "\n"
 561                         "Lintian says this package is not compliant"
 562                         " with the current policy.\n"
 563                         "Please check the current policy and your package.\n"
 564                         "Also see lintian documentation about overrides.\n")
 565                 sys.exit(1)
 566             else:
 567                 signal.signal(signal.SIGPIPE, old_signal)
 568                 return 0
 569         else:
 570             sys.stdout.write(
 571                     "lintian is not installed, skipping package test.\n")
 572     else:
 573         sys.stdout.write("Can't read %s\n" % changes_file)
 574         sys.exit(1)
 575
 576
 577 def guess_upload_host(path, filename, config):
 578     """ Guess the host where the package should be uploaded to.
 579
 580         :param path: Directory path of the upload control file.
 581         :param filename: Filename of the upload control file.
 582         :param config: `ConfigParser` instance for this application.
 583         :return: The hostname determined for this upload.
 584
 585         This is based on information from the upload control
 586         (‘*.changes’) file.
 587
 588         """
 589     non_us = 0
 590     distribution = ""
 591     dist_re = re.compile(r'^Distribution: (.*)')
 592
 593     name_of_file = filename
 594     changes_file = os.path.join(path, name_of_file)
 595
 596     try:
 597         changes_file_fd = open(changes_file, 'r')
 598     except IOError:
 599         sys.stdout.write("Can't open %s\n" % changes_file)
 600         sys.exit(1)
 601     lines = changes_file_fd.readlines()
 602     for line in lines:
 603         match = dist_re.search(line)
 604         if match:
 605             distribution = match.group(1)
 606
 607     # Try to guess a host based on the Distribution: field
 608     if distribution:
 609         for section in config.sections():
 610             host_dists = config.get(section, 'distributions')
 611             if not host_dists:
 612                 continue
 613             for host_dist in host_dists.split(','):
 614                 if distribution == host_dist.strip():
 615                     if debug:
 616                         sys.stdout.write(
 617                                 "D: guessing host %s"
 618                                 " based on distribution %s\n"
 619                                 % (section, host_dist))
 620                     return section
 621
 622     if len(config.get('DEFAULT', 'default_host_main')) != 0:
 623         sys.stdout.write(
 624                 "Trying to upload package to %s\n"
 625                 % config.get('DEFAULT', 'default_host_main'))
 626         return config.get('DEFAULT', 'default_host_main')
 627     else:
 628         sys.stdout.write(
 629                 "Trying to upload package to ftp-master"
 630                 " (ftp.upload.debian.org)\n")
 631         return "ftp-master"
 632
 633
 634 def dinstall_caller(filename, host, fqdn, login, incoming, debug):
 635     """ Run ‘dinstall’ for the package on the remote host.
 636
 637         :param filename: Debian package filename to install.
 638         :param host: Configuration host name.
 639         :param fqdn: Fully-qualified domain name of the remote host.
 640         :param login: Username for login to the remote host.
 641         :param incoming: Filesystem path on remote host for incoming
 642             packages.
 643         :param debug: If true, enable debugging output.
 644         :return: ``None``.
 645
 646         Run ‘dinstall’ on the remote host in test mode, and present
 647         the output to the user.
 648
 649         This is so the user can see if the package would be installed
 650         or not.
 651
 652         """
 653     command = [
 654             'ssh', '%s@%s' % (login, fqdn),
 655             'cd', '%s' % incoming,
 656             ';', 'dinstall', '-n', '%s' % filename]
 657     if debug:
 658         sys.stdout.write(
 659                 "D: Logging into %s@%s:%s\n" % (login, host, incoming))
 660         sys.stdout.write("D: dinstall -n %s\n" % filename)
 661     if dputhelper.check_call(command) != dputhelper.EXIT_STATUS_SUCCESS:
 662         sys.stdout.write(
 663                 "Error occured while trying to connect, or while"
 664                 " attempting to run dinstall.\n")
 665         sys.exit(1)
 666
 667
 668 def version_check(path, changes, debug):
 669     """ Check if the caller has installed the package also on his system.
 670
 671         This is for testing purposes before uploading it. If not, we
 672         reject the upload.
 673
 674         """
 675     files_to_check = []
 676
 677     # Get arch
 678     dpkg_proc = subprocess.Popen(
 679             'dpkg --print-architecture',
 680             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 681             shell=True, close_fds=True)
 682     dpkg_stdout = dputhelper.make_text_stream(dpkg_proc.stdout)
 683     dpkg_stderr = dputhelper.make_text_stream(dpkg_proc.stderr)
 684     dpkg_output = dpkg_stdout.read()
 685     dpkg_architecture = dpkg_output.strip()
 686     dpkg_stdout.close()
 687     dpkg_stderr_output = dpkg_stderr.read()
 688     dpkg_stderr.close()
 689     if debug and dpkg_stderr_output:
 690         sys.stdout.write(
 691                 "D: dpkg-architecture stderr output:"
 692                 " %r\n" % dpkg_stderr_output)
 693     if debug:
 694         sys.stdout.write(
 695                 "D: detected architecture: '%s'\n" % dpkg_architecture)
 696
 697     # Get filenames of deb files:
 698     for file in changes['files'].strip().split('\n'):
 699         filename = os.path.join(path, file.split()[4])
 700         if filename.endswith('.deb'):
 701             if debug:
 702                 sys.stdout.write("D: Debian Package: %s\n" % filename)
 703             dpkg_proc = subprocess.Popen(
 704                     'dpkg --field %s' % filename,
 705                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 706                     shell=True, close_fds=True)
 707             dpkg_stdout = dputhelper.make_text_stream(dpkg_proc.stdout)
 708             dpkg_stderr = dputhelper.make_text_stream(dpkg_proc.stderr)
 709             dpkg_output = dpkg_stdout.read()
 710             dpkg_stdout.close()
 711             dpkg_fields = email.parser.HeaderParser().parsestr(dpkg_output)
 712             dpkg_stderr_output = dpkg_stderr.read()
 713             dpkg_stderr.close()
 714             if debug and dpkg_stderr_output:
 715                 sys.stdout.write(
 716                         "D: dpkg stderr output:"
 717                         " %r\n" % dpkg_stderr_output)
 718             if (
 719                     dpkg_architecture
 720                     and dpkg_fields['architecture'] not in [
 721                         'all', dpkg_architecture]):
 722                 if debug:
 723                     sys.stdout.write(
 724                             "D: not install-checking %s due to arch mismatch\n"
 725                             % filename)
 726             else:
 727                 package_name = dpkg_fields['package']
 728                 version_number = dpkg_fields['version']
 729                 if debug:
 730                     sys.stdout.write(
 731                             "D: Package to Check: %s\n" % package_name)
 732                 if debug:
 733                     sys.stdout.write(
 734                             "D: Version to Check: %s\n" % version_number)
 735                 files_to_check.append((package_name, version_number))
 736
 737     for file, version_to_check in files_to_check:
 738         if debug:
 739             sys.stdout.write("D: Name of Package: %s\n" % file)
 740         dpkg_proc = subprocess.Popen(
 741                 'dpkg -s %s' % file,
 742                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 743                 shell=True, close_fds=True)
 744         dpkg_stdout = dputhelper.make_text_stream(dpkg_proc.stdout)
 745         dpkg_stderr = dputhelper.make_text_stream(dpkg_proc.stderr)
 746         dpkg_output = dpkg_stdout.read()
 747         dpkg_stdout.close()
 748         dpkg_fields = email.parser.HeaderParser().parsestr(dpkg_output)
 749         dpkg_stderr_output = dpkg_stderr.read()
 750         dpkg_stderr.close()
 751         if debug and dpkg_stderr_output:
 752             sys.stdout.write(
 753                     "D: dpkg stderr output:"
 754                     " %r\n" % dpkg_stderr_output)
 755         if 'version' in dpkg_fields:
 756             installed_version = dpkg_fields['version']
 757             if debug:
 758                 sys.stdout.write(
 759                         "D: Installed-Version: %s\n" % installed_version)
 760             if debug:
 761                 sys.stdout.write(
 762                         "D: Check-Version: %s\n" % version_to_check)
 763             if installed_version != version_to_check:
 764                 sys.stdout.write(
 765                         "Package to upload is not installed, but it appears"
 766                         " you have an older version installed.\n")
 767         else:
 768             sys.stdout.write(
 769                     "Uninstalled Package. Test it before uploading it.\n")
 770             sys.exit(1)
 771
 772
 773 def execute_command(command, position, debug=False):
 774     """ Run a command that the user-defined in the config_file.
 775
 776         :param command: Command line to execute.
 777         :param position: Position of the command: 'pre' or 'post'.
 778         :param debug: If true, enable debugging output.
 779         :return: ``None``.
 780
 781         """
 782     if debug:
 783         sys.stdout.write("D: Command: %s\n" % command)
 784     if subprocess.call(command, shell=True):
 785         raise dputhelper.DputUploadFatalException(
 786                 "Error: %s upload command failed." % position)
 787
 788
 789 def check_upload_logfile(
 790         changes_file, host, fqdn,
 791         check_only, call_lintian, force_upload, debug):
 792     """ Check if the user already put this package on the specified host.
 793
 794         :param changes_file: Filesystem path of upload control file.
 795         :param host: Configuration host name.
 796         :param fqdn: Fully-qualified domain name of the remote host.
 797         :param check_only: If true, no upload is requested.
 798         :param call_lintian: If true, a Lintian invocation is requested.
 799         :param force_upload: If true, don't check the upload log file.
 800         :param debug: If true, enable debugging output.
 801         :return: ``None``.
 802
 803         """
 804     uploaded = 0
 805     upload_logfile = changes_file[:-8] + '.' + host + '.upload'
 806     if not check_only and not force_upload:
 807         if not os.path.exists(upload_logfile):
 808             return
 809         try:
 810             fd_logfile = open(upload_logfile)
 811         except IOError:
 812             sys.stdout.write("Couldn't open %s\n" % upload_logfile)
 813             sys.exit(1)
 814         for line in fd_logfile.readlines():
 815             if line.find(fqdn) != -1:
 816                 uploaded = 1
 817         if uploaded:
 818             sys.stdout.write(
 819                     "Package has already been uploaded to %s on %s\n"
 820                     % (host, fqdn))
 821             sys.stdout.write("Nothing more to do for %s\n" % changes_file)
 822             sys.exit(0)
 823
 824
 825 def make_usage_message():
 826     """ Make the program usage help message. """
 827     text = textwrap.dedent("""\
 828         Usage: dput [options] [host] <package(s).changes>
 829          Supported options (see man page for long forms):
 830            -c: Config file to parse.
 831            -d: Enable debug messages.
 832            -D: Run dinstall after upload.
 833            -e: Upload to a delayed queue. Takes an argument from 0 to 15.
 834            -f: Force an upload.
 835            -h: Display this help message.
 836            -H: Display a list of hosts from the config file.
 837            -l: Run lintian before upload.
 838            -U: Do not write a .upload file after uploading.
 839            -o: Only check the package.
 840            -p: Print the configuration.
 841            -P: Use passive mode for ftp uploads.
 842            -s: Simulate the upload only.
 843            -u: Don't check GnuPG signature.
 844            -v: Display version information.
 845            -V: Check the package version and then upload it.
 846         """)
 847     return text
 848
 849
 850 def main():
 851     """ Main function, no further comment needed. :) """
 852
 853     global debug
 854
 855     check_version = config_print = force_upload = 0
 856     call_lintian = no_upload_log = config_host_list = 0
 857     ftp_passive_mode = 0
 858     preferred_host = ''
 859     config_file = ''
 860     dinstall = False
 861     check_only = False
 862     unsigned_upload = False
 863     delay_upload = None
 864     simulate = False
 865
 866     progname = dputhelper.get_progname()
 867     version = dputhelper.get_distribution_version()
 868
 869     # Parse Command Line Options.
 870     (opts, args) = dputhelper.getopt(
 871             sys.argv[1:],
 872             'c:dDe:fhHlUopPsuvV', [
 873                 'debug', 'dinstall', 'check-only',
 874                 'check-version', 'config=', 'force', 'help',
 875                 'host-list', 'lintian', 'no-upload-log',
 876                 'passive', 'print', 'simulate', 'unchecked',
 877                 'delayed=', 'version'])
 878     for option, arg in opts:
 879         if option in ('-h', '--help'):
 880             sys.stdout.write(make_usage_message())
 881             return
 882         elif option in ('-v', '--version'):
 883             sys.stdout.write("{progname} {version}\n".format(
 884                     progname=progname, version=version))
 885             return
 886         elif option in ('-d', '--debug'):
 887             debug = 1
 888         elif option in ('-D', '--dinstall'):
 889             dinstall = True
 890         elif option in ('-c', '--config'):
 891             config_file = arg
 892         elif option in ('-f', '--force'):
 893             force_upload = 1
 894         elif option in ('-H', '--host-list'):
 895             config_host_list = 1
 896         elif option in ('-l', '--lintian'):
 897             call_lintian = 1
 898         elif option in ('-U', '--no-upload-log'):
 899             no_upload_log = 1
 900         elif option in ('-o', '--check-only'):
 901             check_only = True
 902         elif option in ('-p', '--print'):
 903             config_print = 1
 904         elif option in ('-P', '--passive'):
 905             ftp_passive_mode = 1
 906         elif option in ('-s', '--simulate'):
 907             simulate = True
 908         elif option in ('-u', '--unchecked'):
 909             unsigned_upload = True
 910         elif option in ('-e', '--delayed'):
 911             if arg in map(str, range(16)):
 912                 delay_upload = arg
 913             else:
 914                 sys.stdout.write(
 915                         "Incorrect delayed argument,"
 916                         " dput only understands 0 to 15.\n")
 917                 sys.exit(1)
 918         elif option in ('-V', '--check_version'):
 919             check_version = 1
 920
 921     # Always print the version number in the debug output
 922     # so that in case of bugreports, we know which version
 923     # the user has installed
 924     if debug:
 925         sys.stdout.write(
 926                 "D: {progname} {version}\n".format(
 927                     progname=progname, version=version))
 928
 929     # Try to get the login from the enviroment
 930     if 'USER' in os.environ:
 931         login = os.environ['USER']
 932         if debug:
 933             sys.stdout.write("D: Login: %s\n" % login)
 934     else:
 935         sys.stdout.write("$USER not set, will use login information.\n")
 936         # Else use the current username
 937         login = pwd.getpwuid(os.getuid())[0]
 938         if debug:
 939             sys.stdout.write("D: User-ID: %s\n" % os.getuid())
 940             sys.stdout.write("D: Login: %s\n" % login)
 941
 942     # Start Config File Parsing.
 943     config = read_configs(config_file, debug)
 944
 945     if config_print:
 946         print_config(config, debug)
 947         sys.exit(0)
 948
 949     if config_host_list:
 950         sys.stdout.write(
 951                 "\n"
 952                 "Default Method: %s\n"
 953                 "\n" % config.get('DEFAULT', 'method'))
 954         for section in config.sections():
 955             distributions = ""
 956             if config.get(section, 'distributions'):
 957                 distributions = (
 958                         ", distributions: %s" %
 959                         config.get(section, 'distributions'))
 960             sys.stdout.write(
 961                     "%s => %s  (Upload method: %s%s)\n" % (
 962                         section,
 963                         config.get(section, 'fqdn'),
 964                         config.get(section, 'method'),
 965                         distributions))
 966         sys.stdout.write("\n")
 967         sys.exit(0)
 968
 969     # Process further command line options.
 970     if len(args) == 0:
 971         sys.stdout.write(
 972                 "No package or host has been provided, see dput -h\n")
 973         sys.exit(0)
 974     elif len(args) == 1 and not check_only:
 975         package_to_upload = args[0:]
 976     else:
 977         if not check_only:
 978             if debug:
 979                 sys.stdout.write(
 980                         "D: Checking if a host was named"
 981                         " on the command line.\n")
 982             if config.has_section(args[0]):
 983                 if debug:
 984                     sys.stdout.write("D: Host %s found in config\n" % args[0])
 985                 # Host was also named, so only the rest will be a list
 986                 # of packages to upload.
 987                 preferred_host = args[0]
 988                 package_to_upload = args[1:]
 989             elif (
 990                     not config.has_section(args[0])
 991                     and not args[0].endswith('.changes')):
 992                 sys.stderr.write("No host %s found in config\n" % args[0])
 993                 if args[0] == 'gluck_delayed':
 994                     sys.stderr.write("""
 995     The delayed upload queue has been moved back to
 996     ftp-master (aka ftp.upload.debian.org).
 997     """)
 998                 sys.exit(1)
 999             else:
1000                 if debug:
1001                     sys.stdout.write("D: No host named on command line.\n")
1002                 # Only packages have been named on the command line.
1003                 preferred_host = ''
1004                 package_to_upload = args[0:]
1005         else:
1006             if debug:
1007                 sys.stdout.write("D: Checking for the package name.\n")
1008             if config.has_section(args[0]):
1009                 sys.stdout.write("D: Host %s found in config.\n" % args[0])
1010                 preferred_host = args[0]
1011                 package_to_upload = args[1:]
1012             elif not config.has_section(args[0]):
1013                 sys.stdout.write("D: No host %s found in config\n" % args[0])
1014                 package_to_upload = args[0:]
1015
1016     upload_methods = import_upload_functions()
1017
1018     # Run the same checks for all packages that have been given on
1019     # the command line
1020     for package_name in package_to_upload:
1021         # Check that a .changes file was given on the command line
1022         # and no matching .upload file exists.
1023         if package_name[-8:] != '.changes':
1024             sys.stdout.write(
1025                     "Not a .changes file.\n"
1026                     "Please select a .changes file to upload.\n")
1027             sys.stdout.write("Tried to upload: %s\n" % package_name)
1028             sys.exit(1)
1029
1030         # Construct the package name for further usage.
1031         path, name_of_package = os.path.split(package_name)
1032         if path == '':
1033             path = os.getcwd()
1034
1035         # Define the host to upload to.
1036         if preferred_host == '':
1037             host = guess_upload_host(path, name_of_package, config)
1038         else:
1039             host = preferred_host
1040         if config.get(host, 'method') == 'local':
1041             fqdn = 'localhost'
1042         else:
1043             fqdn = config.get(host, 'fqdn')
1044
1045         # Check if we already did this upload or not
1046         check_upload_logfile(
1047                 package_name, host, fqdn,
1048                 check_only, call_lintian, force_upload, debug)
1049
1050         # Run the change file tests.
1051         files_to_upload = verify_files(
1052                 path, name_of_package, host,
1053                 config, check_only, check_version, unsigned_upload, debug)
1054
1055         # Run the lintian test if the user asked us to do so.
1056         if (
1057                 call_lintian or
1058                 config.getboolean(host, 'run_lintian') == 1):
1059             run_lintian_test(os.path.join(path, name_of_package))
1060         elif check_only:
1061             sys.stdout.write(
1062                     "Warning: The option -o does not automatically include \n"
1063                     "a lintian run any more. Please use the option -ol if \n"
1064                     "you want to include running lintian in your checking.\n")
1065
1066         # don't upload, skip to the next item
1067         if check_only:
1068             sys.stdout.write("Package checked by dput.\n")
1069             continue
1070
1071         # Pre-Upload Commands
1072         if len(config.get(host, 'pre_upload_command')) != 0:
1073             position = 'pre'
1074             command = config.get(host, 'pre_upload_command')
1075             execute_command(command, position, debug)
1076
1077         # Check the upload methods that we have as default and per host
1078         if debug:
1079             sys.stdout.write(
1080                     "D: Default Method: %s\n"
1081                     % config.get('DEFAULT', 'method'))
1082         if config.get('DEFAULT', 'method') not in upload_methods:
1083             sys.stdout.write(
1084                     "Unknown upload method: %s\n"
1085                     % config.get('DEFAULT', 'method'))
1086             sys.exit(1)
1087         if debug:
1088             sys.stdout.write(
1089                     "D: Host Method: %s\n" % config.get(host, 'method'))
1090         if config.get(host, 'method') not in upload_methods:
1091             sys.stdout.write(
1092                     "Unknown upload method: %s\n"
1093                     % config.get(host, 'method'))
1094             sys.exit(1)
1095
1096         # Inspect the Config and set appropriate upload method
1097         if not config.get(host, 'method'):
1098             method = config.get('DEFAULT', 'method')
1099         else:
1100             method = config.get(host, 'method')
1101
1102         # Check now the login and redefine it if needed
1103         if (
1104                 len(config.get(host, 'login')) != 0 and
1105                 config.get(host, 'login') != 'username'):
1106             login = config.get(host, 'login')
1107             if debug:
1108                 sys.stdout.write(
1109                         "D: Login %s from section %s used\n" % (login, host))
1110         elif (
1111                 len(config.get('DEFAULT', 'login')) != 0 and
1112                 config.get('DEFAULT', 'login') != 'username'):
1113             login = config.get('DEFAULT', 'login')
1114             if debug:
1115                 sys.stdout.write("D: Default login %s used\n" % login)
1116         else:
1117             if debug:
1118                 sys.stdout.write(
1119                         "D: Neither host %s nor default login used. Using %s\n"
1120                         % (host, login))
1121
1122         incoming = config.get(host, 'incoming')
1123
1124         # if delay_upload wasn't passed via -e/--delayed
1125         if delay_upload is None:
1126             delay_upload = config.get(host, 'delayed')
1127             if not delay_upload:
1128                 delay_upload = config.get('DEFAULT', 'delayed')
1129
1130         if delay_upload:
1131             if int(delay_upload) == 0:
1132                 sys.stdout.write("Uploading to DELAYED/0-day.\n")
1133             if incoming[-1] == '/':
1134                 first_char = ''
1135             else:
1136                 first_char = '/'
1137             incoming += first_char + 'DELAYED/' + delay_upload + '-day'
1138             delayed = ' [DELAYED/' + delay_upload + ']'
1139         else:
1140             delayed = ''
1141
1142         # Do the actual upload
1143         if not simulate:
1144             sys.stdout.write(
1145                     "Uploading to %s%s (via %s to %s):\n"
1146                     % (host, delayed, method, fqdn))
1147             if debug:
1148                 sys.stdout.write("D: FQDN: %s\n" % fqdn)
1149                 sys.stdout.write("D: Login: %s\n" % login)
1150                 sys.stdout.write("D: Incoming: %s\n" % incoming)
1151             progress = config.getint(host, 'progress_indicator')
1152             if not os.isatty(1):
1153                 progress = 0
1154             if method == 'ftp':
1155                 if ':' in fqdn:
1156                     fqdn, port = fqdn.rsplit(":", 1)
1157                 else:
1158                     port = 21
1159                 ftp_mode = config.getboolean(host, 'passive_ftp')
1160                 if ftp_passive_mode == 1:
1161                     ftp_mode = 1
1162                 if debug:
1163                     sys.stdout.write("D: FTP port: %s\n" % port)
1164                     if ftp_mode == 1:
1165                         sys.stdout.write("D: Using passive ftp\n")
1166                     else:
1167                         sys.stdout.write("D: Using active ftp\n")
1168                 upload_methods[method](
1169                         fqdn, login, incoming,
1170                         files_to_upload, debug, ftp_mode,
1171                         progress=progress, port=port)
1172             elif method == 'scp':
1173                 if debug and config.getboolean(host, 'scp_compress'):
1174                     sys.stdout.write("D: Setting compression for scp\n")
1175                 scp_compress = config.getboolean(host, 'scp_compress')
1176                 ssh_config_options = [
1177                         y for y in (
1178                             x.strip() for x in
1179                             config.get(host, 'ssh_config_options').split('\n'))
1180                         if y]
1181                 if debug:
1182                     sys.stdout.write(
1183                             "D: ssh config options:"
1184                             "\n  "
1185                             + "\n  ".join(ssh_config_options)
1186                             + "\n")
1187                 upload_methods[method](
1188                         fqdn, login, incoming,
1189                         files_to_upload, debug, scp_compress,
1190                         ssh_config_options)
1191             else:
1192                 upload_methods[method](
1193                         fqdn, login, incoming,
1194                         files_to_upload, debug, 0, progress=progress)
1195         # Or just simulate it.
1196         else:
1197             for file in files_to_upload:
1198                 sys.stdout.write(
1199                         "Uploading with %s: %s to %s:%s\n"
1200                         % (method, file, fqdn, incoming))
1201
1202         # Create the logfile after the package has
1203         # been put into the archive.
1204         if not simulate:
1205             if not no_upload_log:
1206                 create_upload_file(
1207                         name_of_package, host, fqdn, path,
1208                         files_to_upload, debug)
1209             sys.stdout.write("Successfully uploaded packages.\n")
1210         else:
1211             sys.stdout.write("Simulated upload.\n")
1212
1213         # Run dinstall if the user asked us to do so.
1214         if debug:
1215             sys.stdout.write("D: dinstall: %s\n" % dinstall)
1216             sys.stdout.write(
1217                     "D: Host Config: %s\n"
1218                     % config.getboolean(host, 'run_dinstall'))
1219         if config.getboolean(host, 'run_dinstall') == 1 or dinstall:
1220             if not simulate:
1221                 dinstall_caller(
1222                         name_of_package, host, fqdn, login, incoming, debug)
1223             else:
1224                 sys.stdout.write("Will run dinstall now.\n")
1225
1226         # Post-Upload Command
1227         if len(config.get(host, 'post_upload_command')) != 0:
1228             position = 'post'
1229             command = config.get(host, 'post_upload_command')
1230             execute_command(command, position, debug)
1231
1232     return
1233
1234
1235 if __name__ == '__main__':
1236     try:
1237         main()
1238     except KeyboardInterrupt:
1239         sys.stdout.write("Exiting due to user interrupt.\n")
1240         sys.exit(1)
1241     except dputhelper.DputException as e:
1242         sys.stderr.write("%s\n" % e)
1243         sys.exit(1)
1244
1245 \f
1246 # Local variables:
1247 # coding: utf-8
1248 # mode: python
1249 # End:
1250 # vim: fileencoding=utf-8 filetype=python :