bin/get-bugzilla-attachments-by-mimetype

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10
  11 # This digs through a pile of bugzilla's and populates the cwd with a big
  12 # collection of bug-docs in per-filetype dirs with bug-ids as names with
  13 # prefixes to indicate which bug-tracker, e.g.
  14 #
  15 # fdo-bugid-X.suffix
  16 # rhbz-bugid-X.suffix
  17 # moz-bugid-X.suffix
  18 #
  19 # where X is the n'th attachment of that type in the bug
  20 #
  21 # The results are stored in the current directory, categorized by the
  22 # extension of the downloaded file.  When a file already exists, it is assumed
  23 # it is already downloaded by a previous run, and up-to-date.
  24
  25 from __future__ import print_function
  26 import feedparser
  27 import base64
  28 import datetime
  29 import glob
  30 import re
  31 import os, os.path
  32 import stat
  33 import sys
  34 import threading
  35 try:
  36     import queue
  37 except:
  38     import Queue as queue
  39 try:
  40     from urllib.request import urlopen
  41 except:
  42     from urllib import urlopen
  43 try:
  44     import xmlrpc.client as xmlrpclib
  45 except:
  46     import xmlrpclib
  47 from xml.dom import minidom
  48 from xml.sax.saxutils import escape
  49
  50 def urlopen_retry(url):
  51     maxretries = 3
  52     for i in range(maxretries + 1):
  53         try:
  54             return urlopen(url)
  55         except IOError as e:
  56             print("caught IOError: " + str(e))
  57             if maxretries == i:
  58                 raise
  59             print("retrying...")
  60
  61 def get_from_bug_url_via_xml(url, mimetype, prefix, suffix):
  62     id = url.rsplit('=', 2)[1]
  63     print("id is " + prefix + id + " " + suffix)
  64     print("parsing " + id)
  65     sock = urlopen_retry(url+"&ctype=xml")
  66     dom = minidom.parse(sock)
  67     sock.close()
  68     attachmentid=0
  69     for attachment in dom.getElementsByTagName('attachment'):
  70         attachmentid += 1
  71         print(" mimetype is", end=' ')
  72         for node in attachment.childNodes:
  73             if node.nodeName == 'type':
  74                 # check if attachment is deleted
  75                 if not node.firstChild:
  76                     print('deleted attachment, skipping')
  77                     continue
  78
  79                 print(node.firstChild.nodeValue, end=' ')
  80                 if node.firstChild.nodeValue.lower() != mimetype.lower():
  81                     print('skipping')
  82                     break
  83             elif node.nodeName == 'data':
  84                 # check if attachment is deleted (i.e. https://bugs.kde.org/show_bug.cgi?id=53343&ctype=xml)
  85                 if not node.firstChild:
  86                     print('deleted attachment, skipping')
  87                     continue
  88
  89                 download = suffix + '/' +prefix + id + '-' + str(attachmentid) + '.' + suffix
  90                 if os.path.isfile(download):
  91                     print("assuming " + download + " is up to date")
  92                     continue
  93
  94                 # prevent re-downloading FDO attachments from TDF
  95                 if prefix == "tdf" and int(id) < 88776:
  96                     fdodownload = download.replace("tdf", "fdo")
  97                     if os.path.isfile(fdodownload):
  98                         print("assuming FDO " + fdodownload + " is up to date")
  99                         continue
 100
 101                 print('downloading as ' + download)
 102                 tmpfile = download + ".tmp"
 103                 f = open(tmpfile, 'wb')
 104                 f.write(base64.b64decode(node.firstChild.nodeValue))
 105                 f.close()
 106                 os.rename(tmpfile, download)
 107                 break
 108
 109 def get_novell_bug_via_xml(url, mimetype, prefix, suffix):
 110     id = url.rsplit('=', 2)[1]
 111     print("id is " + prefix + id + " " + suffix)
 112     print("parsing " + id)
 113     sock = urlopen_retry(url+"&ctype=xml")
 114     dom = minidom.parse(sock)
 115     sock.close()
 116     attachmentid=0
 117     for comment in dom.getElementsByTagName('thetext'):
 118         commentText = comment.firstChild.nodeValue
 119         match = re.search(r".*Created an attachment \(id=([0-9]+)\)", commentText)
 120         if not match:
 121             continue
 122
 123         attachmentid += 1
 124
 125         download = suffix + '/' + prefix + id + '-' + str(attachmentid) + '.' + suffix
 126         if os.path.isfile(download):
 127             print("assuming " + download + " is up to date")
 128             continue
 129
 130         realAttachmentId = match.group(1)
 131         handle = urlopen_retry(novellattach + realAttachmentId)
 132         if not handle:
 133             print("attachment %s is not accessible" % realAttachmentId)
 134             continue
 135         print(" mimetype is", end=' ')
 136
 137         info = handle.info()
 138         if info.get_content_type:
 139             remoteMime = info.get_content_type()
 140         else:
 141             remoteMime = info.gettype()
 142         print(remoteMime, end=' ')
 143         if remoteMime != mimetype:
 144             print("skipping")
 145             continue
 146
 147         print('downloading as ' + download)
 148         tmpfile = download + ".tmp"
 149         f = open(tmpfile, 'wb')
 150         f.write(handle.read())
 151         f.close()
 152         os.rename(tmpfile, download)
 153
 154 def create_query(mimetype):
 155     query = dict()
 156     query['query_format']='advanced'
 157     query['field0-0-0']='attachments.mimetype'
 158     query['type0-0-0']='equals'
 159     query['value0-0-0']=mimetype
 160     return query
 161
 162 def get_downloaded_files(prefix, suffix):
 163     return glob.glob(os.path.join(suffix, '%s*.%s' % (prefix, suffix)))
 164
 165 def get_file_bz_ids(files, prefix):
 166     return set([os.path.basename(f).split('-')[0].replace(prefix, '', 1) for f in files])
 167
 168 def get_changed_date(files):
 169     newest = max([os.stat(f)[stat.ST_MTIME] for f in files])
 170     # Subtract a day to avoid timezone differences. The worst thing that
 171     # can happen is that we are going to process more bugs than necessary.
 172     return datetime.date.fromtimestamp(newest - 24 * 60 * 60)
 173
 174 def get_through_rpc_query(rpcurl, showurl, mimetype, prefix, suffix):
 175     try:
 176         os.mkdir(suffix)
 177     except:
 178         pass
 179
 180     def process(query, full, have=[]):
 181         try:
 182             proxy = xmlrpclib.ServerProxy(rpcurl)
 183             result = proxy.Bug.search(query)
 184             bugs = result['bugs']
 185             print(str(len(bugs)) + ' bugs to process')
 186
 187             if full:
 188                 available = set([str(bug['id']) for bug in bugs])
 189                 # we already have files from all available bugs
 190                 if available.difference(set(have)) == set():
 191                     print("assuming all downloaded files are up to date")
 192                     return
 193
 194             for bug in bugs:
 195                 url = showurl + str(bug['id'])
 196                 get_from_bug_url_via_xml(url, mimetype, prefix, suffix)
 197         except xmlrpclib.Fault as err:
 198             print("A fault occurred")
 199             print("Fault code: %s" % err.faultCode)
 200             print(err.faultString)
 201
 202     query = create_query(mimetype)
 203     query['column_list']='bug_id'
 204
 205     files = get_downloaded_files(prefix, suffix)
 206
 207     if files != []:
 208         print('looking for updated bugs having %s attachment(s)' % mimetype)
 209         query_changed = query.copy()
 210         query_changed['field0-1-0'] = 'days_elapsed'
 211         query_changed['type0-1-0'] = 'lessthaneq'
 212         query_changed['value0-1-0'] = str((datetime.date.today() - get_changed_date(files)).days)
 213         process(query_changed, False)
 214
 215     print('looking for all bugs having %s attachment(s)' % mimetype)
 216     process(query, True, get_file_bz_ids(files, prefix))
 217
 218 def get_through_rss_query(queryurl, mimetype, prefix, suffix):
 219     try:
 220         os.mkdir(suffix)
 221     except:
 222         pass
 223
 224     #Getting detailed bug information and downloading an attachment body is not possible without logging in to Novell bugzilla
 225     #get_novell_bug_via_xml function is a workaround for that situation
 226     get_bug_function = get_novell_bug_via_xml if prefix == "novell" else get_from_bug_url_via_xml
 227
 228     def process(query, full, have=[]):
 229         url = queryurl + '?' + '&'.join(['='.join(kv) for kv in query.items()])
 230         print('url is ' + url)
 231         d = feedparser.parse(url)
 232         print(str(len(d['entries'])) + ' bugs to process')
 233
 234         entries = []
 235         for entry in d['entries']:
 236             bugid = entry['id'].split('=')[-1]
 237             entries.append(entry)
 238
 239         if full:
 240             available = set([str(entry['id'].split('=')[-1]) for entry in entries])
 241             # we already have files from all available bugs
 242             if available.difference(set(have)) == set():
 243                 print("assuming all downloaded files are up to date")
 244                 return
 245
 246         for entry in entries:
 247             try:
 248                 get_bug_function(entry['id'], mimetype, prefix, suffix)
 249             except KeyboardInterrupt:
 250                 raise # Ctrl+C should work
 251             except:
 252                 print(entry['id'] + " failed: " + str(sys.exc_info()[0]))
 253                 pass
 254
 255     query = create_query(escape(mimetype.replace("+","%2B")))
 256     query['ctype'] = 'rss'
 257
 258     files = get_downloaded_files(prefix, suffix)
 259
 260     if files != []:
 261         print('looking for updated bugs having %s attachment(s)' % mimetype)
 262         query_changed = query.copy()
 263         query_changed['field0-1-0'] = 'delta_ts'
 264         query_changed['type0-1-0'] = 'greaterthaneq'
 265         query_changed['value0-1-0'] = get_changed_date(files).isoformat()
 266         process(query_changed, False)
 267
 268     print('looking for all bugs having %s attachment(s)' % mimetype)
 269     process(query, True, get_file_bz_ids(files, prefix))
 270
 271 #since searching bugs having attachments with specific mimetypes is not available in launchpad API
 272 #we're iterating over all bugs of the most interesting source packages
 273 launchpad_pkgs = (
 274     "abiword",
 275     "calibre",
 276     "calligra",
 277     "gnumeric",
 278     "inkscape",
 279     "koffice",
 280     "libabw",
 281     "libcdr",
 282     "libe-book",
 283     "libetonyek",
 284     "libfreehand",
 285     "libmspub",
 286     "libmwaw",
 287     "liborcus",
 288     "libpagemaker",
 289     "libreoffice",
 290     "libvisio",
 291     "libwpd",
 292     "libwpg",
 293     "libwps",
 294     "openoffice.org",
 295     "python-uniconvertor",
 296     "scribus",
 297     "sk1",
 298     "unoconv",
 299 )
 300
 301 def get_launchpad_bugs(prefix):
 302     #launchpadlib python module is required to download launchpad attachments
 303     from launchpadlib.launchpad import Launchpad
 304
 305     launchpad = Launchpad.login_anonymously("attachmentdownload", "production")
 306     ubuntu = launchpad.distributions["ubuntu"]
 307
 308     for pkg in launchpad_pkgs:
 309         srcpkg = ubuntu.getSourcePackage(name=pkg)
 310         pkgbugs = srcpkg.searchTasks(status=["New", "Fix Committed", "Invalid", "Won't Fix", "Confirmed", "Triaged", "In Progress", "Incomplete", "Incomplete (with response)", "Incomplete (without response)", "Fix Released", "Opinion", "Expired"])
 311
 312         for bugtask in pkgbugs:
 313             bug = bugtask.bug
 314             id = str(bug.id)
 315             print("parsing " + id + " status: " + bugtask.status + " title: " + bug.title[:50])
 316             attachmentid = 0
 317             for attachment in bug.attachments:
 318                 attachmentid += 1
 319                 handle = attachment.data.open()
 320                 if not handle.content_type in mimetypes:
 321                     #print "skipping"
 322                     continue
 323
 324                 suffix = mimetypes[handle.content_type]
 325                 if not os.path.isdir(suffix):
 326                     try:
 327                         os.mkdir(suffix)
 328                     except:
 329                         pass
 330
 331                 download = suffix + '/' + prefix + id + '-' + str(attachmentid) + '.' + suffix
 332
 333                 if os.path.isfile(download):
 334                     print("assuming " + id + " is up to date")
 335                     break
 336
 337                 print('mimetype is ' + handle.content_type + ' downloading as ' + download)
 338
 339                 tmpfile = download + ".tmp"
 340                 f = open(tmpfile, "wb")
 341                 f.write(handle.read())
 342                 f.close()
 343                 os.rename(tmpfile, download)
 344
 345 rss_bugzillas = (
 346     ( 'abi', 'http://bugzilla.abisource.com/buglist.cgi' ), #added for abiword
 347     ( 'fdo', 'http://bugs.freedesktop.org/buglist.cgi' ),
 348     ( 'gentoo', 'http://bugs.gentoo.org/buglist.cgi' ),
 349     ( 'gnome', 'http://bugzilla.gnome.org/buglist.cgi' ), # added for gnumeric
 350     ( 'kde', 'http://bugs.kde.org/buglist.cgi' ), # added for koffice/calligra
 351     ( 'mandriva', 'https://qa.mandriva.com/buglist.cgi' ),
 352     ( 'moz', 'https://bugzilla.mozilla.org/buglist.cgi' ),
 353     # It seems something has changed and it is no longer possible to
 354     # download any files from there.
 355     # NOTE: I am leaving it in the list, commented out, just so someone
 356     # does not add it back immediately .-)
 357     # 'novell': 'https://bugzilla.novell.com/buglist.cgi',
 358 # note: running this script against bz.apache.org apparently causes one's IP
 359 # to be banned or something; you won't get new files in any case...
 360 #    ( 'ooo', 'https://bz.apache.org/ooo/buglist.cgi' ),
 361     ( 'tdf', 'http://bugs.documentfoundation.org/buglist.cgi' ),
 362 )
 363
 364 redhatrpc = 'https://bugzilla.redhat.com/xmlrpc.cgi'
 365 redhatbug = 'https://bugzilla.redhat.com/show_bug.cgi?id='
 366
 367 #Novell Bugzilla requires users to log in, in order to get details of the bugs such as attachment bodies etc.
 368 #As a dirty workaround, we parse comments containing "Created an attachment (id=xxxxxx)" and download attachments manually
 369 #python-bugzilla claims that it supports Novell bugzilla login but it's not working right now and novell bugzilla login
 370 #system is a nightmare
 371 novellattach = 'https://bugzilla.novell.com/attachment.cgi?id='
 372
 373 mimetypes = {
 374 # ODF
 375     'application/vnd.oasis.opendocument.base': 'odb',
 376     'application/vnd.oasis.opendocument.database': 'odb',
 377     'application/vnd.oasis.opendocument.chart': 'odc',
 378     'application/vnd.oasis.opendocument.chart-template': 'otc',
 379     'application/vnd.oasis.opendocument.formula': 'odf',
 380     'application/vnd.oasis.opendocument.formula-template': 'otf',
 381     'application/vnd.oasis.opendocument.graphics': 'odg',
 382     'application/vnd.oasis.opendocument.graphics-template': 'otg',
 383     'application/vnd.oasis.opendocument.graphics-flat-xml': 'fodg',
 384     'application/vnd.oasis.opendocument.presentation': 'odp',
 385     'application/vnd.oasis.opendocument.presentation-template': 'otp',
 386     'application/vnd.oasis.opendocument.presentation-flat-xml': 'fodp',
 387     'application/vnd.oasis.opendocument.spreadsheet': 'ods',
 388     'application/vnd.oasis.opendocument.spreadsheet-template': 'ots',
 389     'application/vnd.oasis.opendocument.spreadsheet-flat-xml': 'fods',
 390     'application/vnd.oasis.opendocument.text': 'odt',
 391     'application/vnd.oasis.opendocument.text-flat-xml': 'fodt',
 392     'application/vnd.oasis.opendocument.text-master': 'odm',
 393     'application/vnd.oasis.opendocument.text-template': 'ott',
 394     'application/vnd.oasis.opendocument.text-master-template': 'otm',
 395     'application/vnd.oasis.opendocument.text-web': 'oth',
 396 # OOo XML
 397     'application/vnd.sun.xml.base': 'odb',
 398     'application/vnd.sun.xml.calc': 'sxc',
 399     'application/vnd.sun.xml.calc.template': 'stc',
 400     'application/vnd.sun.xml.chart': 'sxs',
 401     'application/vnd.sun.xml.draw': 'sxd',
 402     'application/vnd.sun.xml.draw.template': 'std',
 403     'application/vnd.sun.xml.impress': 'sxi',
 404     'application/vnd.sun.xml.impress.template': 'sti',
 405     'application/vnd.sun.xml.math': 'sxm',
 406     'application/vnd.sun.xml.writer': 'sxw',
 407     'application/vnd.sun.xml.writer.global': 'sxg',
 408     'application/vnd.sun.xml.writer.template': 'stw',
 409     'application/vnd.sun.xml.writer.web': 'stw',
 410 # MSO
 411     'application/rtf': 'rtf',
 412     'text/rtf': 'rtf',
 413     'application/msword': 'doc',
 414     'application/vnd.ms-powerpoint': 'ppt',
 415     'application/vnd.ms-excel': 'xls',
 416     'application/vnd.ms-excel.sheet.binary.macroEnabled.12': 'xlsb',
 417     'application/vnd.ms-excel.sheet.macroEnabled.12': 'xlsm',
 418     'application/vnd.ms-excel.template.macroEnabled.12': 'xltm',
 419     'application/vnd.ms-powerpoint.presentation.macroEnabled.12': 'pptm',
 420     'application/vnd.ms-powerpoint.slide.macroEnabled.12': 'sldm',
 421     'application/vnd.ms-powerpoint.slideshow.macroEnabled.12': 'ppsm',
 422     'application/vnd.ms-powerpoint.template.macroEnabled.12': 'potm',
 423     'application/vnd.ms-word.document.macroEnabled.12': 'docm',
 424     'application/vnd.ms-word.template.macroEnabled.12': 'dotm',
 425     'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
 426     'application/vnd.openxmlformats-officedocument.spreadsheetml.template': 'xltx',
 427     'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
 428     'application/vnd.openxmlformats-officedocument.presentationml.template': 'potx',
 429     'application/vnd.openxmlformats-officedocument.presentationml.slideshow': 'ppsx',
 430     'application/vnd.openxmlformats-officedocument.presentationml.slide': 'sldx',
 431     'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
 432     'application/vnd.openxmlformats-officedocument.wordprocessingml.template': 'dotx',
 433     'application/vnd.visio': 'vsd',
 434     'application/visio.drawing': 'vsd',
 435     'application/vnd.visio2013': 'vsdx',
 436     'application/vnd.visio.xml': 'vdx',
 437     'application/x-mspublisher': 'pub',
 438 #WPS Office
 439     'application/wps-office.doc': 'doc',
 440     'application/wps-office.docx': 'docx',
 441     'application/wps-office.xls': 'xls',
 442     'application/wps-office.xlsx': 'xlsx',
 443     'application/wps-office.ppt': 'ppt',
 444     'application/wps-office.pptx': 'pptx',
 445 # W3C
 446     'application/xhtml+xml': 'xhtml',
 447     'application/mathml+xml': 'mml',
 448     'text/html': 'html',
 449     'application/docbook+xml': 'docbook',
 450 # misc
 451     'text/csv': 'csv',
 452     'text/spreadsheet': 'slk',
 453     'application/x-qpro': 'qpro',
 454     'application/x-dbase': 'dbf',
 455     'application/vnd.corel-draw': 'cdr',
 456     'application/vnd.lotus-wordpro': 'lwp',
 457     'application/vnd.lotus-1-2-3': 'wks',
 458     'application/vnd.wordperfect': 'wpd',
 459     'application/wordperfect5.1': 'wpd',
 460     'application/vnd.ms-works': 'wps',
 461     'application/clarisworks' : 'cwk',
 462     'application/macwriteii' : 'mw',
 463     'application/vnd.apple.keynote': 'key',
 464     'application/vnd.apple.numbers': 'numbers',
 465     'application/vnd.apple.pages': 'pages',
 466     'application/x-iwork-keynote-sffkey': 'key',
 467     'application/x-iwork-numbers-sffnumbers': 'numbers',
 468     'application/x-iwork-pages-sffpages': 'pages',
 469     'application/x-hwp': 'hwp',
 470     'application/x-aportisdoc': 'pdb',
 471     'application/prs.plucker' : 'pdb_plucker',
 472     'application/vnd.palm' : 'pdb_palm',
 473     'application/x-sony-bbeb' : 'lrf',
 474     'application/x-pocket-word': 'psw',
 475     'application/x-t602': '602',
 476     'application/x-fictionbook+xml': 'fb2',
 477     'application/x-abiword': 'abw',
 478     'application/x-pagemaker': 'pmd',
 479     'application/x-gnumeric': 'gnumeric',
 480     'application/vnd.stardivision.calc': 'sdc',
 481     'application/vnd.stardivision.draw': 'sda',
 482     'application/vnd.stardivision.writer': 'sdw',
 483     'application/x-starcalc': 'sdc',
 484     'application/x-stardraw': 'sdd',
 485     'application/x-starwriter': 'sdw',
 486 # relatively uncommon image mimetypes
 487     'image/x-freehand': 'fh',
 488     'image/cgm': 'cgm',
 489     'image/tif': 'tiff',
 490     'image/tiff': 'tiff',
 491     'image/vnd.dxf': 'dxf',
 492     'image/emf': 'emf',
 493     'image/x-emf': 'emf',
 494     'image/x-targa': 'tga',
 495     'image/x-sgf': 'sgf',
 496     'image/x-svm': 'svm',
 497     'image/wmf': 'wmf',
 498     'image/x-wmf': 'wmf',
 499     'image/x-pict': 'pict',
 500     'image/x-cmx': 'cmx',
 501     'image/svg+xml': 'svg',
 502     'image/bmp': 'bmp',
 503     'image/x-ms-bmp': 'bmp',
 504     'image/x-MS-bmp': 'bmp',
 505     'image/x-wpg': 'wpg',
 506     'image/x-eps': 'eps',
 507     'image/x-met': 'met',
 508     'image/x-portable-bitmap': 'pbm',
 509     'image/x-photo-cd': 'pcd',
 510     'image/x-pcx': 'pcx',
 511     'image/x-portable-graymap': 'pgm',
 512     'image/x-portable-pixmap': 'ppm',
 513     'image/vnd.adobe.photoshop': 'psd',
 514     'image/x-cmu-raster': 'ras',
 515     'image/x-sun-raster': 'ras',
 516     'image/x-xbitmap': 'xbm',
 517     'image/x-xpixmap': 'xpm',
 518 }
 519
 520 # disabled for now, this would download gigs of pngs/jpegs...
 521 common_noncore_mimetypes = {
 522 # graphics
 523     'image/gif': 'gif',
 524     'image/jpeg': 'jpeg',
 525     'image/png': 'png',
 526 # pdf, etc.
 527     'application/pdf': 'pdf',
 528 }
 529
 530 class manage_threads(threading.Thread):
 531     def run(self):
 532         #print(threading.current_thread().get_ident())
 533         while 1:
 534             # Try to receive a job from queue
 535             try:
 536                 # Get job from queue
 537                 # Use job parameters to call our query
 538                 # Then let the queue know we are done with this job
 539                 (uri, mimetype, prefix, extension) = jobs.get(True,6)
 540                 try:
 541                     get_through_rss_query(uri, mimetype, prefix, extension)
 542                 finally:
 543                     jobs.task_done()
 544             except KeyboardInterrupt:
 545                 raise # Ctrl+C should work
 546             except queue.Empty:
 547                 break
 548
 549 def generate_multi_threading():
 550     for (prefix, uri) in rss_bugzillas:
 551
 552         # Initialize threads
 553         for i in range(max_threads):
 554             manage_threads().start()
 555
 556         # Create a job for every mimetype for a bugzilla
 557         for (mimetype,extension) in mimetypes.items():
 558             # It seems that bugzilla has problems returning that many results
 559             # (10000 results is probably a limit set somewhere) so we always
 560             # end processing the complete list.
 561             if mimetype == 'text/html' and prefix == 'moz':
 562                     continue
 563
 564             jobs.put([uri, mimetype, prefix, extension], block=True)
 565             print("successfully placed a job in the queue searching for " + mimetype + " in bugtracker " + prefix)
 566
 567         # Continue when all mimetypes are done for a bugzilla
 568         jobs.join()
 569         print("DONE with bugtracker " + prefix)
 570
 571 max_threads = 20 # Number of threads to create, (1 = without multi-threading)
 572 jobs = queue.Queue()
 573
 574 generate_multi_threading()
 575
 576 for (mimetype,extension) in mimetypes.items():
 577     get_through_rpc_query(redhatrpc, redhatbug, mimetype, "rhbz", extension)
 578
 579 try:
 580     get_launchpad_bugs("lp")
 581 except ImportError:
 582     print("launchpadlib unavailable, skipping Ubuntu tracker")
 583
 584 # vim:set shiftwidth=4 softtabstop=4 expandtab: