src/mailman/utilities/importer.py

   1 # Copyright (C) 2010-2013 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """Importer routines."""
  19
  20 from __future__ import absolute_import, print_function, unicode_literals
  21
  22 __metaclass__ = type
  23 __all__ = [
  24     'import_config_pck',
  25     'Import21Error',
  26     ]
  27
  28
  29 import sys
  30 import datetime
  31 import os
  32 from urllib2 import URLError
  33
  34 from mailman.config import config
  35 from mailman.core.errors import MailmanError
  36 from mailman.interfaces.action import FilterAction, Action
  37 from mailman.interfaces.autorespond import ResponseAction
  38 from mailman.interfaces.digests import DigestFrequency
  39 from mailman.interfaces.mailinglist import Personalization, ReplyToMunging
  40 from mailman.interfaces.nntp import NewsgroupModeration
  41 from mailman.interfaces.archiver import ArchivePolicy
  42 from mailman.interfaces.bans import IBanManager
  43 from mailman.interfaces.mailinglist import IAcceptableAliasSet
  44 from mailman.interfaces.bounce import UnrecognizedBounceDisposition
  45 from mailman.interfaces.usermanager import IUserManager
  46 from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
  47 from mailman.interfaces.languages import ILanguageManager
  48 from mailman.handlers.decorate import decorate, decorate_template
  49 from mailman.utilities.i18n import search
  50 from zope.component import getUtility
  51
  52
  53 \f
  54 class Import21Error(MailmanError):
  55     pass
  56
  57 \f
  58 def str_to_unicode(value):
  59     # Convert a string to unicode when the encoding is not declared
  60     if isinstance(value, unicode):
  61         return value
  62     for encoding in ("ascii", "utf-8"):
  63         try:
  64             return unicode(value, encoding)
  65         except UnicodeDecodeError:
  66             continue
  67     # we did our best, use replace
  68     return unicode(value, 'ascii', 'replace')
  69
  70 \f
  71 def seconds_to_delta(value):
  72     return datetime.timedelta(seconds=value)
  73
  74 \f
  75 def days_to_delta(value):
  76     return datetime.timedelta(days=value)
  77
  78 \f
  79 def list_members_to_unicode(value):
  80     return [ unicode(item) for item in value ]
  81
  82 \f
  83 def filter_action_mapping(value):
  84     # The filter_action enum values have changed. In Mailman 2.1 the order was
  85     # 'Discard', 'Reject', 'Forward to List Owner', 'Preserve'.
  86     # In 3.0 it's 'hold', 'reject', 'discard', 'accept', 'defer', 'forward',
  87     # 'preserve'
  88     if value == 0:
  89         return FilterAction.discard
  90     elif value == 1:
  91         return FilterAction.reject
  92     elif value == 2:
  93         return FilterAction.forward
  94     elif value == 3:
  95         return FilterAction.preserve
  96     else:
  97         raise ValueError("Unknown filter_action value: %s" % value)
  98
  99 \f
 100 def member_action_mapping(value):
 101     # The mlist.default_member_action and mlist.default_nonmember_action enum
 102     # values are different in Mailman 2.1, because they have been merged into a
 103     # single enum in Mailman 3
 104     # For default_member_action, which used to be called
 105     # member_moderation_action, the values were:
 106     # 0==Hold, 1=Reject, 2==Discard
 107     if value == 0:
 108         return Action.hold
 109     elif value == 1:
 110         return Action.reject
 111     elif value == 2:
 112         return Action.discard
 113 def nonmember_action_mapping(value):
 114     # For default_nonmember_action, which used to be called
 115     # generic_nonmember_action, the values were:
 116     # 0==Accept, 1==Hold, 2==Reject, 3==Discard
 117     if value == 0:
 118         return Action.accept
 119     elif value == 1:
 120         return Action.hold
 121     elif value == 2:
 122         return Action.reject
 123     elif value == 3:
 124         return Action.discard
 125
 126 \f
 127 def unicode_to_string(value):
 128     return str(value) if value is not None else None
 129
 130 \f
 131 def check_language_code(code):
 132     if code is None:
 133         return None
 134     code = unicode(code)
 135     if code not in getUtility(ILanguageManager):
 136         msg = """Missing language: {0}
 137 You must add a section describing this language in your mailman.cfg file.
 138 This section should look like this:
 139 [language.{0}]
 140 # The English name for the language.
 141 description: CHANGE ME
 142 # And the default character set for the language.
 143 charset: utf-8
 144 # Whether the language is enabled or not.
 145 enabled: yes
 146 """.format(code)
 147         raise Import21Error(msg)
 148     return code
 149
 150
 151 # Attributes in Mailman 2 which have a different type in Mailman 3.
 152 TYPES = dict(
 153     autorespond_owner=ResponseAction,
 154     autorespond_postings=ResponseAction,
 155     autorespond_requests=ResponseAction,
 156     autoresponse_grace_period=days_to_delta,
 157     bounce_info_stale_after=seconds_to_delta,
 158     bounce_you_are_disabled_warnings_interval=seconds_to_delta,
 159     digest_volume_frequency=DigestFrequency,
 160     filter_action=filter_action_mapping,
 161     newsgroup_moderation=NewsgroupModeration,
 162     personalize=Personalization,
 163     reply_goes_to_list=ReplyToMunging,
 164     filter_types=list_members_to_unicode,
 165     pass_types=list_members_to_unicode,
 166     filter_extensions=list_members_to_unicode,
 167     pass_extensions=list_members_to_unicode,
 168     forward_unrecognized_bounces_to=UnrecognizedBounceDisposition,
 169     default_member_action=member_action_mapping,
 170     default_nonmember_action=nonmember_action_mapping,
 171     moderator_password=unicode_to_string,
 172     preferred_language=check_language_code,
 173     )
 174
 175
 176 # Attribute names in Mailman 2 which are renamed in Mailman 3.
 177 NAME_MAPPINGS = dict(
 178     include_list_post_header='allow_list_posts',
 179     real_name='display_name',
 180     last_post_time='last_post_at',
 181     autoresponse_graceperiod='autoresponse_grace_period',
 182     autorespond_admin='autorespond_owner',
 183     autoresponse_admin_text='autoresponse_owner_text',
 184     filter_mime_types='filter_types',
 185     pass_mime_types='pass_types',
 186     filter_filename_extensions='filter_extensions',
 187     pass_filename_extensions='pass_extensions',
 188     bounce_processing='process_bounces',
 189     bounce_unrecognized_goes_to_list_owner='forward_unrecognized_bounces_to',
 190     mod_password='moderator_password',
 191     news_moderation='newsgroup_moderation',
 192     news_prefix_subject_too='nntp_prefix_subject_too',
 193     send_welcome_msg='send_welcome_message',
 194     send_goodbye_msg='send_goodbye_message',
 195     member_moderation_action='default_member_action',
 196     generic_nonmember_action='default_nonmember_action',
 197     )
 198
 199 EXCLUDES = (
 200     "members",
 201     "digest_members",
 202     )
 203
 204
 205 \f
 206 def import_config_pck(mlist, config_dict):
 207     """Apply a config.pck configuration dictionary to a mailing list.
 208
 209     :param mlist: The mailing list.
 210     :type mlist: IMailingList
 211     :param config_dict: The Mailman 2.1 configuration dictionary.
 212     :type config_dict: dict
 213     """
 214     for key, value in config_dict.items():
 215         # Some attributes must not be directly imported
 216         if key in EXCLUDES:
 217             continue
 218         # Some attributes from Mailman 2 were renamed in Mailman 3.
 219         key = NAME_MAPPINGS.get(key, key)
 220         # Handle the simple case where the key is an attribute of the
 221         # IMailingList and the types are the same (modulo 8-bit/unicode
 222         # strings).
 223         # When attributes raise an exception, hasattr may think they don't
 224         # exist (see python issue 9666). Add them here.
 225         if hasattr(mlist, key) or key in ("preferred_language", ):
 226             if isinstance(value, str):
 227                 value = str_to_unicode(value)
 228             # Some types require conversion.
 229             converter = TYPES.get(key)
 230             if converter is not None:
 231                 value = converter(value)
 232             try:
 233                 setattr(mlist, key, value)
 234             except TypeError:
 235                 print('Type conversion error:', key, file=sys.stderr)
 236                 raise
 237     # Handle the archiving policy.  In MM2.1 there were two boolean options
 238     # but only three of the four possible states were valid.  Now there's just
 239     # an enum.
 240     if config_dict.get('archive'):
 241         # For maximum safety, if for some strange reason there's no
 242         # archive_private key, treat the list as having private archives.
 243         if config_dict.get('archive_private', True):
 244             mlist.archive_policy = ArchivePolicy.private
 245         else:
 246             mlist.archive_policy = ArchivePolicy.public
 247     else:
 248         mlist.archive_policy = ArchivePolicy.never
 249     # Handle ban list
 250     for addr in config_dict.get('ban_list', []):
 251         IBanManager(mlist).ban(str_to_unicode(addr))
 252     # Handle acceptable aliases
 253     acceptable_aliases = config_dict.get('acceptable_aliases', '')
 254     if isinstance(acceptable_aliases, basestring):
 255         acceptable_aliases = acceptable_aliases.splitlines()
 256     for addr in acceptable_aliases:
 257         addr = addr.strip()
 258         if not addr:
 259             continue
 260         addr = str_to_unicode(addr)
 261         try:
 262             IAcceptableAliasSet(mlist).add(addr)
 263         except ValueError:
 264             IAcceptableAliasSet(mlist).add("^" + addr)
 265     # Handle conversion to URIs
 266     convert_to_uri = {
 267         "welcome_msg": "welcome_message_uri",
 268         "goodbye_msg": "goodbye_message_uri",
 269         "msg_header": "header_uri",
 270         "msg_footer": "footer_uri",
 271         "digest_header": "digest_header_uri",
 272         "digest_footer": "digest_footer_uri",
 273         }
 274     convert_placeholders = { # only the most common ones
 275         "%(real_name)s": "$display_name",
 276         "%(real_name)s@%(host_name)s": "$fqdn_listname",
 277         "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s": "$listinfo_uri",
 278     }
 279     # Collect defaults
 280     defaults = {}
 281     for oldvar, newvar in convert_to_uri.iteritems():
 282         default_value = getattr(mlist, newvar)
 283         if not default_value:
 284             continue
 285         # Check if the value changed from the default
 286         try:
 287             default_text = decorate(mlist, default_value)
 288         except (URLError, KeyError):
 289             # Use case: importing the old a@ex.com into b@ex.com
 290             # We can't check if it changed from the default
 291             # -> don't import, we may do more harm than good and it's easy to
 292             # change if needed
 293             continue
 294         defaults[newvar] = (default_value, default_text)
 295     for oldvar, newvar in convert_to_uri.iteritems():
 296         if oldvar not in config_dict:
 297             continue
 298         text = config_dict[oldvar]
 299         text = unicode(text, "utf-8", "replace")
 300         for oldph, newph in convert_placeholders.iteritems():
 301             text = text.replace(oldph, newph)
 302         default_value, default_text  = defaults.get(newvar, (None, None))
 303         if not text and not (default_value or default_text):
 304             continue # both are empty, leave it
 305         # Check if the value changed from the default
 306         try:
 307             expanded_text = decorate_template(mlist, text)
 308         except KeyError:
 309             # Use case: importing the old a@ex.com into b@ex.com
 310             # We can't check if it changed from the default
 311             # -> don't import, we may do more harm than good and it's easy to
 312             # change if needed
 313             continue
 314         if expanded_text and default_text \
 315                 and expanded_text.strip() == default_text.strip():
 316             continue # keep the default
 317         # Write the custom value to the right file
 318         base_uri = "mailman:///$listname/$language/"
 319         if default_value:
 320             filename = default_value.rpartition("/")[2]
 321         else:
 322             filename = "%s.txt" % newvar[:-4]
 323         if not default_value or not default_value.startswith(base_uri):
 324             setattr(mlist, newvar, base_uri + filename)
 325         filepath = list(search(filename, mlist))[0]
 326         try:
 327             os.makedirs(os.path.dirname(filepath))
 328         except OSError, e:
 329             if e.errno != 17: # Already exists
 330                 raise
 331         with open(filepath, "w") as template:
 332             template.write(text.encode('utf-8'))
 333     # Import rosters
 334     members = set(config_dict.get("members", {}).keys()
 335                 + config_dict.get("digest_members", {}).keys())
 336     import_roster(mlist, config_dict, members, MemberRole.member)
 337     import_roster(mlist, config_dict, config_dict.get("owner", []),
 338                   MemberRole.owner)
 339     import_roster(mlist, config_dict, config_dict.get("moderator", []),
 340                   MemberRole.moderator)
 341
 342
 343 \f
 344 def import_roster(mlist, config_dict, members, role):
 345     """
 346     Import members lists from a config.pck configuration dictionary to a
 347     mailing list.
 348
 349     :param mlist: The mailing list.
 350     :type  mlist: IMailingList
 351     :param config_dict: The Mailman 2.1 configuration dictionary.
 352     :type  config_dict: dict
 353     :param members: The members list to import
 354     :type  members: list
 355     :param role: The MemberRole to import them as
 356     :type  role: MemberRole enum
 357     """
 358     usermanager = getUtility(IUserManager)
 359     for email in members:
 360         # for owners and members, the emails can have a mixed case, so
 361         # lowercase them all
 362         email = str_to_unicode(email).lower()
 363         roster = mlist.get_roster(role)
 364         if roster.get_member(email) is not None:
 365             print("%s is already imported with role %s" % (email, role),
 366                   file=sys.stderr)
 367             continue
 368         address = usermanager.get_address(email)
 369         user = usermanager.get_user(email)
 370         if user is None:
 371             user = usermanager.create_user()
 372             if address is None:
 373                 merged_members = {}
 374                 merged_members.update(config_dict.get("members", {}))
 375                 merged_members.update(config_dict.get("digest_members", {}))
 376                 if merged_members.get(email, 0) != 0:
 377                     original_email = str_to_unicode(merged_members[email])
 378                 else:
 379                     original_email = email
 380                 address = usermanager.create_address(original_email)
 381                 address.verified_on = datetime.datetime.now()
 382             user.link(address)
 383         mlist.subscribe(address, role)
 384         member = roster.get_member(email)
 385         assert member is not None
 386         prefs = config_dict.get("user_options", {}).get(email, 0)
 387         if email in config_dict.get("members", {}):
 388             member.preferences.delivery_mode = DeliveryMode.regular
 389         elif email in config_dict.get("digest_members", {}):
 390             if prefs & 8: # DisableMime
 391                 member.preferences.delivery_mode = DeliveryMode.plaintext_digests
 392             else:
 393                 member.preferences.delivery_mode = DeliveryMode.mime_digests
 394         else:
 395             # probably not adding a member role here
 396             pass
 397         if email in config_dict.get("language", {}):
 398             member.preferences.preferred_language = \
 399                 check_language_code(config_dict["language"][email])
 400         # if the user already exists, display_name and password will be
 401         # overwritten
 402         if email in config_dict.get("usernames", {}):
 403             address.display_name = \
 404                     str_to_unicode(config_dict["usernames"][email])
 405             user.display_name    = \
 406                     str_to_unicode(config_dict["usernames"][email])
 407         if email in config_dict.get("passwords", {}):
 408             user.password = config.password_context.encrypt(
 409                                     config_dict["passwords"][email])
 410         # delivery_status
 411         oldds = config_dict.get("delivery_status", {}).get(email, (0, 0))[0]
 412         if oldds == 0:
 413             member.preferences.delivery_status = DeliveryStatus.enabled
 414         elif oldds == 1:
 415             member.preferences.delivery_status = DeliveryStatus.unknown
 416         elif oldds == 2:
 417             member.preferences.delivery_status = DeliveryStatus.by_user
 418         elif oldds == 3:
 419             member.preferences.delivery_status = DeliveryStatus.by_moderator
 420         elif oldds == 4:
 421             member.preferences.delivery_status = DeliveryStatus.by_bounces
 422         # moderation
 423         if prefs & 128:
 424             member.moderation_action = Action.hold
 425         # other preferences
 426         member.preferences.acknowledge_posts = bool(prefs & 4) # AcknowledgePosts
 427         member.preferences.hide_address = bool(prefs & 16) # ConcealSubscription
 428         member.preferences.receive_own_postings = not bool(prefs & 2) # DontReceiveOwnPosts
 429         member.preferences.receive_list_copy = not bool(prefs & 256) # DontReceiveDuplicates