bin/utils/fix_character_encoding.py

   1 # This file is part of Indico.
   2 # Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
   3 #
   4 # Indico is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU General Public License as
   6 # published by the Free Software Foundation; either version 3 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # Indico is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with Indico; if not, see <http://www.gnu.org/licenses/>.
  16
  17 from indico.core.db import DBMgr
  18 from MaKaC.conference import ConferenceHolder
  19 from indico.util.console import conferenceHolderIterator
  20
  21 dbi = DBMgr.getInstance()
  22 dbi.startRequest()
  23
  24 ENCODINGS = ['Windows-1252', 'iso-8859-1', 'latin1']
  25
  26 def fix(getter, setter):
  27     txt = getter()
  28     print "fixing... ",
  29     for encoding in ENCODINGS:
  30         try:
  31             utxt = txt.decode(encoding)
  32             print encoding
  33             setter(utxt.encode('utf-8'))
  34             return
  35         except (UnicodeDecodeError, UnicodeEncodeError):
  36             pass
  37     print "error! %s" % repr(txt)
  38
  39 with dbi.transaction() as conn:
  40     i = 0
  41     for level, conf in conferenceHolderIterator(ConferenceHolder(), deepness='event', verbose=False):
  42         try:
  43             conf.getTitle().decode('utf-8')
  44         except (UnicodeDecodeError, UnicodeEncodeError):
  45             print '\r%s title' % conf
  46             fix(conf.getTitle, conf.setTitle)
  47
  48         try:
  49             conf.getDescription().decode('utf-8')
  50         except (UnicodeDecodeError, UnicodeEncodeError):
  51             print '\r%s description' % conf
  52             fix(conf.getDescription, conf.setDescription)
  53
  54         if i % 999 == 0:
  55             dbi.commit()
  56         i += 1