admin/utfdbmigrate.php

   1 <?php //$Id$
   2
   3 ///dummy field names are used to help adding and dropping indexes. There's only 1 case now, in scorm_scoes_track
   4 //testing
   5     require_once('../config.php');
   6     require_once($CFG->libdir.'/adminlib.php');
   7     require_once($CFG->libdir.'/environmentlib.php');
   8     require_once($CFG->dirroot.'/course/lib.php');
   9     require_once($CFG->libdir.'/ddllib.php');      //We are going to need DDL services here
  10     require_once($CFG->dirroot.'/backup/lib.php');  //We are going to need BACKUP services here
  11     define ('BACKUP_UNIQUE_CODE', '1100110011');    //One code in the past to store UTF8 temp indexes info
  12     require_login();
  13
  14     // declare once
  15     global $enc;
  16
  17     $customlang = array();
  18
  19     $enc = array('af' => 'iso-8859-1', 'ar' => 'windows-1256', 'be' => 'windows-1251', 'bg' => 'windows-1251', 'bs' => 'windows-1250', 'ca' => 'iso-8859-1', 'cs' => 'iso-8859-2', 'da' => 'iso-8859-1', 'de' => 'iso-8859-1', 'de_du' => 'iso-8859-1', 'de_utf8' => 'utf-8', 'el' => 'windows-1253', 'en' => 'iso-8859-1', 'en_ja' => 'euc-jp', 'en_us' => 'iso-8859-1', 'en_utf8' => 'utf-8', 'es' => 'iso-8859-1', 'es_ar' => 'iso-8859-1', 'es_es' => 'iso-8859-1', 'es_mx' => 'iso-8859-1', 'et' => 'iso-8859-1', 'eu' => 'iso-8859-1', 'fa' => 'windows-1256', 'fa_utf8' => 'utf-8', 'fi' => 'iso-8859-1', 'fil' => 'iso-8859-15', 'fr' => 'iso-8859-1', 'fr_ca' => 'iso-8859-15', 'ga' => 'iso-8859-1', 'gl' => 'iso-8859-1', 'he' => 'ISO-8859-8-I', 'he_utf8' => 'utf-8', 'hi' => 'iso-8859-1', 'hr' => 'windows-1250', 'hr_utf8' => 'utf-8', 'hu' => 'iso-8859-2', 'id' => 'iso-8859-1', 'is' => 'iso-8859-1', 'it' => 'iso-8859-1', 'ja' => 'EUC-JP', 'ja_utf8' => 'UTF-8', 'ka_utf8' => 'UTF-8', 'km_utf8' => 'UTF-8', 'kn_utf8' => 'utf-8', 'ko' => 'EUC-KR', 'ko_utf8' => 'UTF-8', 'lt' => 'windows-1257', 'lv' => 'ISO-8859-4', 'mi_nt' => 'iso-8859-1', 'mi_tn_utf8' => 'utf-8', 'ms' => 'iso-8859-1', 'nl' => 'iso-8859-1', 'nn' => 'iso-8859-1', 'no' => 'iso-8859-1', 'no_gr' => 'iso-8859-1', 'pl' => 'iso-8859-2', 'pt' => 'iso-8859-1', 'pt_br' => 'iso-8859-1', 'ro' => 'iso-8859-2', 'ru' => 'windows-1251', 'sk' => 'iso-8859-2', 'sl' => 'iso-8859-2', 'sl_utf8' => 'utf-8', 'so' => 'iso-8859-1', 'sq' => 'iso-8859-1', 'sr_utf8' => 'utf-8', 'sv' => 'iso-8859-1', 'th' => 'TIS-620', 'th_utf8' => 'UTF-8', 'tl' => 'iso-8859-15', 'tl_utf8' => 'UTF-8', 'tr' => 'iso-8859-9', 'uk' => 'windows-1251', 'vi_utf8' => 'UTF-8', 'zh_cn' => 'GB18030', 'zh_cn_utf8' => 'UTF-8', 'zh_tw' => 'Big5', 'zh_tw_utf8' => 'UTF-8');
  20
  21     /**************************************
  22      * Custom lang pack handling           *
  23      **************************************/
  24
  25     // scan list of langs, including customs packs
  26     $langs = get_list_of_languages();
  27
  28     // foreach lang
  29     foreach ($langs as $lang => $lang1) {
  30
  31         if (in_array($lang, array_keys($enc))) {
  32               // if already in array, ignore
  33             continue;
  34         }
  35
  36         // if this lang has got a charset
  37
  38         if ($result = get_string_from_file('thischarset',$CFG->dirroot.'/lang/'.$lang.'/moodle.php', "\$resultstring")) {
  39             eval($result);
  40             $enc[$lang] = $resultstring;
  41         } else if ($result = get_string_from_file('parentlanguage',$CFG->dirroot.'/lang/'.$lang.'/moodle.php',"\$resultstring")) {
  42         // else if there's a parent lang we can use
  43             eval($result);
  44               $enc[$lang] = $enc[$resultstring];
  45         } else {
  46              notify ('unknown lang pack detected '.$lang);
  47         }
  48
  49     }
  50
  51     /**************************************
  52      * End custom lang pack handling      *
  53      **************************************/
  54
  55     require_capability('moodle/site:config', get_context_instance(CONTEXT_SYSTEM, SITEID));
  56
  57     if (!$site = get_site()) {
  58         redirect('index.php');
  59     }
  60
  61     if (!empty($CFG->unicodedb)) {
  62         error ('unicode db migration has already been performed!');
  63     }
  64
  65     $migrate = optional_param('migrate', 0, PARAM_BOOL);
  66     $confirm = optional_param('confirm', 0, PARAM_BOOL);
  67
  68     $textlib = textlib_get_instance();
  69
  70     $stradministration   = get_string('administration');
  71     $strdbmigrate = get_string('dbmigrate','admin');
  72
  73     $filename = $CFG->dataroot.'/'.SITEID.'/maintenance.html';    //maintenance file
  74
  75     print_header("$site->shortname: $stradministration", "$site->fullname",
  76                  '<a href="index.php">'. "$stradministration</a> -> $strdbmigrate");
  77
  78     print_heading($strdbmigrate);
  79
  80     if ($CFG->dbtype == 'postgres7') {
  81         $CFG->pagepath = 'admin/utfdbmigrate/postgresql';
  82     }
  83     //if $confirm
  84     if ($confirm && confirm_sesskey()) {
  85         //do the real migration of db
  86         print_simple_box_start('center','50%');
  87         print_string('importlangreminder','admin');
  88         print_simple_box_end();
  89         db_migrate2utf8();
  90         print_heading('db unicode migration has been completed!');
  91         if (!get_config('', 'migrate_maintmode')) { // already in maint mode
  92             unlink($filename);    //no longer in maintenance mode
  93         }
  94         unset_config('migrate_maintmode');
  95         @require_logout();
  96         print_continue($CFG->wwwroot.'/'.$CFG->admin.'/langimport.php');
  97     }
  98
  99     //else if $migrate
 100     else if ($migrate && confirm_sesskey()) {
 101         if ($CFG->dbtype == 'postgres7' && !is_postgres_utf8()) {
 102             $continue = false;
 103             if (($form = data_submitted()) && isset($form->dbhost)) {
 104                 validate_form($form, $err);
 105
 106                 if (count($err) == 0) {
 107                     $_SESSION['newpostgresdb'] = $form;
 108                     $continue = true;
 109                 }
 110             }
 111         } else {
 112             $continue = true;
 113         }
 114         if ($continue) {
 115             echo '<div align="center">';
 116             print_simple_box_start('center','50%');
 117             print_string('dbmigratewarning2','admin');
 118             print_simple_box_end();
 119             //put the site in maintenance mode
 120             check_dir_exists($CFG->dataroot.'/'.SITEID, true);
 121
 122             if (file_exists($filename)) {
 123                 set_config('migrate_maintmode', 1); // already in maintenance mode
 124             } else {
 125                 if (touch($filename)) {
 126                     $file = fopen($filename, 'w');
 127                     fwrite($file, get_string('maintinprogress','admin'));
 128                     fclose($file);
 129                 } else {
 130                     notify (get_string('maintfileopenerror','admin'));
 131                 }
 132             }
 133             //print second confirmation box
 134             echo '<form name="migratefrom" action="utfdbmigrate.php" method="POST">';
 135             echo '<input name="confirm" type="hidden" value="1" />';
 136             echo '<input name="sesskey" type="hidden" value="'.sesskey().'" />';
 137
 138             $xmls = utf_get_xml();
 139             $sumrecords = 0;   //this is the sum of all records of relavent tables.
 140             foreach ($xmls as $xml) {    ///foreach xml file, we must get a list of tables
 141                 $dbtables = $xml['DBMIGRATION']['#']['TABLES'][0]['#']['TABLE'];    //real db tables
 142
 143                 foreach ($dbtables as $dbtable) {
 144                     $dbtablename = $dbtable['@']['name'];
 145
 146                     if ($dbtablename=='adodb_logsql') {
 147                         $prefix = '';
 148                     } else {
 149                         $prefix = $CFG->prefix;
 150                     }
 151                     $sumrecords += count_records_sql("SELECT COUNT(*) FROM {$prefix}$dbtablename");
 152                 }
 153             }
 154             echo 'Total number of records in your database is <b>'.$sumrecords.'</b>';
 155             if ($sumrecords > 10000) {
 156                 echo '<br />Number of Records to process before halting (Leave blank for no limit) <input name="maxrecords" type="text" value="" />';
 157             }
 158
 159             //print the "i-know-what-lang-to-use" menu
 160
 161             echo '<br />The whole site is in this encoding: (leave blank if you are not sure)';
 162             echo '<select name="globallang"><option value="">I am not sure</option>';
 163             foreach ($enc as $lang => $encoding) {
 164                 echo '<option value="'.$encoding.'">'.$lang.'</option>';
 165             }
 166             echo '</select>';
 167
 168             echo '<p /><input type="submit" value="'.get_string('continue').'"/>';
 169             echo '<input type="button" value="'.get_string('cancel').'" onclick="javascript:history.go(-1)" />';
 170             echo '</form>';
 171             echo '</div>';
 172
 173         } else {
 174             echo '<div align="center">';
 175             print_simple_box_start('center','50%');
 176             print_string('dbmigratepostgres','admin');
 177             print_simple_box_end();
 178
 179             print_simple_box_start("center", "");
 180             include("utfdbmigrate.html");
 181             print_simple_box_end();
 182         }
 183     }
 184
 185     else {    //else, print welcome to migrate page message
 186         echo '<div align="center">';
 187         print_simple_box_start('center','50%');
 188         print_string('dbmigratewarning','admin');
 189         print_simple_box_end();
 190
 191         /*************************************
 192          * Eloy's environement checking code *
 193          *************************************/
 194
 195         $current_version = $CFG->release;
 196
 197     /// Gather and show results
 198         $status = check_moodle_environment($current_version, $environment_results);
 199
 200         //end of env checking
 201
 202     /// We only allow to continue if environmental checks have been passed ok
 203         if ($status) {
 204             echo '<form name="migratefrom" action="utfdbmigrate.php" method="POST">';
 205             echo '<input name="migrate" type="hidden" value="1" />';
 206             echo '<input name="sesskey" type="hidden" value="'.sesskey().'" />';
 207             echo '<input type="submit" value="'.get_string('continue').'"/>';
 208             echo '&nbsp;<input type="button" value="'.get_string('cancel').'" onclick="javascript:history.go(-1)" />';
 209             echo '</form>';
 210             echo '</div>';
 211         }
 212     }
 213
 214     print_footer();
 215
 216
 217 function db_migrate2utf8(){   //Eloy: Perhaps some type of limit parameter here
 218                               //pointing to the num of records to process would be
 219                               //useful. And it won't break anything, because the
 220                               //crash system will continue the next time it was
 221                               //executed. Also, the function could return:
 222                               //0 = Some sort of error
 223                               //1 = Finished completelly!
 224                               //2 = Finished limit records
 225                               //(using constants, of course ;-))
 226                               //Then, all those errors, should be converted to
 227                               //mtrace() and return 0. (showing the current
 228                               //table/field/recordid)
 229
 230     global $db, $CFG, $dbtablename, $fieldname, $record, $processedrecords;
 231     $debug = debugging();
 232
 233     if ($CFG->dbtype == 'mysql') {
 234         check_and_create_backup_dir(BACKUP_UNIQUE_CODE);  //Create the backup temp dir
 235     }
 236
 237     ignore_user_abort(false); // see bug report 5352. This should kill this thread as soon as user aborts.
 238
 239     @set_time_limit(0);
 240     @ob_implicit_flush(true);
 241     @ob_end_flush();
 242
 243     $maxrecords = optional_param('maxrecords', 0, PARAM_INT);
 244     $globallang = optional_param('globallang', '', PARAM_FILE);
 245     $processedrecords = 0;
 246
 247     $ignoretables = array();    //list of tables to ignore, optional
 248
 249     //one gigantic array to hold all db table information read from all the migrate2utf8.xml file.
 250     require_once($CFG->dirroot.'/lib/xmlize.php');
 251     $xmls = utf_get_xml(1);
 252     $tablestoconvert = 0; // total number of tables to convert
 253     foreach ($xmls as $xml) {    ///foreach xml file, we must get a list of tables
 254         $dbtables = $xml['DBMIGRATION']['#']['TABLES'][0]['#']['TABLE'];    //real db tables
 255         foreach ($dbtables as $dbtable) {
 256             $tablestoconvert++;
 257         }
 258     }
 259     // progress bar handling
 260     // first let's find out how many tables there are
 261
 262     $done = 0;
 263     print_progress($done, $tablestoconvert, 5, 1);
 264
 265
 266     $textlib = textlib_get_instance();    //only 1 reference
 267
 268     //if unicodedb is set, migration is complete. die here;
 269     if (!$crash = get_record('config','name','dbmigration')) {
 270         //Duplicate the database if not unicode for postgres7
 271         if ($CFG->dbtype == 'postgres7' && !is_postgres_utf8() && !is_postgres_setup()) {
 272             echo '<script>';
 273             echo 'document.getElementById("text").innerHTML = "Copying data to the UTF8 database for processing...";'."\n";
 274             echo '</script>';
 275
 276             if ($_SESSION['newpostgresdb']->dbcluster) {
 277                 $cluster = ' --cluster ' . $_SESSION['newpostgresdb']->dbcluster;
 278             } else {
 279                 $cluster = '';
 280             }
 281             $pgdump = 'pg_dump';
 282             if (!empty($_SESSION['newpostgresdb']->pathtopgdump)) {
 283                 $pgdump = $_SESSION['newpostgresdb']->pathtopgdump;
 284             }
 285             $psql = 'psql';
 286             if (!empty($_SESSION['newpostgresdb']->pathtopsql)) {
 287                 $pgsql = $_SESSION['newpostgresdb']->pathtopsql;
 288             }
 289
 290             $cmd = "PGPASSWORD={$CFG->dbpass} PGCLIENTENCODING='UNICODE' PGDATABASE={$CFG->dbname} $pgdump -Fp -O -x -U {$CFG->dbuser}$cluster";
 291             if ($CFG->dbhost)  {
 292                 $host = split(":", $CFG->dbhost);
 293                 if ($host[0]) $cmd .= " -h {$host[0]}";
 294                 if (isset($host[1])) $cmd .= " -p {$host[1]}";
 295             }
 296             $cmds[] = $cmd;
 297             $cmds[] = 'grep -v "COMMENT ON SCHEMA"';
 298             $cmds[] = 'iconv -f UTF-8 -t UTF-8 -c';
 299             $cmd = "PGPASSWORD={$_SESSION['newpostgresdb']->dbpass} PGDATABASE={$_SESSION['newpostgresdb']->dbname} $psql -q -U {$_SESSION['newpostgresdb']->dbuser} -v ON_ERROR_STOP=1$cluster";
 300             if ($_SESSION['newpostgresdb']->dbhost)  {
 301                 $host = split(":", $_SESSION['newpostgresdb']->dbhost);
 302                 if ($host[0]) $cmd .= " -h {$host[0]}";
 303                 if (isset($host[1])) $cmd .= " -p {$host[1]}";
 304             }
 305             $cmds[] = $cmd;
 306             foreach ($cmds as $key => $cmd) {
 307                 $files[] = tempnam($CFG->dataroot, 'utf8_');
 308                 $cmd = $cmd . ($key?" < {$files[$key-1]}":'') . " 2>&1 > {$files[$key]}";
 309                 if (stripos(PHP_OS, 'darwin') !== false && stripos($cmd,'iconv') !== false) {
 310                     // I know this looks DREADFULLY hackish, but the iconv in mac os x seems to have a return code of 1 for warnings
 311                     // and I cannot figure out why, it's a very different version of iconv to most *nix versions, even seems to be a
 312                     // different gnu project.
 313                     // If someone can figure out a better way to do this, FEEL FREE :)
 314                     // - Penny
 315                     $cmd .= ' || true';
 316                 }
 317                 exec($cmd, $output, $return_var);
 318                 if ($key) {
 319                     unlink($files[$key-1]);
 320                 }
 321                 if ($return_var) { // we are dead!
 322                     unlink($files[$key]);
 323                     echo '<br />';
 324                     print_simple_box_start('center','50%');
 325                     print_string('dbmigrationdupfailed','admin',htmlspecialchars(implode("\n", $output)));
 326                     print_simple_box_end();
 327                     print_footer();
 328                     exit;
 329                 }
 330             }
 331             unlink(array_pop($files));
 332         }
 333
 334         $migrationconfig = new object;
 335         $migrationconfig->name = 'dbmigration';
 336         $migrationconfig->value = '-1';
 337         insert_record('config',$migrationconfig);  //process initiated
 338
 339         //langs used, to help make recommendations on what lang packs to install
 340         $langsused = new object;
 341         $langsused->name = 'langsused';
 342         $langsused->value = '';
 343         insert_record('config',$langsused);
 344
 345     } else {
 346
 347         $crashdata = explode('##',$crash->value);
 348         $crash->table = $crashdata[0];
 349         $crash->field = $crashdata[1];
 350         $crash->record = $crashdata[2];
 351
 352         notify("Resuming migration from: $crash->table / .$crash->field, Record: $crash->record");
 353     }
 354
 355     /************************************************************************
 356      * Now we got all our tables in order                                   *
 357      ************************************************************************/
 358
 359     foreach ($xmls as $xml) {    ///foreach xml file, we must get a list of tables
 360         $dir = $xml['DBMIGRATION']['@']['type'];
 361         $dbtables = $xml['DBMIGRATION']['#']['TABLES'][0]['#']['TABLE'];    //real db tables
 362
 363         foreach ($dbtables as $dbtable) {
 364
 365             $done++;
 366             print_progress($done, $tablestoconvert, 5, 1);
 367
 368             $dbtablename = $dbtable['@']['name'];
 369
 370             // exception handling for adodb_logsql
 371             // see bug 5003
 372             if ($dbtablename == 'adodb_logsql') {
 373                 $prefix = '';
 374             } else {
 375                 $prefix = $CFG->prefix;
 376             }
 377
 378             if ($crash && ($dbtablename != $crash->table)) {  //resuming from crash
 379                 $done++; // need to update progress bar
 380                 continue;
 381             }
 382
 383             if ($debug) {
 384                 print_heading("<br><b>Processsing db table ".$dbtablename.'...</b>');
 385             }
 386
 387         /*  Insted of relying in the indexes defined for the table in utfmigrate.xml
 388             files, we are going to use the MetaIndexes() DB call in order to detect
 389             all the table indexes. Once fetched, they are saved in backup tables for
 390             safe storage and they are dropped from the table.
 391             At the end of the table, we'll fetch them from backup tables and all them
 392             will be recreated again.
 393             This will ensure that no index in lost in the UTF8 migration process and
 394             they will be exactly the same for each server (custom indexes...)
 395             Also this will leave free to keep the utfmigrate.xml files in sync for
 396             all the existing indexes and we only have to maintain fields in such
 397             files
 398         */
 399
 400         /// Calculate all the indexes of the table
 401             if ($CFG->dbtype == 'mysql' && $allindexes = $db->MetaIndexes($prefix.$dbtablename)) {
 402             /// Send them to backup_ids table for temporal storage if crash
 403                 backup_putid(BACKUP_UNIQUE_CODE, $prefix.$dbtablename, 1, 1, $allindexes);
 404             /// Drop all the indexes
 405                 $sqlarr = array();
 406                 foreach ($allindexes as $onekey => $oneindex) {
 407                     $sqlarr[] = 'ALTER TABLE '.$prefix.$dbtablename.' DROP INDEX '.$onekey;
 408                 }
 409                 execute_sql_arr($sqlarr, true, $debug);
 410             }
 411
 412             /**********************************************************
 413              * This is the by pass structure. It allows us to process *
 414              * tables on row basis instead of column/field basis      *
 415              * It relies on a single function in migrate2utf8.php     *
 416              **********************************************************/
 417
 418             /// first, check to see if there's a function for the whole table. By pass point (1)
 419             if (file_exists($CFG->dirroot.'/'.$dir.'/db/migrate2utf8.php')) {
 420                 require_once($CFG->dirroot.'/'.$dir.'/db/migrate2utf8.php');
 421                 // this is a function to process table on role basis, e.g. user table in moodorg
 422                 $tablefunction = 'migrate2utf8_'.$dbtablename;
 423             }
 424             if ($CFG->dbtype=='mysql' && function_exists($tablefunction)) {
 425                 $tablefunction($dbtable['#']['FIELDS'][0]['#']['FIELD'], $crash, $debug, $maxrecords, $done, $tablestoconvert); // execute it.
 426             } else {
 427
 428             /******************************************************
 429              * No function for converting whole table, we proceed *
 430              ******************************************************/
 431
 432                 if (!empty($dbtable['#']) && ($fields = $dbtable['#']['FIELDS'][0]['#']['FIELD']) and (!in_array($dbtablename, $ignoretables))) {
 433
 434                     $colnames = array();
 435                     $coltypes = array();    //array to hold all column types for the table
 436                     $collengths = array();    //array to hold all column lengths for the table
 437                     $defaults = array();    //array to hold defaults, if any
 438                     //reset holders
 439                     $addindexarray = array();
 440                     $adduniqueindexarray = array();
 441                     $addprimaryarray = array();
 442
 443                     foreach ($fields as $field){
 444
 445                         //if in crash state, and field name is not the same as crash field name
 446
 447                         $fieldname = isset($field['@']['name'])?$field['@']['name']:"";
 448                         $method = isset($field['@']['method'])?$field['@']['method']:"";
 449                         $type = isset($field['@']['type'])?$field['@']['type']:"";
 450                         $length = isset($field['@']['length'])?$field['@']['length']:"";
 451
 452                         if ($crash && ($crash->field != $fieldname)) {
 453                             continue;
 454                         }
 455
 456                         $dropindex = isset($field['@']['dropindex'])?$field['@']['dropindex']:"";
 457                         $addindex = isset($field['@']['addindex'])?$field['@']['addindex']:"";
 458                         $adduniqueindex = isset($field['@']['adduniqueindex'])?$field['@']['adduniqueindex']:"";
 459
 460                         $dropprimary = isset($field['@']['dropprimary'])?$field['@']['dropprimary']:"";
 461                         $addprimary = isset($field['@']['addprimary'])?$field['@']['addprimary']:"";
 462                         $default = isset($field['@']['default'])?"'".$field['@']['default']."'":"''";
 463
 464                         if ($fieldname != 'dummy') {
 465                             $colnames[] = $fieldname;
 466                             $coltypes[] = $type;
 467                             $collengths[]= $length;
 468                         }
 469
 470                         if ($debug) {
 471                             echo "<br>--><b>processing db field ".$fieldname.'</b>';
 472                             echo "<br>---><b>method ".$method.'</b>';
 473                         }
 474
 475
 476                         if ($CFG->dbtype == 'mysql') {
 477
 478                             /* Drop the index, because with index on, you can't change it to longblob
 479
 480                                NOTE: We aren't going to drop individual indexes anymore, because we have
 481                                      dropped them at the begining of the table iteration, saving them to
 482                                      backup temp tables. At the end of the table iteration we are going
 483                                      to rebuild them back
 484
 485                             if ($dropindex){    //drop index if index is varchar, text etc type
 486                                 $SQL = 'ALTER TABLE '.$prefix.$dbtablename.' DROP INDEX '.$dropindex.';';
 487                                 $SQL1 = 'ALTER TABLE '.$prefix.$dbtablename.' DROP INDEX '.$CFG->prefix.$dropindex.';'; // see bug 5205
 488                                 if ($debug) {
 489                                     $db->debug=999;
 490                                 }
 491
 492                                 execute_sql($SQL, false); // see bug 5205
 493                                 execute_sql($SQL1, false); // see bug 5205
 494
 495                                 if ($debug) {
 496                                     $db->debug=0;
 497                                 }
 498                             } else */
 499                             if ($dropprimary) {    // drop primary key
 500                                 $SQL = 'ALTER TABLE '.$prefix.$dbtablename.' DROP PRIMARY KEY;';
 501                                 if ($debug) {
 502                                     $db->debug=999;
 503                                 }
 504                                 execute_sql($SQL, $debug);
 505                                 if ($debug) {
 506                                     $db->debug=0;
 507                                 }
 508                             }
 509
 510                             /* Previously to change the field to LONGBLOB, we are going to
 511                                use Meta info to fetch the NULL/NOT NULL status of the field.
 512                                Then, when converting back the field to its final UTF8 status
 513                                we'll apply such status (and default)
 514                                This has been added on 1.7 because we are in the process of
 515                                converting some fields to NULL and the assumption of all the
 516                                CHAR/TEXT fields being always NOT NULL isn't valid anymore!
 517                                Note that this code will leave remaining NOT NULL fiels
 518                                unmodified at all, folowing the old approach
 519                             */
 520                             if(($cols = $db->MetaColumns($prefix.$dbtablename)) && $fieldname != 'dummy') {
 521                                 $cols = array_change_key_case($cols, CASE_LOWER); ///lowercase col names
 522                                 $notnull = 'NOT NULL';  ///Old default
 523                                 if ($col = $cols[strtolower($fieldname)]) {
 524                                 /// If the column was null before UTF-8 migration, save it
 525                                     if (!$col->not_null) {
 526                                         $notnull = 'NULL';
 527                                     /// And, if the column had an empty string as default, make it NULL now
 528                                         if ($default == "''") {
 529                                             $default = 'NULL';
 530                                         }
 531                                     }
 532                                 }
 533                             }
 534
 535                             /* Change to longblob, serves 2 purposes:
 536                                1. column loses encoding, so when we finally change it to unicode,
 537                                   mysql does not do a double convertion
 538                                2. longblobs puts no limit (ok, not really but it's large enough)
 539                                   to handle most of the problems such as in bug 5194
 540                             */
 541
 542                             $SQL = 'ALTER TABLE '.$prefix.$dbtablename;
 543                             $SQL.= ' CHANGE '.$fieldname.' '.$fieldname.' LONGBLOB';
 544
 545                             /*
 546                             if ($length > 0) {
 547                                 $SQL.='('.$length.') ';
 548                             }
 549                             $SQL .= ' CHARACTER SET binary NOT NULL DEFAULT '.$default.';';
 550                             */
 551                             if ($debug) {
 552                                 $db->debug=999;
 553                             }
 554                             if ($fieldname != 'dummy') {
 555                                 execute_sql($SQL, $debug);
 556                             }
 557                             if ($debug) {
 558                                 $db->debug=0;
 559                             }
 560
 561                         }
 562
 563                         $patterns[]='/RECORDID/';    //for preg_replace
 564                         $patterns[]='/\{\$CFG\-\>prefix\}/i';    //same here
 565
 566                         if ($method == 'PLAIN_SQL_UPDATE') {
 567                             $sqldetectuser = $field['#']['SQL_DETECT_USER'][0]['#'];
 568                             $sqldetectcourse = $field['#']['SQL_DETECT_COURSE'][0]['#'];
 569                         }
 570                         else if ($method == 'PHP_FUNCTION') {
 571                             $phpfunction = 'migrate2utf8_'.$dbtablename.'_'.$fieldname;
 572                         }
 573
 574                         ///get the total number of records for this field
 575
 576                         // could not use count_records because it addes prefix to adodb_logsql
 577                         $totalrecords = count_records_sql("select count(*) from {$prefix}$dbtablename");
 578                         $counter = 0;
 579                         $recordsetsize = 50;
 580
 581                         if ($crash) {    //if resuming from crash
 582                             //find the number of records with id smaller than the crash id
 583                             $indexSQL = 'SELECT COUNT(*) FROM '.$prefix.$dbtablename.' WHERE id < '.$crash->record;
 584                             $counter = count_records_sql($indexSQL);
 585                         }
 586
 587                         if ($debug) {
 588                             echo "<br>Total number of records is ..".$totalrecords;
 589                             echo "<br/>Counter is $counter";
 590                         }
 591
 592
 593                         /**************************
 594                          * converting each record *
 595                          **************************/
 596                         while(($counter < $totalrecords) and ($fieldname !='dummy') and ($method!='NO_CONV')) {    //while there is still something
 597                             $SQL = 'SELECT * FROM '.$prefix.$dbtablename.' ORDER BY id ASC';
 598                             if ($records = get_records_sql($SQL, $counter, $recordsetsize)) {
 599                                 foreach ($records as $record) {
 600
 601                                     //if we are up this far, either no crash, or crash with same table, field name.
 602                                     if ($crash){
 603                                         if ($crash->record != $record->id) {    //might set to < just in case record is deleted
 604                                             continue;
 605                                         } else {
 606                                             $crash = 0;
 607                                         }
 608                                     }
 609
 610                                     $migrationconfig = get_record('config','name','dbmigration');
 611                                     $migrationconfig->name = 'dbmigration';
 612                                     $migrationconfig->value = $dbtablename.'##'.$fieldname.'##'.$record->id;
 613                                     update_record('config',$migrationconfig);
 614
 615                                     $replacements = array();    //manual refresh
 616                                     $replacements[] = $record->id;
 617                                     $replacements[] = $prefix;
 618
 619                                     if (!empty($record->{$fieldname})) {    //only update if not empty
 620                                         switch ($method){
 621                                             case 'PLAIN_SQL_UPDATE':    //use the 2 statements to update
 622
 623                                                 if ($debug) {
 624                                                     $db->debug=999;
 625                                                 }
 626
 627                                                 //if global lang is set, we just use that
 628
 629                                                 if ($globallang) {
 630                                                     $fromenc = $globallang;
 631                                                 } else {
 632                                                     $userid = get_record_sql(preg_replace($patterns, $replacements, $sqldetectuser));
 633                                                     $courseid = get_record_sql(preg_replace($patterns, $replacements, $sqldetectcourse));
 634
 635                                                     $sitelang   = $CFG->lang;
 636                                                     $courselang = get_course_lang(isset($courseid->course)?$courseid->course:1);
 637                                                     $userlang   = get_user_lang(isset($userid->userid)?$userid->userid:1);
 638
 639                                                     $fromenc = get_original_encoding($sitelang, $courselang, $userlang);
 640                                                 }
 641
 642                                                 //only update if non utf8
 643                                                 if (($fromenc != 'utf-8') && ($fromenc != 'UTF-8')) {
 644                                                     $result = utfconvert($record->{$fieldname}, $fromenc);
 645                                                     $newrecord = new object;
 646                                                     $newrecord->id = $record->id;
 647                                                     $newrecord->{$fieldname} = $result;
 648                                                     migrate2utf8_update_record($dbtablename,$newrecord);
 649                                                 }
 650                                                 if ($debug) {
 651                                                     $db->debug=0;
 652                                                 }
 653                                             break;
 654
 655                                             case 'PHP_FUNCTION':    //use the default php function to execute
 656                                                 if ($debug) {
 657                                                     $db->debug=999;
 658                                                 }
 659                                                 require_once($CFG->dirroot.'/'.$dir.'/db/migrate2utf8.php');
 660                                                 $phpfunction($record->id);
 661                                                 if ($debug) {
 662                                                     $db->debug=0;
 663                                                 }
 664                                             break;
 665
 666                                             default:    //no_conv, don't do anything ;-)
 667                                             break;
 668                                         }
 669                                     }
 670                                     $counter++;
 671                                     if ($maxrecords) {
 672                                         if ($processedrecords == $maxrecords) {
 673                                             notify($maxrecords.' records processed. Migration Process halted');
 674                                             print_continue('utfdbmigrate.php?confirm=1&amp;maxrecords='.$maxrecords.'&amp;sesskey='.sesskey());
 675                                             print_footer();
 676                                             die();
 677                                         }
 678                                     }
 679
 680                                     $processedrecords++;
 681                                     //print some output once in a while
 682                                     if (($processedrecords) % 1000 == 0) {
 683                                         print_progress($done, $tablestoconvert, 5, 1,
 684                                                        'Processing: '.$dbtablename.'/'.$fieldname.' ');
 685                                     }
 686                                 }
 687                             }else {
 688                                 if ($debug) {
 689                                     notify('no records found!');
 690                                 }
 691                             }
 692                         }   //close the while loop
 693
 694                         /********************
 695                          * Drop index here **
 696                          ********************/
 697
 698                         if ($CFG->dbtype == 'mysql') {
 699
 700                             /*********************************
 701                              * Change column encoding 2 phase*
 702                              *********************************/
 703
 704                             /*
 705                             $SQL = 'ALTER TABLE '.$CFG->prefix.$dbtablename;
 706                             $SQL.= ' CHANGE '.$fieldname.' '.$fieldname.' LONGTEXT';
 707                            // if ($length > 0) {
 708                            //     $SQL.='('.$length.') ';
 709                            // }
 710                             $SQL .= ' CHARACTER SET binary NOT NULL DEFAULT '.$default.';';
 711                             if ($debug) {
 712                                 $db->debug=999;
 713                             }
 714                             if ($fieldname != 'dummy') {
 715                                 execute_sql($SQL, $debug);
 716                             }
 717                             if ($debug) {
 718                                 $db->debug=0;
 719                             }*/
 720                             //phase 2
 721                             $SQL = 'ALTER TABLE '.$prefix.$dbtablename;
 722                             $SQL.= ' CHANGE '.$fieldname.' '.$fieldname.' '.$type;
 723                             if ($length > 0) {
 724                                 $SQL.='('.$length.') ';
 725                             }
 726                             $SQL.=' CHARACTER SET utf8 ' . $notnull . ' DEFAULT '. $default . ';';
 727                             if ($debug) {
 728                                 $db->debug=999;
 729                             }
 730                             if ($fieldname != 'dummy') {
 731                                 execute_sql($SQL, $debug);
 732                             }
 733                             if ($debug) {
 734                                 $db->debug=0;
 735                             }
 736                             /********************************************
 737                              * build an array to add index back together*
 738                              ********************************************/
 739                             if ($addindex){
 740                                 $addindexarray[] = $addindex;
 741                             } else if ($adduniqueindex) {
 742                                 $adduniqueindexarray[] = $adduniqueindex;
 743                             } else if ($addprimary) {
 744                                 $addprimaryarray[] = $addprimary;
 745                             }
 746
 747                         } else {
 748
 749                         //posgresql code here
 750                         //No we don't need to do anything here
 751
 752                         }
 753
 754                     }
 755
 756                     /********************************
 757                      * Adding the index back        *
 758                      ********************************/
 759                     $alter = 0;
 760
 761                     if ($CFG->dbtype=='mysql'){
 762
 763                         $SQL = 'ALTER TABLE '.$prefix.$dbtablename;
 764                     /*
 765                         NOTE: We aren't going to create the indexes back here any more because they
 766                               are going to be recreated at the end of the table iteration with
 767                               the info saved at the begining of it.
 768
 769                         if (!empty($addindexarray)) {
 770                             foreach ($addindexarray as $aidx){
 771                                 $SQL .= ' ADD INDEX '.$aidx.',';
 772                                 $alter++;
 773                             }
 774                         }
 775
 776                         if (!empty($adduniqueindexarray)) {
 777                             foreach ($adduniqueindexarray as $auidx){
 778                                 $SQL .= ' ADD UNIQUE INDEX '.$auidx.',';
 779                                 $alter++;
 780                             }
 781                         }
 782                     */
 783
 784                         if (!empty($addprimaryarray)) {
 785                             foreach ($addprimaryarray as $apm){
 786                                 $SQL .= ' ADD PRIMARY KEY '.$apm.',';
 787                                 $alter++;
 788                             }
 789                         }
 790
 791                         $SQL = rtrim($SQL, ', ');
 792                         $SQL.=';';
 793
 794                     } else {
 795                       ///posgresql code here
 796                       ///No we don't need to do anything here
 797
 798                     }
 799
 800                     if ($alter) {
 801                         if ($debug) {
 802                             $db->debug=999;
 803                         }
 804                         execute_sql($SQL, $debug);
 805                         if ($debug) {
 806                             $db->debug=0;
 807                         }
 808                     }
 809
 810                 } //if there are fields
 811
 812
 813             } /// Point 1 - bypass should end here.
 814
 815
 816             /************************************
 817              * now we modify the table encoding *
 818              ************************************/
 819             if ($CFG->dbtype=='mysql'){
 820                 $SQL = 'ALTER TABLE '.$prefix.$dbtablename.' CHARACTER SET utf8';
 821                 if ($debug) {
 822                     $db->debug=999;
 823                 }
 824                 execute_sql($SQL, $debug);
 825                 if ($debug) {
 826                     $db->debug=0;
 827                 }
 828
 829             } else {
 830
 831                 ///posgresql code here
 832                 ///No we don't need to do anything here
 833             }
 834
 835         /// Recreate all the indexes previously dropped and sent to backup
 836         /// tables. Retrieve information from backup tables
 837             if ($backupindexes = backup_getid(BACKUP_UNIQUE_CODE, $prefix.$dbtablename, 1)) {
 838             /// Confirm we have indexes
 839                 if ($allindexes = $backupindexes->info) {
 840                 /// Recreate all the indexes
 841                     $sqlarr = array();
 842                     foreach ($allindexes as $onekey => $oneindex) {
 843                         $unique = $oneindex['unique']? 'UNIQUE ' : '';
 844                         $sqlarr[] = 'ALTER TABLE '.$prefix.$dbtablename.' ADD '.$unique.'INDEX '.$onekey.
 845                                     ' ('.implode(', ', $oneindex['columns']).')';
 846                     }
 847                     execute_sql_arr($sqlarr, true, $debug);
 848                 }
 849             }
 850         }
 851     }
 852
 853     if ($CFG->dbtype=='mysql') {
 854         /*********************************
 855          * now we modify the db encoding *
 856          *********************************/
 857         $SQL = 'ALTER DATABASE '.$CFG->dbname.' CHARACTER SET utf8';
 858         execute_sql($SQL, $debug);
 859     } else {
 860         if (!is_postgres_utf8()) {
 861             //This old database is now deprecated
 862             set_config('migrated_to_new_db','1');
 863         }
 864     }
 865     delete_records('config','name','dbmigration');    //bye bye
 866
 867     //These have to go!
 868     if ($debug) {
 869         $db->debug=true;
 870     }
 871
 872     if ($CFG->dbtype == 'postgres7') {
 873         $backup_db = $GLOBALS['db'];
 874         $GLOBALS['db'] = &get_postgres_db();
 875     }
 876
 877     execute_sql('TRUNCATE TABLE '.$CFG->prefix.'cache_text', $debug);
 878     execute_sql('TRUNCATE TABLE '.$CFG->prefix.'cache_filters', $debug);
 879
 880     if ($CFG->dbtype == 'postgres7') {
 881         $GLOBALS['db'] = $backup_db;
 882         unset($backup_db);
 883     }
 884     if ($debug) {
 885         $db->debug=0;
 886     }
 887
 888     //update site language
 889     $sitelanguage = get_record('config','name', 'lang');
 890     if (strstr($sitelanguage->value, 'utf8')===false and $sitelanguage->value) {
 891         $sitelanguage->value.='_utf8';
 892         migrate2utf8_update_record('config',$sitelanguage);
 893     }
 894
 895     //finish the javascript bar
 896     $done = $tablestoconvert;
 897     print_progress($done, $tablestoconvert, 5, 1);
 898
 899     //prints the list of langs used in this site
 900     print_simple_box_start('center','50%');
 901     echo '<div align="center">The following Language Packs are needed for your users and courses. Please install the following Language Packs:<br><b>';
 902     $langsused = get_record('config','name', 'langsused');
 903     $langs = explode (',',$langsused->value);
 904
 905     foreach ($langs as $lang) {
 906         if (!empty($lang) and $lang != 'en_utf8') {
 907             echo $lang.', ';
 908         }
 909     }
 910     echo '</b><br/><a href="'.$CFG->wwwroot.'/'.$CFG->admin.'/langimport.php">Language Import Utility</a></div>';
 911     print_simple_box_end();
 912     delete_records('config','name','langsused');
 913
 914     //remove the cache file!
 915     @unlink($CFG->dataroot.'/cache/languages');
 916
 917     //remove backup temp storage
 918     if ($CFG->dbtype = 'mysql') {
 919         $pref->backup_unique_code = BACKUP_UNIQUE_CODE;
 920         clean_temp_data($pref);
 921     }
 922
 923     // Regenerate some cached data
 924
 925     if ($CFG->dbtype == 'mysql') {
 926         $db->Execute("SET NAMES 'utf8'");
 927     } else if ($CFG->dbtype == 'postgres7') {
 928         $db->Execute("SET NAMES 'utf8'");
 929     }
 930
 931     rebuild_course_cache();
 932
 933     //set the final flag
 934     migrate2utf8_set_config('unicodedb','true');    //this is the main flag for unicode db
 935
 936      //echo date("H:i:s");
 937 }
 938
 939
 940 /* returns the course lang
 941  * @param int courseid
 942  * @return string
 943  */
 944 function get_course_lang($courseid) {
 945
 946     static $coursecache;
 947
 948     if (!isset($coursecache[$courseid])) {
 949         if ($course = get_record('course','id',$courseid)){
 950             $coursecache[$courseid] = $course->lang;
 951             return $course->lang;
 952         }
 953         return false;
 954     } else {
 955         return $coursecache[$courseid];
 956     }
 957 }
 958
 959 /* returns the teacher's lang
 960  * @param int courseid
 961  * @return string
 962  */
 963 function get_main_teacher_lang($courseid) {
 964     //editting teacher > non editting teacher
 965     global $CFG;
 966     static $mainteachercache;
 967
 968     if ($courseid == SITEID || $courseid==0) {
 969         $admin = get_admin();
 970         $mainteachercache[$courseid] = $admin->lang;
 971         return $admin->lang;
 972     }
 973
 974     if (!isset($mainteachercache[$courseid])) {
 975
 976         /// this is a worse guess
 977         if (!empty($CFG->rolesactive)) {
 978
 979             $context = get_context_instance(CONTEXT_COURSE, $courseid);
 980             $teachers = get_users_by_capability($context, 'moodle/legacy:editingteacher', 'u.id, u.lang', 'ra.id ASC', 0, 1); // only need first one
 981             if (is_array($teachers)) {
 982                 $teacher = reset($teachers);
 983                 $mainteachercache[$courseid] = $teacher->lang;
 984                 return $teacher->lang;
 985             }
 986         /// this is a better guess
 987         } else {
 988
 989             $SQL = 'SELECT u.id, u.lang
 990                     FROM '.$CFG->prefix.'user_teachers ut,
 991                          '.$CFG->prefix.'course c,
 992                          '.$CFG->prefix.'user u
 993                     WHERE c.id = ut.course AND
 994                           ut.course = '.$courseid.' AND
 995                           u.id = ut.userid
 996                     ORDER BY ut.authority ASC';
 997
 998             if ($teacher = get_record_sql($SQL, true)) {
 999                 $mainteachercache[$courseid] = $teacher->lang;
1000                 return $teacher->lang;
1001             }
1002         }
1003     /// Arriving here we haven't been able to select any teacher lang
1004     /// so use the admin lang
1005         $admin = get_admin();
1006         $mainteachercache[$courseid] = $admin->lang;
1007         return $admin->lang;
1008     } else {
1009         return $mainteachercache[$courseid];
1010     }
1011 }
1012
1013 function get_original_encoding($sitelang, $courselang, $userlang){
1014
1015     global $CFG, $enc;
1016
1017     $lang = '';
1018     if ($courselang) {
1019         $lang = $courselang;
1020     }
1021     else if ($userlang) {
1022         $lang = $userlang;
1023     }
1024     else if ($sitelang) {
1025         $lang = $sitelang;
1026     }
1027     else {
1028         error ('no language found!');
1029     }
1030
1031     if ($enc[$lang]) {
1032         return $enc[$lang];
1033     } else {
1034         notify ('unknown language detected: '.$lang);
1035         return false;
1036     }
1037 }
1038
1039 /* returns the user's lang
1040  * @param int userid
1041  * @return string
1042  */
1043 function get_user_lang($userid) {
1044
1045     static $usercache;
1046
1047     if (!isset($usercache[$userid])) {
1048         if ($user = get_record('user','id',$userid)) {
1049             $usercache[$userid] = $user->lang;
1050             return $user->lang;
1051         }
1052     } else {
1053         return $usercache[$userid];
1054     }
1055     return false;
1056 }
1057
1058 // a placeholder for now
1059 function log_the_problem_somewhere() {  //Eloy: Nice function, perhaps we could use it, perhpas no. :-)
1060     global $CFG, $dbtablename, $fieldname, $record;
1061     if ($CFG->debug>7) {
1062         echo "<br />Problem converting: $dbtablename -> $fieldname -> {$record->id}!";
1063     }
1064 }
1065
1066 // only this function should be used during db migraton, because of addslashes at the end of the convertion
1067 function utfconvert($string, $enc, $slash=true) {
1068     global $textlib;
1069     if ($result = $textlib->convert($string, $enc)) {
1070         if ($slash) {
1071             $result = addslashes($result);
1072         }
1073     }
1074     return $result;
1075 }
1076
1077 function validate_form(&$form, &$err) {
1078     global $CFG;
1079
1080     $newdb = &ADONewConnection('postgres7');
1081     error_reporting(0);  // Hide errors
1082     $dbconnected = $newdb->Connect($form->dbhost,$form->dbuser,$form->dbpass,$form->dbname);
1083     error_reporting($CFG->debug);  // Show errors
1084     if (!$dbconnected) {
1085         $err['dbconnect'] = get_string('dbmigrateconnecerror', 'admin');
1086         return;
1087     }
1088
1089     if (!is_postgres_utf8($newdb)) {
1090         $encoding = $newdb->GetOne('SHOW server_encoding');
1091         $err['dbconnect'] = get_string('dbmigrateencodingerror', 'admin', $encoding);
1092         return;
1093     }
1094
1095     if (!empty($form->pathtopgdump) && !is_executable($form->pathtopgdump)) {
1096         $err['pathtopgdump'] = get_string('pathtopgdumpinvalid','admin');
1097         return;
1098     }
1099
1100     if (!empty($form->pathtopsql) && !is_executable($form->pathtopsql)) {
1101         $err['pathtopsql'] = get_string('pathtopsqlinvalid','admin');
1102         return;
1103     }
1104
1105     return;
1106 }
1107
1108 function is_postgres_utf8($thedb = null) {
1109     if ($thedb === null) {
1110         $thedb = &$GLOBALS['db'];
1111     }
1112
1113     $db_encoding_postgres = $thedb->GetOne('SHOW server_encoding');
1114     if (strtoupper($db_encoding_postgres) == 'UNICODE' || strtoupper($db_encoding_postgres) == 'UTF8') {
1115         return true;
1116     } else {
1117         return false;
1118     }
1119 }
1120
1121 function &get_postgres_db() {
1122     static $postgres_db;
1123
1124     if (!$postgres_db) {
1125         if (is_postgres_utf8()) {
1126             $postgres_db = &$GLOBALS['db'];
1127         } else {
1128             $postgres_db = &ADONewConnection('postgres7');
1129             $postgres_db->Connect($_SESSION['newpostgresdb']->dbhost,$_SESSION['newpostgresdb']->dbuser,$_SESSION['newpostgresdb']->dbpass,$_SESSION['newpostgresdb']->dbname);
1130         }
1131     }
1132
1133     return $postgres_db;
1134 }
1135
1136 function is_postgres_setup() {
1137     $postgres_db = &get_postgres_db();
1138
1139     return $GLOBALS['db']->MetaTables() == $postgres_db->MetaTables();
1140 }
1141
1142 function migrate2utf8_update_record($table,$record) {
1143     global $CFG;
1144
1145     if ($CFG->dbtype == 'mysql') {
1146         update_record($table,$record);
1147     } else {
1148         $backup_db = $GLOBALS['db'];
1149         $GLOBALS['db'] = &get_postgres_db();
1150         global $in;
1151         $in = true;
1152         update_record($table,$record);
1153         $GLOBALS['db'] = $backup_db;
1154     }
1155 }
1156
1157 function migrate2utf8_set_config($name, $value, $plugin=NULL) {
1158     global $CFG;
1159     if ($CFG->dbtype == 'mysql') {
1160         set_config($name, $value, $plugin);
1161     } else {
1162         $backup_db = $GLOBALS['db'];
1163         $GLOBALS['db'] = &get_postgres_db();
1164         set_config($name, $value, $plugin);
1165         $GLOBALS['db'] = $backup_db;
1166     }
1167 }
1168
1169 // this needs to print an error when a mod does not have a migrate2utf8.xml
1170 function utf_get_xml ($mode=0) { // if mode is 1, do not perform check for script validity
1171     global $CFG;
1172
1173     $xmls = array();
1174     $noscript = 0; // we assume all mod and all blocks have migration scripts
1175
1176     /*****************************************************************************
1177      * traverse order is mod->backup->block->block_plugin->enroll_plugin->global *
1178      *****************************************************************************/
1179
1180     ///mod
1181     if (!$mods = get_list_of_plugins('mod')) {
1182         error('No modules installed!');
1183     }
1184
1185     foreach ($mods as $mod){
1186         if (file_exists($CFG->dirroot.'/mod/'.$mod.'/db/migrate2utf8.xml')) {
1187             $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/mod/'.$mod.'/db/migrate2utf8.xml'));
1188         } else if (!$mode) {
1189             $noscript = 1;
1190             notify('warning, there is no migration script detected for this module - '.$mod);
1191         }
1192     }
1193
1194     ///Backups
1195     $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/backup/db/migrate2utf8.xml'));
1196
1197     ///Blocks
1198     $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/blocks/db/migrate2utf8.xml'));
1199
1200     ///Block Plugins
1201     if (!$blocks = get_list_of_plugins('blocks')) {
1202         //error('No blocks installed!');    //Eloy: Is this a cause to stop?
1203     }
1204
1205     foreach ($blocks as $block){
1206         if (file_exists($CFG->dirroot.'/blocks/'.$block.'/db/migrate2utf8.xml')) {
1207             $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/blocks/'.$block.'/db/migrate2utf8.xml'));
1208         } else if (!$mode) {
1209             if (file_exists($CFG->dirroot.'/blocks/'.$block.'/db/mysql.sql') && filesize($CFG->dirroot.'/blocks/'.$block.'/db/mysql.sql')) { // if no migration script, and have db script, we are in trouble
1210                 notify('warning, there is no migration script detected for this block - '.$block);
1211                 $noscript = 1;
1212             }
1213         }
1214     }
1215
1216     ///Enrol
1217
1218     if (!$enrols = get_list_of_plugins('enrol')) {
1219         //error('No enrol installed!');   //Eloy: enrol, not blocks :-) Is this a cause to stop?
1220     }
1221
1222     foreach ($enrols as $enrol){
1223         if (file_exists($CFG->dirroot.'/enrol/'.$enrol.'/db/migrate2utf8.xml')) {
1224            $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/enrol/'.$enrol.'/db/migrate2utf8.xml'));
1225         }
1226     }
1227
1228     ///Lastly, globals
1229
1230     $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/lib/db/migrate2utf8.xml'));
1231
1232     if ($noscript) {
1233         notify ('Some of your modules or Blocks do not have a migration script. It is very likely that these are contrib modules. If your Moodle site uses non-UTF8 language packs and non-en language packs, data inside these moduels or blocks will not be displayed correctly after the migration. Please proceed with caution.');
1234     }
1235
1236     return $xmls;
1237
1238 }