admin/utfdbmigrate.php

   1 <?php //$Id$
   2
   3 ///dummy field names are used to help adding and dropping indexes. There's only 1 case now, in scorm_scoes_track
   4
   5     require_once('../config.php');
   6     require_once($CFG->libdir.'/adminlib.php');
   7     require_once($CFG->libdir.'/environmentlib.php');
   8     require_once($CFG->dirroot.'/course/lib.php');
   9     require_login();
  10
  11         // decalre once
  12         global $enc;
  13
  14         $customlang = array();
  15
  16         $enc = array('af' => 'iso-8859-1', 'ar' => 'windows-1256', 'be' => 'windows-1251', 'bg' => 'windows-1251', 'bs' => 'windows-1250', 'ca' => 'iso-8859-1', 'cs' => 'iso-8859-2', 'da' => 'iso-8859-1', 'de' => 'iso-8859-1', 'de_du' => 'iso-8859-1', 'de_utf8' => 'utf-8', 'el' => 'windows-1253', 'en' => 'iso-8859-1', 'en_ja' => 'euc-jp', 'en_us' => 'iso-8859-1', 'en_utf8' => 'utf-8', 'es' => 'iso-8859-1', 'es_ar' => 'iso-8859-1', 'es_es' => 'iso-8859-1', 'es_mx' => 'iso-8859-1', 'et' => 'iso-8859-1', 'eu' => 'iso-8859-1', 'fa' => 'windows-1256', 'fa_utf8' => 'utf-8', 'fi' => 'iso-8859-1', 'fil' => 'iso-8859-15', 'fr' => 'iso-8859-1', 'fr_ca' => 'iso-8859-15', 'ga' => 'iso-8859-1', 'gl' => 'iso-8859-1', 'he' => 'ISO-8859-8-I', 'he_utf8' => 'utf-8', 'hi' => 'iso-8859-1', 'hr' => 'windows-1250', 'hr_utf8' => 'utf-8', 'hu' => 'iso-8859-2', 'id' => 'iso-8859-1', 'is' => 'iso-8859-1', 'it' => 'iso-8859-1', 'ja' => 'EUC-JP', 'ja_utf8' => 'UTF-8', 'ka_utf8' => 'UTF-8', 'km_utf8' => 'UTF-8', 'kn_utf8' => 'utf-8', 'ko' => 'EUC-KR', 'ko_utf8' => 'UTF-8', 'lt' => 'windows-1257', 'lv' => 'ISO-8859-4', 'mi_nt' => 'iso-8859-1', 'mi_tn_utf8' => 'utf-8', 'ms' => 'iso-8859-1', 'nl' => 'iso-8859-1', 'nn' => 'iso-8859-1', 'no' => 'iso-8859-1', 'no_gr' => 'iso-8859-1', 'pl' => 'iso-8859-2', 'pt' => 'iso-8859-1', 'pt_br' => 'iso-8859-1', 'ro' => 'iso-8859-2', 'ru' => 'windows-1251', 'sk' => 'iso-8859-2', 'sl' => 'iso-8859-2', 'sl_utf8' => 'utf-8', 'so' => 'iso-8859-1', 'sq' => 'iso-8859-1', 'sr_utf8' => 'utf-8', 'sv' => 'iso-8859-1', 'th' => 'TIS-620', 'th_utf8' => 'UTF-8', 'tl' => 'iso-8859-15', 'tl_utf8' => 'UTF-8', 'tr' => 'iso-8859-9', 'uk' => 'windows-1251', 'vi_utf8' => 'UTF-8', 'zh_cn' => 'GB18030', 'zh_cn_utf8' => 'UTF-8', 'zh_tw' => 'Big5', 'zh_tw_utf8' => 'UTF-8');
  17
  18         /**************************************
  19          * Custom lang pack handling              *
  20          **************************************/
  21
  22         // scan list of langs, including customs packs
  23         $langs = get_list_of_languages();
  24
  25         // foreach lang
  26         foreach ($langs as $lang => $lang1) {
  27
  28                 if (in_array($lang, array_keys($enc))) {
  29                         // if already in array, ignore
  30                         continue;
  31                 }
  32
  33                 // if this lang has got a charset
  34
  35                 if ($result = get_string_from_file('thischarset',$CFG->dirroot.'/lang/'.$lang.'/moodle.php', "\$resultstring")) {
  36                         eval($result);
  37                         $enc[$lang] = $resultstring;
  38                 } else if ($result = get_string_from_file('parentlanguage',$CFG->dirroot.'/lang/'.$lang.'/moodle.php',"\$resultstring")) {
  39                 // else if there's a parent lang we can use
  40                     eval($result);
  41                         $enc[$lang] = $enc[$resultstring];
  42                 } else {
  43                         notify ('unknown lang pack detected '.$lang);
  44                 }
  45
  46         }
  47
  48         /**************************************
  49          * End custom lang pack handling      *
  50          **************************************/
  51
  52     if (!isadmin()) {
  53         error('Only admins can access this page');
  54     }
  55
  56     if (!$site = get_site()) {
  57         redirect('index.php');
  58     }
  59
  60     if (!empty($CFG->unicodedb)) {
  61         error ('unicode db migration has already been performed!');
  62     }
  63
  64     $migrate = optional_param('migrate', 0, PARAM_BOOL);
  65     $confirm = optional_param('confirm', 0, PARAM_BOOL);
  66
  67     $textlib = textlib_get_instance();
  68
  69     $stradministration   = get_string('administration');
  70     $strdbmigrate = get_string('dbmigrate','admin');
  71
  72     $filename = $CFG->dataroot.'/'.SITEID.'/maintenance.html';    //maintenance file
  73
  74     print_header("$site->shortname: $stradministration", "$site->fullname",
  75                  '<a href="index.php">'. "$stradministration</a> -> $strdbmigrate");
  76
  77     print_heading($strdbmigrate);
  78
  79     if ($CFG->dbtype == 'postgres7') {
  80         $CFG->pagepath = 'admin/utfdbmigrate/postgresql';
  81     }
  82     //if $confirm
  83     if ($confirm && confirm_sesskey()) {
  84         //do the real migration of db
  85         print_simple_box_start('center','50%');
  86         print_string('importlangreminder','admin');
  87         print_simple_box_end();
  88         db_migrate2utf8();
  89         print_heading('db unicode migration has been completed!');
  90         if (!get_config('', 'migrate_maintmode')) { // already in maint mode
  91             unlink($filename);    //no longer in maintenance mode
  92         }
  93         delete_records('config', 'name', 'migrate_maintmode');
  94         @require_logout();
  95         print_continue($CFG->wwwroot.'/'.$CFG->admin.'/langimport.php');
  96     }
  97
  98     //else if $migrate
  99     else if ($migrate && confirm_sesskey()) {
 100         if ($CFG->dbtype == 'postgres7' && !is_postgres_utf8()) {
 101             $continue = false;
 102             if (($form = data_submitted()) && isset($form->dbhost)) {
 103                 validate_form($form, $err);
 104
 105                 if (count($err) == 0) {
 106                     $_SESSION['newpostgresdb'] = $form;
 107                     $continue = true;
 108                 }
 109             }
 110         } else {
 111             $continue = true;
 112         }
 113         if ($continue) {
 114             echo '<div align="center">';
 115             print_simple_box_start('center','50%');
 116             print_string('dbmigratewarning2','admin');
 117             print_simple_box_end();
 118             //put the site in maintenance mode
 119             check_dir_exists($CFG->dataroot.'/'.SITEID, true);
 120
 121             if (file_exists($filename)) {
 122                 set_config('migrate_maintmode', 1); // already in maintenance mode
 123             } else {
 124                 if (touch($filename)) {
 125                     $file = fopen($filename, 'w');
 126                     fwrite($file, get_string('maintinprogress','admin'));
 127                     fclose($file);
 128                 } else {
 129                     notify (get_string('maintfileopenerror','admin'));
 130                 }
 131             }
 132             //print second confirmation box
 133             echo '<form name="migratefrom" action="utfdbmigrate.php" method="POST">';
 134             echo '<input name="confirm" type="hidden" value="1" />';
 135             echo '<input name="sesskey" type="hidden" value="'.sesskey().'" />';
 136
 137             $xmls = utf_get_xml();
 138             $sumrecords = 0;   //this is the sum of all records of relavent tables.
 139             foreach ($xmls as $xml) {    ///foreach xml file, we must get a list of tables
 140                 $dbtables = $xml['DBMIGRATION']['#']['TABLES'][0]['#']['TABLE'];    //real db tables
 141
 142                 foreach ($dbtables as $dbtable) {
 143                     $dbtablename = $dbtable['@']['name'];
 144
 145                     if ($dbtablename=='adodb_logsql') {
 146                         $prefix = '';
 147                     } else {
 148                         $prefix = $CFG->prefix;
 149                     }
 150                     $sumrecords += count_records_sql("SELECT COUNT(*) FROM {$prefix}$dbtablename");
 151                 }
 152             }
 153             echo 'Total number of records in your database is <b>'.$sumrecords.'</b>';
 154             if ($sumrecords > 10000) {
 155                 echo '<br />Number of Records to process before halting (Leave blank for no limit) <input name="maxrecords" type="text" value="" />';
 156             }
 157
 158             //print the "i-know-what-lang-to-use" menu
 159
 160             echo '<br />The whole site is in this encoding: (leave blank if you are not sure)';
 161             echo '<select name="globallang"><option value="">I am not sure</option>';
 162             foreach ($enc as $lang => $encoding) {
 163                 echo '<option value="'.$encoding.'">'.$lang.'</option>';
 164             }
 165             echo '</select>';
 166
 167             echo '<p /><input type="submit" value="'.get_string('continue').'"/>';
 168             echo '<input type="button" value="'.get_string('cancel').'" onclick="javascript:history.go(-1)" />';
 169             echo '</form>';
 170             echo '</div>';
 171
 172         } else {
 173             echo '<div align="center">';
 174             print_simple_box_start('center','50%');
 175             print_string('dbmigratepostgres','admin');
 176             print_simple_box_end();
 177
 178             print_simple_box_start("center", "");
 179             include("utfdbmigrate.html");
 180             print_simple_box_end();
 181         }
 182     }
 183
 184     else {    //else, print welcome to migrate page message
 185         echo '<div align="center">';
 186         print_simple_box_start('center','50%');
 187         print_string('dbmigratewarning','admin');
 188         print_simple_box_end();
 189
 190         /*************************************
 191          * Eloy's environement checking code *
 192          *************************************/
 193
 194         $current_version = $CFG->release;
 195
 196     /// Gather and show results
 197         $status = check_moodle_environment($current_version, $environment_results);
 198
 199         //end of env checking
 200
 201     /// We only allow to continue if environmental checks have been passed ok
 202         if ($status) {
 203             echo '<form name="migratefrom" action="utfdbmigrate.php" method="POST">';
 204             echo '<input name="migrate" type="hidden" value="1" />';
 205             echo '<input name="sesskey" type="hidden" value="'.sesskey().'" />';
 206             echo '<input type="submit" value="'.get_string('continue').'"/>';
 207             echo '&nbsp;<input type="button" value="'.get_string('cancel').'" onclick="javascript:history.go(-1)" />';
 208             echo '</form>';
 209             echo '</div>';
 210         }
 211     }
 212
 213     print_footer();
 214
 215
 216 function db_migrate2utf8(){   //Eloy: Perhaps some type of limit parameter here
 217                               //pointing to the num of records to process would be
 218                                                           //useful. And it won't break anything, because the
 219                                                           //crash system will continue the next time it was
 220                                                           //executed. Also, the function could return:
 221                                                           //0 = Some sort of error
 222                                                           //1 = Finished completelly!
 223                                                           //2 = Finished limit records
 224                                                           //(using constants, of course ;-))
 225                                                           //Then, all those errors, should be converted to
 226                                                           //mtrace() and return 0. (showing the current
 227                                                           //table/field/recordid)
 228
 229     global $db, $CFG, $dbtablename, $fieldname, $record, $processedrecords;
 230     $debug = ($CFG->debug > 7);
 231
 232      //echo date("H:i:s");
 233
 234     ignore_user_abort(false); // see bug report 5352. This should kill this thread as soon as user aborts.
 235
 236     @set_time_limit(0);
 237     @ob_implicit_flush(true);
 238     @ob_end_flush();
 239
 240     $maxrecords = optional_param('maxrecords', 0, PARAM_INT);
 241     $globallang = optional_param('globallang', '', PARAM_FILE);
 242     $processedrecords = 0;
 243
 244     $ignoretables = array();    //list of tables to ignore, optional
 245
 246     //one gigantic array to hold all db table information read from all the migrate2utf8.xml file.
 247     require_once($CFG->dirroot.'/lib/xmlize.php');
 248     $xmls = utf_get_xml(1);
 249     $tablestoconvert = 0; // total number of tables to convert
 250     foreach ($xmls as $xml) {    ///foreach xml file, we must get a list of tables
 251         $dbtables = $xml['DBMIGRATION']['#']['TABLES'][0]['#']['TABLE'];    //real db tables
 252         foreach ($dbtables as $dbtable) {
 253             $tablestoconvert++;
 254         }
 255     }
 256     // progress bar handling
 257     // first let's find out how many tables there are
 258
 259     $done = 0;
 260     print_progress($done, $tablestoconvert, 5, 1);
 261
 262
 263     $textlib = textlib_get_instance();    //only 1 reference
 264
 265     //if unicodedb is set, migration is complete. die here;
 266     if (!$crash = get_record('config','name','dbmigration')) {
 267         //Duplicate the database if not unicode for postgres7
 268         if ($CFG->dbtype == 'postgres7' && !is_postgres_utf8() && !is_postgres_setup()) {
 269             echo '<script>';
 270             echo 'document.getElementById("text").innerHTML = "Copying data to the UTF8 database for processing...";'."\n";
 271             echo '</script>';
 272
 273             if ($_SESSION['newpostgresdb']->dbcluster) {
 274                 $cluster = ' --cluster ' . $_SESSION['newpostgresdb']->dbcluster;
 275             } else {
 276                 $cluster = '';
 277             }
 278             $pgdump = 'pg_dump';
 279             if (!empty($_SESSION['newpostgresdb']->pathtopgdump)) {
 280                 $pgdump = $_SESSION['newpostgresdb']->pathtopgdump;
 281             }
 282             $psql = 'psql';
 283             if (!empty($_SESSION['newpostgresdb']->pathtopsql)) {
 284                 $pgsql = $_SESSION['newpostgresdb']->pathtopsql;
 285             }
 286
 287             $cmd = "PGPASSWORD={$CFG->dbpass} PGCLIENTENCODING='UNICODE' PGDATABASE={$CFG->dbname} $pgdump -Fp -O -x -U {$CFG->dbuser}$cluster";
 288             if ($CFG->dbhost)  {
 289                 $host = split(":", $CFG->dbhost);
 290                 if ($host[0]) $cmd .= " -h {$host[0]}";
 291                 if (isset($host[1])) $cmd .= " -p {$host[1]}";
 292             }
 293             $cmds[] = $cmd;
 294             $cmds[] = 'grep -v "COMMENT ON SCHEMA"';
 295             $cmds[] = 'iconv -f UTF-8 -t UTF-8 -c';
 296             $cmd = "PGPASSWORD={$_SESSION['newpostgresdb']->dbpass} PGDATABASE={$_SESSION['newpostgresdb']->dbname} $psql -q -U {$_SESSION['newpostgresdb']->dbuser} -v ON_ERROR_STOP=1$cluster";
 297             if ($_SESSION['newpostgresdb']->dbhost)  {
 298                 $host = split(":", $_SESSION['newpostgresdb']->dbhost);
 299                 if ($host[0]) $cmd .= " -h {$host[0]}";
 300                 if (isset($host[1])) $cmd .= " -p {$host[1]}";
 301             }
 302             $cmds[] = $cmd;
 303             foreach ($cmds as $key => $cmd) {
 304                 $files[] = tempnam($CFG->dataroot, 'utf8_');
 305                 $cmd = $cmd . ($key?" < {$files[$key-1]}":'') . " 2>&1 > {$files[$key]}";
 306                 if (stripos(PHP_OS, 'darwin') !== false && stripos($cmd,'iconv') !== false) {
 307                     // I know this looks DREADFULLY hackish, but the iconv in mac os x seems to have a return code of 1 for warnings
 308                     // and I cannot figure out why, it's a very different version of iconv to most *nix versions, even seems to be a
 309                     // different gnu project.
 310                     // If someone can figure out a better way to do this, FEEL FREE :)
 311                     // - Penny
 312                     $cmd .= ' || true';
 313                 }
 314                 exec($cmd, $output, $return_var);
 315                 if ($key) {
 316                     unlink($files[$key-1]);
 317                 }
 318                 if ($return_var) { // we are dead!
 319                     unlink($files[$key]);
 320                     echo '<br />';
 321                     print_simple_box_start('center','50%');
 322                     print_string('dbmigrationdupfailed','admin',htmlspecialchars(implode("\n", $output)));
 323                     print_simple_box_end();
 324                     print_footer();
 325                     exit;
 326                 }
 327             }
 328             unlink(array_pop($files));
 329         }
 330
 331         $migrationconfig = new object;
 332         $migrationconfig->name = 'dbmigration';
 333         $migrationconfig->value = '-1';
 334         insert_record('config',$migrationconfig);  //process initiated
 335
 336         //langs used, to help make recommendations on what lang packs to install
 337         $langsused = new object;
 338         $langsused->name = 'langsused';
 339         $langsused->value = '';
 340         insert_record('config',$langsused);
 341
 342     } else {
 343
 344         $crashdata = explode('##',$crash->value);
 345         $crash->table = $crashdata[0];
 346         $crash->field = $crashdata[1];
 347         $crash->record = $crashdata[2];
 348
 349         notify("Resuming migration from: $crash->table / .$crash->field, Record: $crash->record");
 350     }
 351
 352     /************************************************************************
 353      * Now we got all our tables in order                                   *
 354      ************************************************************************/
 355
 356     foreach ($xmls as $xml) {    ///foreach xml file, we must get a list of tables
 357         $dir = $xml['DBMIGRATION']['@']['type'];
 358         $dbtables = $xml['DBMIGRATION']['#']['TABLES'][0]['#']['TABLE'];    //real db tables
 359
 360         foreach ($dbtables as $dbtable) {
 361
 362             $done++;
 363             print_progress($done, $tablestoconvert, 5, 1);
 364
 365             $dbtablename = $dbtable['@']['name'];
 366
 367             // exception handling for adodb_logsql
 368             // see bug 5003
 369             if ($dbtablename == 'adodb_logsql') {
 370                 $prefix = '';
 371             } else {
 372                 $prefix = $CFG->prefix;
 373             }
 374
 375             if ($crash && ($dbtablename != $crash->table)) {  //resuming from crash
 376                 $done++; // need to update progress bar
 377                 continue;
 378             }
 379
 380             if ($debug) {
 381                 print_heading("<br><b>Processsing db table ".$dbtablename.'...</b>');
 382             }
 383
 384             /**********************************************************
 385              * This is the by pass structure. It allows us to process *
 386              * tables on row basis instead of column/field basis      *
 387              * It relies on a single function in migrate2utf8.php     *
 388              **********************************************************/
 389
 390             /// first, check to see if there's a function for the whole table. By pass point (1)
 391             if (file_exists($CFG->dirroot.'/'.$dir.'/db/migrate2utf8.php')) {
 392                 require_once($CFG->dirroot.'/'.$dir.'/db/migrate2utf8.php');
 393                 // this is a function to process table on role basis, e.g. user table in moodorg
 394                 $tablefunction = 'migrate2utf8_'.$dbtablename;
 395             }
 396             if ($CFG->dbtype=='mysql' && function_exists($tablefunction)) {
 397                 $tablefunction($dbtable['#']['FIELDS'][0]['#']['FIELD'], $crash, $debug, $maxrecords, $done, $tablestoconvert); // execute it.
 398             } else {
 399
 400             /******************************************************
 401              * No function for converting whole table, we proceed *
 402              ******************************************************/
 403
 404                 if (!empty($dbtable['#']) && ($fields = $dbtable['#']['FIELDS'][0]['#']['FIELD']) and (!in_array($dbtablename, $ignoretables))) {
 405
 406                     $colnames = array();
 407                     $coltypes = array();    //array to hold all column types for the table
 408                     $collengths = array();    //array to hold all column lengths for the table
 409                     $defaults = array();    //array to hold defaults, if any
 410                     //reset holders
 411                     $addindexarray = array();
 412                     $adduniqueindexarray = array();
 413                     $addprimaryarray = array();
 414
 415                     foreach ($fields as $field){
 416
 417                         //if in crash state, and field name is not the same as crash field name
 418
 419                         $fieldname = isset($field['@']['name'])?$field['@']['name']:"";
 420                         $method = isset($field['@']['method'])?$field['@']['method']:"";
 421                         $type = isset($field['@']['type'])?$field['@']['type']:"";
 422                         $length = isset($field['@']['length'])?$field['@']['length']:"";
 423
 424                         if ($crash && ($crash->field != $fieldname)) {
 425                             continue;
 426                         }
 427
 428                         $dropindex = isset($field['@']['dropindex'])?$field['@']['dropindex']:"";
 429                         $addindex = isset($field['@']['addindex'])?$field['@']['addindex']:"";
 430                         $adduniqueindex = isset($field['@']['adduniqueindex'])?$field['@']['adduniqueindex']:"";
 431
 432                         $dropprimary = isset($field['@']['dropprimary'])?$field['@']['dropprimary']:"";
 433                         $addprimary = isset($field['@']['addprimary'])?$field['@']['addprimary']:"";
 434                         $default = isset($field['@']['default'])?"'".$field['@']['default']."'":"''";
 435
 436                         if ($fieldname != 'dummy') {
 437                             $colnames[] = $fieldname;
 438                             $coltypes[] = $type;
 439                             $collengths[]= $length;
 440                         }
 441
 442                         if ($debug) {
 443                             echo "<br>--><b>processing db field ".$fieldname.'</b>';
 444                             echo "<br>---><b>method ".$method.'</b>';
 445                         }
 446
 447
 448                         if ($CFG->dbtype == 'mysql') {
 449
 450                             /* Drop the index, because with index on, you can't change it to longblob */
 451
 452                             if ($dropindex){    //drop index if index is varchar, text etc type
 453                                 $SQL = 'ALTER TABLE '.$prefix.$dbtablename.' DROP INDEX '.$dropindex.';';
 454                                 $SQL1 = 'ALTER TABLE '.$prefix.$dbtablename.' DROP INDEX '.$CFG->prefix.$dropindex.';'; // see bug 5205
 455                                 if ($debug) {
 456                                     $db->debug=999;
 457                                 }
 458                                 execute_sql($SQL, false); // see bug 5205
 459                                 execute_sql($SQL1, false); // see bug 5205
 460
 461                                 if ($debug) {
 462                                     $db->debug=0;
 463                                 }
 464                             } else if ($dropprimary) {    // drop primary key
 465                                 $SQL = 'ALTER TABLE '.$prefix.$dbtablename.' DROP PRIMARY KEY;';
 466                                 if ($debug) {
 467                                     $db->debug=999;
 468                                 }
 469                                 execute_sql($SQL, $debug);
 470                                 if ($debug) {
 471                                     $db->debug=0;
 472                                 }
 473                             }
 474
 475                             /* Change to longblob, serves 2 purposes:
 476                                1. column loses encoding, so when we finally change it to unicode,
 477                                   mysql does not do a double convertion
 478                                2. longblobs puts no limit (ok, not really but it's large enough)
 479                                   to handle most of the problems such as in bug 5194
 480                             */
 481
 482                             $SQL = 'ALTER TABLE '.$prefix.$dbtablename;
 483                             $SQL.= ' CHANGE '.$fieldname.' '.$fieldname.' LONGBLOB';
 484
 485                             /*
 486                             if ($length > 0) {
 487                                 $SQL.='('.$length.') ';
 488                             }
 489                             $SQL .= ' CHARACTER SET binary NOT NULL DEFAULT '.$default.';';
 490                             */
 491                             if ($debug) {
 492                                 $db->debug=999;
 493                             }
 494                             if ($fieldname != 'dummy') {
 495                                 execute_sql($SQL, $debug);
 496                             }
 497                             if ($debug) {
 498                                 $db->debug=0;
 499                             }
 500
 501                         }
 502
 503                         $patterns[]='/RECORDID/';    //for preg_replace
 504                         $patterns[]='/\{\$CFG\-\>prefix\}/i';    //same here
 505
 506                         if ($method == 'PLAIN_SQL_UPDATE') {
 507                             $sqldetectuser = $field['#']['SQL_DETECT_USER'][0]['#'];
 508                             $sqldetectcourse = $field['#']['SQL_DETECT_COURSE'][0]['#'];
 509                         }
 510                         else if ($method == 'PHP_FUNCTION') {
 511                             $phpfunction = 'migrate2utf8_'.$dbtablename.'_'.$fieldname;
 512                         }
 513
 514                         ///get the total number of records for this field
 515
 516                         // could not use count_records because it addes prefix to adodb_logsql
 517                         $totalrecords = count_records_sql("select count(*) from {$prefix}$dbtablename");
 518                         $counter = 0;
 519                         $recordsetsize = 50;
 520
 521                         if ($crash) {    //if resuming from crash
 522                             //find the number of records with id smaller than the crash id
 523                             $indexSQL = 'SELECT COUNT(*) FROM '.$prefix.$dbtablename.' WHERE id < '.$crash->record;
 524                             $counter = count_records_sql($indexSQL);
 525                         }
 526
 527                         if ($debug) {
 528                             echo "<br>Total number of records is ..".$totalrecords;
 529                             echo "<br/>Counter is $counter";
 530                         }
 531
 532
 533                         /**************************
 534                          * converting each record *
 535                          **************************/
 536                         while(($counter < $totalrecords) and ($fieldname !='dummy') and ($method!='NO_CONV')) {    //while there is still something
 537                             $SQL = 'SELECT * FROM '.$prefix.$dbtablename.' ORDER BY id ASC '.sql_paging_limit($counter, $recordsetsize);
 538                             if ($records = get_records_sql($SQL)) {
 539                                 foreach ($records as $record) {
 540
 541                                     //if we are up this far, either no crash, or crash with same table, field name.
 542                                     if ($crash){
 543                                         if ($crash->record != $record->id) {    //might set to < just in case record is deleted
 544                                             continue;
 545                                         } else {
 546                                             $crash = 0;
 547                                         }
 548                                     }
 549
 550                                     $migrationconfig = get_record('config','name','dbmigration');
 551                                     $migrationconfig->name = 'dbmigration';
 552                                     $migrationconfig->value = $dbtablename.'##'.$fieldname.'##'.$record->id;
 553                                     update_record('config',$migrationconfig);
 554
 555                                     $replacements = array();    //manual refresh
 556                                     $replacements[] = $record->id;
 557                                     $replacements[] = $prefix;
 558
 559                                     if (!empty($record->{$fieldname})) {    //only update if not empty
 560                                         switch ($method){
 561                                             case 'PLAIN_SQL_UPDATE':    //use the 2 statements to update
 562
 563                                                 if ($debug) {
 564                                                     $db->debug=999;
 565                                                 }
 566
 567                                                 //if global lang is set, we just use that
 568
 569                                                 if ($globallang) {
 570                                                     $fromenc = $globallang;
 571                                                 } else {
 572                                                     $userid = get_record_sql(preg_replace($patterns, $replacements, $sqldetectuser));
 573                                                     $courseid = get_record_sql(preg_replace($patterns, $replacements, $sqldetectcourse));
 574
 575                                                     $sitelang   = $CFG->lang;
 576                                                     $courselang = get_course_lang(isset($courseid->course)?$courseid->course:1);
 577                                                     $userlang   = get_user_lang(isset($userid->userid)?$userid->userid:1);
 578
 579                                                     $fromenc = get_original_encoding($sitelang, $courselang, $userlang);
 580                                                 }
 581
 582                                                 //only update if non utf8
 583                                                 if (($fromenc != 'utf-8') && ($fromenc != 'UTF-8')) {
 584                                                     $result = utfconvert($record->{$fieldname}, $fromenc);
 585                                                     $newrecord = new object;
 586                                                     $newrecord->id = $record->id;
 587                                                     $newrecord->{$fieldname} = $result;
 588                                                     migrate2utf8_update_record($dbtablename,$newrecord);
 589                                                 }
 590                                                 if ($debug) {
 591                                                     $db->debug=0;
 592                                                 }
 593                                             break;
 594
 595                                             case 'PHP_FUNCTION':    //use the default php function to execute
 596                                                 if ($debug) {
 597                                                     $db->debug=999;
 598                                                 }
 599                                                 require_once($CFG->dirroot.'/'.$dir.'/db/migrate2utf8.php');
 600                                                 $phpfunction($record->id);
 601                                                 if ($debug) {
 602                                                     $db->debug=0;
 603                                                 }
 604                                             break;
 605
 606                                             default:    //no_conv, don't do anything ;-)
 607                                             break;
 608                                         }
 609                                     }
 610                                     $counter++;
 611                                     if ($maxrecords) {
 612                                         if ($processedrecords == $maxrecords) {
 613                                             notify($maxrecords.' records processed. Migration Process halted');
 614                                             print_continue('utfdbmigrate.php?confirm=1&amp;maxrecords='.$maxrecords.'&amp;sesskey='.sesskey());
 615                                             print_footer();
 616                                             die();
 617                                         }
 618                                     }
 619
 620                                     $processedrecords++;
 621                                     //print some output once in a while
 622                                     if (($processedrecords) % 1000 == 0) {
 623                                         print_progress($done, $tablestoconvert, 5, 1,
 624                                                        'Processing: '.$dbtablename.'/'.$fieldname.' ');
 625                                     }
 626                                 }
 627                             }else {
 628                                 if ($debug) {
 629                                     notify('no records found!');
 630                                 }
 631                             }
 632                         }   //close the while loop
 633
 634                         /********************
 635                          * Drop index here **
 636                          ********************/
 637
 638                         if ($CFG->dbtype == 'mysql') {
 639
 640                             /*********************************
 641                              * Change column encoding 2 phase*
 642                              *********************************/
 643
 644                             /*
 645                             $SQL = 'ALTER TABLE '.$CFG->prefix.$dbtablename;
 646                             $SQL.= ' CHANGE '.$fieldname.' '.$fieldname.' LONGTEXT';
 647                            // if ($length > 0) {
 648                            //     $SQL.='('.$length.') ';
 649                            // }
 650                             $SQL .= ' CHARACTER SET binary NOT NULL DEFAULT '.$default.';';
 651                             if ($debug) {
 652                                 $db->debug=999;
 653                             }
 654                             if ($fieldname != 'dummy') {
 655                                 execute_sql($SQL, $debug);
 656                             }
 657                             if ($debug) {
 658                                 $db->debug=0;
 659                             }*/
 660                             //phase 2
 661                             $SQL = 'ALTER TABLE '.$prefix.$dbtablename;
 662                             $SQL.= ' CHANGE '.$fieldname.' '.$fieldname.' '.$type;
 663                             if ($length > 0) {
 664                                 $SQL.='('.$length.') ';
 665                             }
 666                             $SQL.=' CHARACTER SET utf8 NOT NULL DEFAULT '.$default.';';
 667                             if ($debug) {
 668                                 $db->debug=999;
 669                             }
 670                             if ($fieldname != 'dummy') {
 671                                 execute_sql($SQL, $debug);
 672                             }
 673                             if ($debug) {
 674                                 $db->debug=0;
 675                             }
 676                             /********************************************
 677                              * build an array to add index back together*
 678                              ********************************************/
 679                             if ($addindex){
 680                                 $addindexarray[] = $addindex;
 681                             } else if ($adduniqueindex) {
 682                                 $adduniqueindexarray[] = $adduniqueindex;
 683                             } else if ($addprimary) {
 684                                 $addprimaryarray[] = $addprimary;
 685                             }
 686
 687                         } else {
 688
 689                         //posgresql code here
 690                         //No we don't need to do anything here
 691
 692                         }
 693
 694                     }
 695
 696                     /********************************
 697                      * Adding the index back        *
 698                      ********************************/
 699                     $alter = 0;
 700
 701                     if ($CFG->dbtype=='mysql'){
 702                         $SQL = 'ALTER TABLE '.$prefix.$dbtablename;
 703
 704                         if (!empty($addindexarray)) {
 705                             foreach ($addindexarray as $aidx){
 706                                 $SQL .= ' ADD INDEX '.$aidx.',';
 707                                 $alter++;
 708                             }
 709                         }
 710
 711                         if (!empty($adduniqueindexarray)) {
 712                             foreach ($adduniqueindexarray as $auidx){
 713                                 $SQL .= ' ADD UNIQUE INDEX '.$auidx.',';
 714                                 $alter++;
 715                             }
 716                         }
 717
 718                         if (!empty($addprimaryarray)) {
 719                             foreach ($addprimaryarray as $apm){
 720                                 $SQL .= ' ADD PRIMARY KEY '.$apm.',';
 721                                 $alter++;
 722                             }
 723                         }
 724
 725                         $SQL = rtrim($SQL, ', ');
 726                         $SQL.=';';
 727
 728                     } else {
 729                       ///posgresql code here
 730                       ///No we don't need to do anything here
 731
 732                     }
 733
 734                     if ($alter) {
 735                         if ($debug) {
 736                             $db->debug=999;
 737                         }
 738                         execute_sql($SQL, $debug);
 739                         if ($debug) {
 740                             $db->debug=0;
 741                         }
 742                     }
 743
 744                 } //if there are fields
 745
 746
 747             } /// Point 1 - bypass should end here.
 748
 749
 750             /************************************
 751              * now we modify the table encoding *
 752              ************************************/
 753             if ($CFG->dbtype=='mysql'){
 754                 $SQL = 'ALTER TABLE '.$prefix.$dbtablename.' CHARACTER SET utf8';
 755                 if ($debug) {
 756                     $db->debug=999;
 757                 }
 758                 execute_sql($SQL, $debug);
 759                 if ($debug) {
 760                     $db->debug=0;
 761                 }
 762             } else {
 763
 764                 ///posgresql code here
 765                 ///No we don't need to do anything here
 766             }
 767         }
 768     }
 769
 770     if ($CFG->dbtype=='mysql') {
 771         /*********************************
 772          * now we modify the db encoding *
 773          *********************************/
 774         $SQL = 'ALTER DATABASE '.$CFG->dbname.' CHARACTER SET utf8';
 775         execute_sql($SQL, $debug);
 776     } else {
 777         if (!is_postgres_utf8()) {
 778             //This old database is now deprecated
 779             set_config('migrated_to_new_db','1');
 780         }
 781     }
 782     delete_records('config','name','dbmigration');    //bye bye
 783
 784     //These have to go!
 785     if ($debug) {
 786         $db->debug=999;
 787     }
 788     if ($CFG->dbtype == 'postgres7') {
 789         $backup_db = $GLOBALS['db'];
 790         $GLOBALS['db'] = &get_postgres_db();
 791     }
 792     execute_sql('TRUNCATE TABLE '.$CFG->prefix.'cache_text', $debug);
 793     execute_sql('TRUNCATE TABLE '.$CFG->prefix.'cache_filters', $debug);
 794     if ($CFG->dbtype == 'postgres7') {
 795         $GLOBALS['db'] = $backup_db;
 796         unset($backup_db);
 797     }
 798     if ($debug) {
 799         $db->debug=0;
 800     }
 801     //update site language
 802     $sitelanguage = get_record('config','name', 'lang');
 803     if (strstr($sitelanguage->value, 'utf8')===false and $sitelanguage->value) {
 804         $sitelanguage->value.='_utf8';
 805         migrate2utf8_update_record('config',$sitelanguage);
 806     }
 807
 808     //finish the javascript bar
 809     $done = $tablestoconvert;
 810     print_progress($done, $tablestoconvert, 5, 1);
 811
 812     //prints the list of langs used in this site
 813     print_simple_box_start('center','50%');
 814     echo '<div align="center">The following Language Packs are needed for your users and courses. Please install the following Language Packs:<br><b>';
 815     $langsused = get_record('config','name', 'langsused');
 816     $langs = explode (',',$langsused->value);
 817
 818     foreach ($langs as $lang) {
 819         if (!empty($lang) and $lang != 'en_utf8') {
 820             echo $lang.', ';
 821         }
 822     }
 823     echo '</b><br/><a href="'.$CFG->wwwroot.'/'.$CFG->admin.'/langimport.php">Language Import Utility</a></div>';
 824     print_simple_box_end();
 825     delete_records('config','name','langsused');
 826
 827     //remove the cache file!
 828     @unlink($CFG->dataroot.'/cache/languages');
 829
 830     // Regenerate some cached data
 831
 832     if ($CFG->dbtype == 'mysql') {
 833         $db->Execute("SET NAMES 'utf8'");
 834     } else if ($CFG->dbtype == 'postgres7') {
 835         $db->Execute("SET NAMES 'utf8'");
 836     }
 837
 838     rebuild_course_cache();
 839
 840     //set the final flag
 841     migrate2utf8_set_config('unicodedb','true');    //this is the main flag for unicode db
 842
 843      //echo date("H:i:s");
 844 }
 845
 846
 847 /* returns the course lang
 848  * @param int courseid
 849  * @return string
 850  */
 851 function get_course_lang($courseid) {
 852
 853     static $coursecache;
 854
 855     if (!isset($coursecache[$courseid])) {
 856         if ($course = get_record('course','id',$courseid)){
 857             $coursecache[$courseid] = $course->lang;
 858             return $course->lang;
 859         }
 860         return false;
 861     } else {
 862         return $coursecache[$courseid];
 863     }
 864 }
 865
 866 /* returns the teacher's lang
 867  * @param int courseid
 868  * @return string
 869  */
 870 function get_main_teacher_lang($courseid) {
 871     //editting teacher > non editting teacher
 872     global $CFG;
 873     static $mainteachercache;
 874
 875     if (!isset($mainteachercache[$courseid])) {
 876         $SQL = 'SELECT u.lang from '.$CFG->prefix.'user_teachers ut,
 877                '.$CFG->prefix.'course c,
 878                '.$CFG->prefix.'user u WHERE
 879                c.id = ut.course AND ut.course = '.$courseid.' AND u.id = ut.userid ORDER BY ut.authority ASC';
 880
 881         if ($teacher = get_record_sql($SQL, true)) {
 882             $mainteachercache[$courseid] = $teacher->lang;
 883             return $teacher->lang;
 884         } else {
 885             $admin = get_admin();
 886             $mainteachercache[$courseid] = $admin->lang;
 887             return $admin->lang;
 888         }
 889     } else {
 890         return $mainteachercache[$courseid];
 891     }
 892 }
 893
 894 function get_original_encoding($sitelang, $courselang, $userlang){
 895
 896     global $CFG, $enc;
 897
 898     $lang = '';
 899     if ($courselang) {
 900         $lang = $courselang;
 901     }
 902     else if ($userlang) {
 903         $lang = $userlang;
 904     }
 905     else if ($sitelang) {
 906         $lang = $sitelang;
 907     }
 908     else {
 909         error ('no language found!');
 910     }
 911
 912     if ($enc[$lang]) {
 913         return $enc[$lang];
 914     } else {
 915         notify ('unknown language detected: '.$lang);
 916         return false;
 917     }
 918 }
 919
 920 /* returns the user's lang
 921  * @param int userid
 922  * @return string
 923  */
 924 function get_user_lang($userid) {
 925
 926     static $usercache;
 927
 928     if (!isset($usercache[$userid])) {
 929         if ($user = get_record('user','id',$userid)) {
 930             $usercache[$userid] = $user->lang;
 931             return $user->lang;
 932         }
 933     } else {
 934         return $usercache[$userid];
 935     }
 936     return false;
 937 }
 938
 939 // a placeholder for now
 940 function log_the_problem_somewhere() {  //Eloy: Nice function, perhaps we could use it, perhpas no. :-)
 941     global $CFG, $dbtablename, $fieldname, $record;
 942     if ($CFG->debug>7) {
 943         echo "<br />Problem converting: $dbtablename -> $fieldname -> {$record->id}!";
 944     }
 945 }
 946
 947 // only this function should be used during db migraton, because of addslashes at the end of the convertion
 948 function utfconvert($string, $enc, $slash=true) {
 949     global $textlib;
 950     if ($result = $textlib->convert($string, $enc)) {
 951         if ($slash) {
 952             $result = addslashes($result);
 953         }
 954     }
 955     return $result;
 956 }
 957
 958 function validate_form(&$form, &$err) {
 959     global $CFG;
 960
 961     $newdb = &ADONewConnection('postgres7');
 962     error_reporting(0);  // Hide errors
 963     $dbconnected = $newdb->Connect($form->dbhost,$form->dbuser,$form->dbpass,$form->dbname);
 964     error_reporting($CFG->debug);  // Show errors
 965     if (!$dbconnected) {
 966         $err['dbconnect'] = get_string('dbmigrateconnecerror', 'admin');
 967         return;
 968     }
 969
 970     if (!is_postgres_utf8($newdb)) {
 971         $encoding = $newdb->GetOne('SHOW server_encoding');
 972         $err['dbconnect'] = get_string('dbmigrateencodingerror', 'admin', $encoding);
 973         return;
 974     }
 975
 976     if (!empty($form->pathtopgdump) && !is_executable($form->pathtopgdump)) {
 977         $err['pathtopgdump'] = get_string('pathtopgdumpinvalid','admin');
 978         return;
 979     }
 980
 981     if (!empty($form->pathtopsql) && !is_executable($form->pathtopsql)) {
 982         $err['pathtopsql'] = get_string('pathtopsqlinvalid','admin');
 983         return;
 984     }
 985
 986     return;
 987 }
 988
 989 function is_postgres_utf8($thedb = null) {
 990     if ($thedb === null) {
 991         $thedb = &$GLOBALS['db'];
 992     }
 993
 994     $db_encoding_postgres = $thedb->GetOne('SHOW server_encoding');
 995     if (strtoupper($db_encoding_postgres) == 'UNICODE' || strtoupper($db_encoding_postgres) == 'UTF8') {
 996         return true;
 997     } else {
 998         return false;
 999     }
1000 }
1001
1002 function &get_postgres_db() {
1003     static $postgres_db;
1004
1005     if (!$postgres_db) {
1006         if (is_postgres_utf8()) {
1007             $postgres_db = &$GLOBALS['db'];
1008         } else {
1009             $postgres_db = &ADONewConnection('postgres7');
1010             $postgres_db->Connect($_SESSION['newpostgresdb']->dbhost,$_SESSION['newpostgresdb']->dbuser,$_SESSION['newpostgresdb']->dbpass,$_SESSION['newpostgresdb']->dbname);
1011         }
1012     }
1013
1014     return $postgres_db;
1015 }
1016
1017 function is_postgres_setup() {
1018     $postgres_db = &get_postgres_db();
1019
1020     return $GLOBALS['db']->MetaTables() == $postgres_db->MetaTables();
1021 }
1022
1023 function migrate2utf8_update_record($table,$record) {
1024     global $CFG;
1025
1026     if ($CFG->dbtype == 'mysql') {
1027         update_record($table,$record);
1028     } else {
1029         $backup_db = $GLOBALS['db'];
1030         $GLOBALS['db'] = &get_postgres_db();
1031         global $in;
1032         $in = true;
1033         update_record($table,$record);
1034         $GLOBALS['db'] = $backup_db;
1035     }
1036 }
1037
1038 function migrate2utf8_set_config($name, $value, $plugin=NULL) {
1039     global $CFG;
1040     if ($CFG->dbtype == 'mysql') {
1041         set_config($name, $value, $plugin);
1042     } else {
1043         $backup_db = $GLOBALS['db'];
1044         $GLOBALS['db'] = &get_postgres_db();
1045         set_config($name, $value, $plugin);
1046         $GLOBALS['db'] = $backup_db;
1047     }
1048 }
1049
1050 // this needs to print an error when a mod does not have a migrate2utf8.xml
1051 function utf_get_xml ($mode=0) { // if mode is 1, do not perform check for script validity
1052     global $CFG;
1053
1054     $xmls = array();
1055     $noscript = 0; // we assume all mod and all blocks have migration scripts
1056
1057     /*****************************************************************************
1058      * traverse order is mod->backup->block->block_plugin->enroll_plugin->global *
1059      *****************************************************************************/
1060
1061     ///mod
1062     if (!$mods = get_list_of_plugins('mod')) {
1063         error('No modules installed!');
1064     }
1065
1066     foreach ($mods as $mod){
1067         if (file_exists($CFG->dirroot.'/mod/'.$mod.'/db/migrate2utf8.xml')) {
1068             $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/mod/'.$mod.'/db/migrate2utf8.xml'));
1069         } else if (!$mode) {
1070             $noscript = 1;
1071             notify('warning, there is no migration script detected for this module - '.$mod);
1072         }
1073     }
1074
1075     ///Backups
1076     $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/backup/db/migrate2utf8.xml'));
1077
1078     ///Blocks
1079     $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/blocks/db/migrate2utf8.xml'));
1080
1081     ///Block Plugins
1082     if (!$blocks = get_list_of_plugins('blocks')) {
1083         //error('No blocks installed!');    //Eloy: Is this a cause to stop?
1084     }
1085
1086     foreach ($blocks as $block){
1087         if (file_exists($CFG->dirroot.'/blocks/'.$block.'/db/migrate2utf8.xml')) {
1088             $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/blocks/'.$block.'/db/migrate2utf8.xml'));
1089         } else if (!$mode) {
1090             if (file_exists($CFG->dirroot.'/blocks/'.$block.'/db/mysql.sql') && filesize($CFG->dirroot.'/blocks/'.$block.'/db/mysql.sql')) { // if no migration script, and have db script, we are in trouble
1091                 notify('warning, there is no migration script detected for this block - '.$block);
1092                 $noscript = 1;
1093             }
1094         }
1095     }
1096
1097     ///Enrol
1098
1099     if (!$enrols = get_list_of_plugins('enrol')) {
1100         //error('No enrol installed!');   //Eloy: enrol, not blocks :-) Is this a cause to stop?
1101     }
1102
1103     foreach ($enrols as $enrol){
1104         if (file_exists($CFG->dirroot.'/enrol/'.$enrol.'/db/migrate2utf8.xml')) {
1105            $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/enrol/'.$enrol.'/db/migrate2utf8.xml'));
1106         }
1107     }
1108
1109     ///Lastly, globals
1110
1111     $xmls[] = xmlize(file_get_contents($CFG->dirroot.'/lib/db/migrate2utf8.xml'));
1112
1113     if ($noscript) {
1114         notify ('Some of your modules or Blocks do not have a migration script. It is very likely that these are contrib modules. If your Moodle site uses non-UTF8 language packs and non-en language packs, data inside these moduels or blocks will not be displayed correctly after the migration. Please proceed with caution.');
1115     }
1116
1117     return $xmls;
1118
1119 }