contrib/mw-to-git/git-remote-mediawiki

   1 #! /usr/bin/perl
   2
   3 # Copyright (C) 2011
   4 #     Jérémie Nikaes <jeremie.nikaes@ensimag.imag.fr>
   5 #     Arnaud Lacurie <arnaud.lacurie@ensimag.imag.fr>
   6 #     Claire Fousse <claire.fousse@ensimag.imag.fr>
   7 #     David Amouyal <david.amouyal@ensimag.imag.fr>
   8 #     Matthieu Moy <matthieu.moy@grenoble-inp.fr>
   9 # License: GPL v2 or later
  10
  11 # Gateway between Git and MediaWiki.
  12 #   https://github.com/Bibzball/Git-Mediawiki/wiki
  13 #
  14 # Known limitations:
  15 #
  16 # - Poor performance in the best case: it takes forever to check
  17 #   whether we're up-to-date (on fetch or push) or to fetch a few
  18 #   revisions from a large wiki, because we use exclusively a
  19 #   page-based synchronization. We could switch to a wiki-wide
  20 #   synchronization when the synchronization involves few revisions
  21 #   but the wiki is large.
  22 #
  23 # - Git renames could be turned into MediaWiki renames (see TODO
  24 #   below)
  25 #
  26 # - No way to import "one page, and all pages included in it"
  27 #
  28 # - Multiple remote MediaWikis have not been very well tested.
  29
  30 use strict;
  31 use MediaWiki::API;
  32 use DateTime::Format::ISO8601;
  33
  34 # By default, use UTF-8 to communicate with Git and the user
  35 binmode STDERR, ":utf8";
  36 binmode STDOUT, ":utf8";
  37
  38 use URI::Escape;
  39 use IPC::Open2;
  40
  41 use warnings;
  42
  43 # Mediawiki filenames can contain forward slashes. This variable decides by which pattern they should be replaced
  44 use constant SLASH_REPLACEMENT => "%2F";
  45
  46 # It's not always possible to delete pages (may require some
  47 # priviledges). Deleted pages are replaced with this content.
  48 use constant DELETED_CONTENT => "[[Category:Deleted]]\n";
  49
  50 # It's not possible to create empty pages. New empty files in Git are
  51 # sent with this content instead.
  52 use constant EMPTY_CONTENT => "<!-- empty page -->\n";
  53
  54 # used to reflect file creation or deletion in diff.
  55 use constant NULL_SHA1 => "0000000000000000000000000000000000000000";
  56
  57 my $remotename = $ARGV[0];
  58 my $url = $ARGV[1];
  59
  60 # Accept both space-separated and multiple keys in config file.
  61 # Spaces should be written as _ anyway because we'll use chomp.
  62 my @tracked_pages = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".pages"));
  63 chomp(@tracked_pages);
  64
  65 # Just like @tracked_pages, but for MediaWiki categories.
  66 my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories"));
  67 chomp(@tracked_categories);
  68
  69 # Import media files too.
  70 my $import_media = run_git("config --get --bool remote.". $remotename .".mediaimport");
  71 chomp($import_media);
  72 $import_media = ($import_media eq "true");
  73
  74 my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin");
  75 # Note: mwPassword is discourraged. Use the credential system instead.
  76 my $wiki_passwd = run_git("config --get remote.". $remotename .".mwPassword");
  77 my $wiki_domain = run_git("config --get remote.". $remotename .".mwDomain");
  78 chomp($wiki_login);
  79 chomp($wiki_passwd);
  80 chomp($wiki_domain);
  81
  82 # Import only last revisions (both for clone and fetch)
  83 my $shallow_import = run_git("config --get --bool remote.". $remotename .".shallow");
  84 chomp($shallow_import);
  85 $shallow_import = ($shallow_import eq "true");
  86
  87 # Dumb push: don't update notes and mediawiki ref to reflect the last push.
  88 #
  89 # Configurable with mediawiki.dumbPush, or per-remote with
  90 # remote.<remotename>.dumbPush.
  91 #
  92 # This means the user will have to re-import the just-pushed
  93 # revisions. On the other hand, this means that the Git revisions
  94 # corresponding to MediaWiki revisions are all imported from the wiki,
  95 # regardless of whether they were initially created in Git or from the
  96 # web interface, hence all users will get the same history (i.e. if
  97 # the push from Git to MediaWiki loses some information, everybody
  98 # will get the history with information lost). If the import is
  99 # deterministic, this means everybody gets the same sha1 for each
 100 # MediaWiki revision.
 101 my $dumb_push = run_git("config --get --bool remote.$remotename.dumbPush");
 102 unless ($dumb_push) {
 103         $dumb_push = run_git("config --get --bool mediawiki.dumbPush");
 104 }
 105 chomp($dumb_push);
 106 $dumb_push = ($dumb_push eq "true");
 107
 108 my $wiki_name = $url;
 109 $wiki_name =~ s/[^\/]*:\/\///;
 110 # If URL is like http://user:password@example.com/, we clearly don't
 111 # want the password in $wiki_name. While we're there, also remove user
 112 # and '@' sign, to avoid author like MWUser@HTTPUser@host.com
 113 $wiki_name =~ s/^.*@//;
 114
 115 # Commands parser
 116 my $entry;
 117 my @cmd;
 118 while (<STDIN>) {
 119         chomp;
 120         @cmd = split(/ /);
 121         if (defined($cmd[0])) {
 122                 # Line not blank
 123                 if ($cmd[0] eq "capabilities") {
 124                         die("Too many arguments for capabilities") unless (!defined($cmd[1]));
 125                         mw_capabilities();
 126                 } elsif ($cmd[0] eq "list") {
 127                         die("Too many arguments for list") unless (!defined($cmd[2]));
 128                         mw_list($cmd[1]);
 129                 } elsif ($cmd[0] eq "import") {
 130                         die("Invalid arguments for import") unless ($cmd[1] ne "" && !defined($cmd[2]));
 131                         mw_import($cmd[1]);
 132                 } elsif ($cmd[0] eq "option") {
 133                         die("Too many arguments for option") unless ($cmd[1] ne "" && $cmd[2] ne "" && !defined($cmd[3]));
 134                         mw_option($cmd[1],$cmd[2]);
 135                 } elsif ($cmd[0] eq "push") {
 136                         mw_push($cmd[1]);
 137                 } else {
 138                         print STDERR "Unknown command. Aborting...\n";
 139                         last;
 140                 }
 141         } else {
 142                 # blank line: we should terminate
 143                 last;
 144         }
 145
 146         BEGIN { $| = 1 } # flush STDOUT, to make sure the previous
 147                          # command is fully processed.
 148 }
 149
 150 ########################## Functions ##############################
 151
 152 ## credential API management (generic functions)
 153
 154 sub credential_from_url {
 155         my $url = shift;
 156         my $parsed = URI->new($url);
 157         my %credential;
 158
 159         if ($parsed->scheme) {
 160                 $credential{protocol} = $parsed->scheme;
 161         }
 162         if ($parsed->host) {
 163                 $credential{host} = $parsed->host;
 164         }
 165         if ($parsed->path) {
 166                 $credential{path} = $parsed->path;
 167         }
 168         if ($parsed->userinfo) {
 169                 if ($parsed->userinfo =~ /([^:]*):(.*)/) {
 170                         $credential{username} = $1;
 171                         $credential{password} = $2;
 172                 } else {
 173                         $credential{username} = $parsed->userinfo;
 174                 }
 175         }
 176
 177         return %credential;
 178 }
 179
 180 sub credential_read {
 181         my %credential;
 182         my $reader = shift;
 183         my $op = shift;
 184         while (<$reader>) {
 185                 my ($key, $value) = /([^=]*)=(.*)/;
 186                 if (not defined $key) {
 187                         die "ERROR receiving response from git credential $op:\n$_\n";
 188                 }
 189                 $credential{$key} = $value;
 190         }
 191         return %credential;
 192 }
 193
 194 sub credential_write {
 195         my $credential = shift;
 196         my $writer = shift;
 197         while (my ($key, $value) = each(%$credential) ) {
 198                 if ($value) {
 199                         print $writer "$key=$value\n";
 200                 }
 201         }
 202 }
 203
 204 sub credential_run {
 205         my $op = shift;
 206         my $credential = shift;
 207         my $pid = open2(my $reader, my $writer, "git credential $op");
 208         credential_write($credential, $writer);
 209         print $writer "\n";
 210         close($writer);
 211
 212         if ($op eq "fill") {
 213                 %$credential = credential_read($reader, $op);
 214         } else {
 215                 if (<$reader>) {
 216                         die "ERROR while running git credential $op:\n$_";
 217                 }
 218         }
 219         close($reader);
 220         waitpid($pid, 0);
 221         my $child_exit_status = $? >> 8;
 222         if ($child_exit_status != 0) {
 223                 die "'git credential $op' failed with code $child_exit_status.";
 224         }
 225 }
 226
 227 # MediaWiki API instance, created lazily.
 228 my $mediawiki;
 229
 230 sub mw_connect_maybe {
 231         if ($mediawiki) {
 232                 return;
 233         }
 234         $mediawiki = MediaWiki::API->new;
 235         $mediawiki->{config}->{api_url} = "$url/api.php";
 236         if ($wiki_login) {
 237                 my %credential = credential_from_url($url);
 238                 $credential{username} = $wiki_login;
 239                 $credential{password} = $wiki_passwd;
 240                 credential_run("fill", \%credential);
 241                 my $request = {lgname => $credential{username},
 242                                lgpassword => $credential{password},
 243                                lgdomain => $wiki_domain};
 244                 if ($mediawiki->login($request)) {
 245                         credential_run("approve", \%credential);
 246                         print STDERR "Logged in mediawiki user \"$credential{username}\".\n";
 247                 } else {
 248                         print STDERR "Failed to log in mediawiki user \"$credential{username}\" on $url\n";
 249                         print STDERR "  (error " .
 250                                 $mediawiki->{error}->{code} . ': ' .
 251                                 $mediawiki->{error}->{details} . ")\n";
 252                         credential_run("reject", \%credential);
 253                         exit 1;
 254                 }
 255         }
 256 }
 257
 258 ## Functions for listing pages on the remote wiki
 259 sub get_mw_tracked_pages {
 260         my $pages = shift;
 261         get_mw_page_list(\@tracked_pages, $pages);
 262 }
 263
 264 sub get_mw_page_list {
 265         my $page_list = shift;
 266         my $pages = shift;
 267         my @some_pages = @$page_list;
 268         while (@some_pages) {
 269                 my $last = 50;
 270                 if ($#some_pages < $last) {
 271                         $last = $#some_pages;
 272                 }
 273                 my @slice = @some_pages[0..$last];
 274                 get_mw_first_pages(\@slice, $pages);
 275                 @some_pages = @some_pages[51..$#some_pages];
 276         }
 277 }
 278
 279 sub get_mw_tracked_categories {
 280         my $pages = shift;
 281         foreach my $category (@tracked_categories) {
 282                 if (index($category, ':') < 0) {
 283                         # Mediawiki requires the Category
 284                         # prefix, but let's not force the user
 285                         # to specify it.
 286                         $category = "Category:" . $category;
 287                 }
 288                 my $mw_pages = $mediawiki->list( {
 289                         action => 'query',
 290                         list => 'categorymembers',
 291                         cmtitle => $category,
 292                         cmlimit => 'max' } )
 293                         || die $mediawiki->{error}->{code} . ': '
 294                                 . $mediawiki->{error}->{details};
 295                 foreach my $page (@{$mw_pages}) {
 296                         $pages->{$page->{title}} = $page;
 297                 }
 298         }
 299 }
 300
 301 sub get_mw_all_pages {
 302         my $pages = shift;
 303         # No user-provided list, get the list of pages from the API.
 304         my $mw_pages = $mediawiki->list({
 305                 action => 'query',
 306                 list => 'allpages',
 307                 aplimit => 'max'
 308         });
 309         if (!defined($mw_pages)) {
 310                 print STDERR "fatal: could not get the list of wiki pages.\n";
 311                 print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
 312                 print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
 313                 exit 1;
 314         }
 315         foreach my $page (@{$mw_pages}) {
 316                 $pages->{$page->{title}} = $page;
 317         }
 318 }
 319
 320 # queries the wiki for a set of pages. Meant to be used within a loop
 321 # querying the wiki for slices of page list.
 322 sub get_mw_first_pages {
 323         my $some_pages = shift;
 324         my @some_pages = @{$some_pages};
 325
 326         my $pages = shift;
 327
 328         # pattern 'page1|page2|...' required by the API
 329         my $titles = join('|', @some_pages);
 330
 331         my $mw_pages = $mediawiki->api({
 332                 action => 'query',
 333                 titles => $titles,
 334         });
 335         if (!defined($mw_pages)) {
 336                 print STDERR "fatal: could not query the list of wiki pages.\n";
 337                 print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
 338                 print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
 339                 exit 1;
 340         }
 341         while (my ($id, $page) = each(%{$mw_pages->{query}->{pages}})) {
 342                 if ($id < 0) {
 343                         print STDERR "Warning: page $page->{title} not found on wiki\n";
 344                 } else {
 345                         $pages->{$page->{title}} = $page;
 346                 }
 347         }
 348 }
 349
 350 # Get the list of pages to be fetched according to configuration.
 351 sub get_mw_pages {
 352         mw_connect_maybe();
 353
 354         my %pages; # hash on page titles to avoid duplicates
 355         my $user_defined;
 356         if (@tracked_pages) {
 357                 $user_defined = 1;
 358                 # The user provided a list of pages titles, but we
 359                 # still need to query the API to get the page IDs.
 360                 get_mw_tracked_pages(\%pages);
 361         }
 362         if (@tracked_categories) {
 363                 $user_defined = 1;
 364                 get_mw_tracked_categories(\%pages);
 365         }
 366         if (!$user_defined) {
 367                 get_mw_all_pages(\%pages);
 368         }
 369         if ($import_media) {
 370                 print STDERR "Getting media files for selected pages...\n";
 371                 if ($user_defined) {
 372                         get_linked_mediafiles(\%pages);
 373                 } else {
 374                         get_all_mediafiles(\%pages);
 375                 }
 376         }
 377         return values(%pages);
 378 }
 379
 380 # usage: $out = run_git("command args");
 381 #        $out = run_git("command args", "raw"); # don't interpret output as UTF-8.
 382 sub run_git {
 383         my $args = shift;
 384         my $encoding = (shift || "encoding(UTF-8)");
 385         open(my $git, "-|:$encoding", "git " . $args);
 386         my $res = do { local $/; <$git> };
 387         close($git);
 388
 389         return $res;
 390 }
 391
 392
 393 sub get_all_mediafiles {
 394         my $pages = shift;
 395         # Attach list of all pages for media files from the API,
 396         # they are in a different namespace, only one namespace
 397         # can be queried at the same moment
 398         my $mw_pages = $mediawiki->list({
 399                 action => 'query',
 400                 list => 'allpages',
 401                 apnamespace => get_mw_namespace_id("File"),
 402                 aplimit => 'max'
 403         });
 404         if (!defined($mw_pages)) {
 405                 print STDERR "fatal: could not get the list of pages for media files.\n";
 406                 print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
 407                 print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
 408                 exit 1;
 409         }
 410         foreach my $page (@{$mw_pages}) {
 411                 $pages->{$page->{title}} = $page;
 412         }
 413 }
 414
 415 sub get_linked_mediafiles {
 416         my $pages = shift;
 417         my @titles = map $_->{title}, values(%{$pages});
 418
 419         # The query is split in small batches because of the MW API limit of
 420         # the number of links to be returned (500 links max).
 421         my $batch = 10;
 422         while (@titles) {
 423                 if ($#titles < $batch) {
 424                         $batch = $#titles;
 425                 }
 426                 my @slice = @titles[0..$batch];
 427
 428                 # pattern 'page1|page2|...' required by the API
 429                 my $mw_titles = join('|', @slice);
 430
 431                 # Media files could be included or linked from
 432                 # a page, get all related
 433                 my $query = {
 434                         action => 'query',
 435                         prop => 'links|images',
 436                         titles => $mw_titles,
 437                         plnamespace => get_mw_namespace_id("File"),
 438                         pllimit => 'max'
 439                 };
 440                 my $result = $mediawiki->api($query);
 441
 442                 while (my ($id, $page) = each(%{$result->{query}->{pages}})) {
 443                         my @media_titles;
 444                         if (defined($page->{links})) {
 445                                 my @link_titles = map $_->{title}, @{$page->{links}};
 446                                 push(@media_titles, @link_titles);
 447                         }
 448                         if (defined($page->{images})) {
 449                                 my @image_titles = map $_->{title}, @{$page->{images}};
 450                                 push(@media_titles, @image_titles);
 451                         }
 452                         if (@media_titles) {
 453                                 get_mw_page_list(\@media_titles, $pages);
 454                         }
 455                 }
 456
 457                 @titles = @titles[($batch+1)..$#titles];
 458         }
 459 }
 460
 461 sub get_mw_mediafile_for_page_revision {
 462         # Name of the file on Wiki, with the prefix.
 463         my $filename = shift;
 464         my $timestamp = shift;
 465         my %mediafile;
 466
 467         # Search if on a media file with given timestamp exists on
 468         # MediaWiki. In that case download the file.
 469         my $query = {
 470                 action => 'query',
 471                 prop => 'imageinfo',
 472                 titles => "File:" . $filename,
 473                 iistart => $timestamp,
 474                 iiend => $timestamp,
 475                 iiprop => 'timestamp|archivename|url',
 476                 iilimit => 1
 477         };
 478         my $result = $mediawiki->api($query);
 479
 480         my ($fileid, $file) = each( %{$result->{query}->{pages}} );
 481         # If not defined it means there is no revision of the file for
 482         # given timestamp.
 483         if (defined($file->{imageinfo})) {
 484                 $mediafile{title} = $filename;
 485
 486                 my $fileinfo = pop(@{$file->{imageinfo}});
 487                 $mediafile{timestamp} = $fileinfo->{timestamp};
 488                 # Mediawiki::API's download function doesn't support https URLs
 489                 # and can't download old versions of files.
 490                 print STDERR "\tDownloading file $mediafile{title}, version $mediafile{timestamp}\n";
 491                 $mediafile{content} = download_mw_mediafile($fileinfo->{url});
 492         }
 493         return %mediafile;
 494 }
 495
 496 sub download_mw_mediafile {
 497         my $url = shift;
 498
 499         my $response = $mediawiki->{ua}->get($url);
 500         if ($response->code == 200) {
 501                 return $response->decoded_content;
 502         } else {
 503                 print STDERR "Error downloading mediafile from :\n";
 504                 print STDERR "URL: $url\n";
 505                 print STDERR "Server response: " . $response->code . " " . $response->message . "\n";
 506                 exit 1;
 507         }
 508 }
 509
 510 sub get_last_local_revision {
 511         # Get note regarding last mediawiki revision
 512         my $note = run_git("notes --ref=$remotename/mediawiki show refs/mediawiki/$remotename/master 2>/dev/null");
 513         my @note_info = split(/ /, $note);
 514
 515         my $lastrevision_number;
 516         if (!(defined($note_info[0]) && $note_info[0] eq "mediawiki_revision:")) {
 517                 print STDERR "No previous mediawiki revision found";
 518                 $lastrevision_number = 0;
 519         } else {
 520                 # Notes are formatted : mediawiki_revision: #number
 521                 $lastrevision_number = $note_info[1];
 522                 chomp($lastrevision_number);
 523                 print STDERR "Last local mediawiki revision found is $lastrevision_number";
 524         }
 525         return $lastrevision_number;
 526 }
 527
 528 # Remember the timestamp corresponding to a revision id.
 529 my %basetimestamps;
 530
 531 sub get_last_remote_revision {
 532         mw_connect_maybe();
 533
 534         my @pages = get_mw_pages();
 535
 536         my $max_rev_num = 0;
 537
 538         foreach my $page (@pages) {
 539                 my $id = $page->{pageid};
 540
 541                 my $query = {
 542                         action => 'query',
 543                         prop => 'revisions',
 544                         rvprop => 'ids|timestamp',
 545                         pageids => $id,
 546                 };
 547
 548                 my $result = $mediawiki->api($query);
 549
 550                 my $lastrev = pop(@{$result->{query}->{pages}->{$id}->{revisions}});
 551
 552                 $basetimestamps{$lastrev->{revid}} = $lastrev->{timestamp};
 553
 554                 $max_rev_num = ($lastrev->{revid} > $max_rev_num ? $lastrev->{revid} : $max_rev_num);
 555         }
 556
 557         print STDERR "Last remote revision found is $max_rev_num.\n";
 558         return $max_rev_num;
 559 }
 560
 561 # Clean content before sending it to MediaWiki
 562 sub mediawiki_clean {
 563         my $string = shift;
 564         my $page_created = shift;
 565         # Mediawiki does not allow blank space at the end of a page and ends with a single \n.
 566         # This function right trims a string and adds a \n at the end to follow this rule
 567         $string =~ s/\s+$//;
 568         if ($string eq "" && $page_created) {
 569                 # Creating empty pages is forbidden.
 570                 $string = EMPTY_CONTENT;
 571         }
 572         return $string."\n";
 573 }
 574
 575 # Filter applied on MediaWiki data before adding them to Git
 576 sub mediawiki_smudge {
 577         my $string = shift;
 578         if ($string eq EMPTY_CONTENT) {
 579                 $string = "";
 580         }
 581         # This \n is important. This is due to mediawiki's way to handle end of files.
 582         return $string."\n";
 583 }
 584
 585 sub mediawiki_clean_filename {
 586         my $filename = shift;
 587         $filename =~ s/@{[SLASH_REPLACEMENT]}/\//g;
 588         # [, ], |, {, and } are forbidden by MediaWiki, even URL-encoded.
 589         # Do a variant of URL-encoding, i.e. looks like URL-encoding,
 590         # but with _ added to prevent MediaWiki from thinking this is
 591         # an actual special character.
 592         $filename =~ s/[\[\]\{\}\|]/sprintf("_%%_%x", ord($&))/ge;
 593         # If we use the uri escape before
 594         # we should unescape here, before anything
 595
 596         return $filename;
 597 }
 598
 599 sub mediawiki_smudge_filename {
 600         my $filename = shift;
 601         $filename =~ s/\//@{[SLASH_REPLACEMENT]}/g;
 602         $filename =~ s/ /_/g;
 603         # Decode forbidden characters encoded in mediawiki_clean_filename
 604         $filename =~ s/_%_([0-9a-fA-F][0-9a-fA-F])/sprintf("%c", hex($1))/ge;
 605         return $filename;
 606 }
 607
 608 sub literal_data {
 609         my ($content) = @_;
 610         print STDOUT "data ", bytes::length($content), "\n", $content;
 611 }
 612
 613 sub literal_data_raw {
 614         # Output possibly binary content.
 615         my ($content) = @_;
 616         # Avoid confusion between size in bytes and in characters
 617         utf8::downgrade($content);
 618         binmode STDOUT, ":raw";
 619         print STDOUT "data ", bytes::length($content), "\n", $content;
 620         binmode STDOUT, ":utf8";
 621 }
 622
 623 sub mw_capabilities {
 624         # Revisions are imported to the private namespace
 625         # refs/mediawiki/$remotename/ by the helper and fetched into
 626         # refs/remotes/$remotename later by fetch.
 627         print STDOUT "refspec refs/heads/*:refs/mediawiki/$remotename/*\n";
 628         print STDOUT "import\n";
 629         print STDOUT "list\n";
 630         print STDOUT "push\n";
 631         print STDOUT "\n";
 632 }
 633
 634 sub mw_list {
 635         # MediaWiki do not have branches, we consider one branch arbitrarily
 636         # called master, and HEAD pointing to it.
 637         print STDOUT "? refs/heads/master\n";
 638         print STDOUT "\@refs/heads/master HEAD\n";
 639         print STDOUT "\n";
 640 }
 641
 642 sub mw_option {
 643         print STDERR "remote-helper command 'option $_[0]' not yet implemented\n";
 644         print STDOUT "unsupported\n";
 645 }
 646
 647 sub fetch_mw_revisions_for_page {
 648         my $page = shift;
 649         my $id = shift;
 650         my $fetch_from = shift;
 651         my @page_revs = ();
 652         my $query = {
 653                 action => 'query',
 654                 prop => 'revisions',
 655                 rvprop => 'ids',
 656                 rvdir => 'newer',
 657                 rvstartid => $fetch_from,
 658                 rvlimit => 500,
 659                 pageids => $id,
 660         };
 661
 662         my $revnum = 0;
 663         # Get 500 revisions at a time due to the mediawiki api limit
 664         while (1) {
 665                 my $result = $mediawiki->api($query);
 666
 667                 # Parse each of those 500 revisions
 668                 foreach my $revision (@{$result->{query}->{pages}->{$id}->{revisions}}) {
 669                         my $page_rev_ids;
 670                         $page_rev_ids->{pageid} = $page->{pageid};
 671                         $page_rev_ids->{revid} = $revision->{revid};
 672                         push(@page_revs, $page_rev_ids);
 673                         $revnum++;
 674                 }
 675                 last unless $result->{'query-continue'};
 676                 $query->{rvstartid} = $result->{'query-continue'}->{revisions}->{rvstartid};
 677         }
 678         if ($shallow_import && @page_revs) {
 679                 print STDERR "  Found 1 revision (shallow import).\n";
 680                 @page_revs = sort {$b->{revid} <=> $a->{revid}} (@page_revs);
 681                 return $page_revs[0];
 682         }
 683         print STDERR "  Found ", $revnum, " revision(s).\n";
 684         return @page_revs;
 685 }
 686
 687 sub fetch_mw_revisions {
 688         my $pages = shift; my @pages = @{$pages};
 689         my $fetch_from = shift;
 690
 691         my @revisions = ();
 692         my $n = 1;
 693         foreach my $page (@pages) {
 694                 my $id = $page->{pageid};
 695
 696                 print STDERR "page $n/", scalar(@pages), ": ". $page->{title} ."\n";
 697                 $n++;
 698                 my @page_revs = fetch_mw_revisions_for_page($page, $id, $fetch_from);
 699                 @revisions = (@page_revs, @revisions);
 700         }
 701
 702         return ($n, @revisions);
 703 }
 704
 705 sub import_file_revision {
 706         my $commit = shift;
 707         my %commit = %{$commit};
 708         my $full_import = shift;
 709         my $n = shift;
 710         my $mediafile = shift;
 711         my %mediafile;
 712         if ($mediafile) {
 713                 %mediafile = %{$mediafile};
 714         }
 715
 716         my $title = $commit{title};
 717         my $comment = $commit{comment};
 718         my $content = $commit{content};
 719         my $author = $commit{author};
 720         my $date = $commit{date};
 721
 722         print STDOUT "commit refs/mediawiki/$remotename/master\n";
 723         print STDOUT "mark :$n\n";
 724         print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n";
 725         literal_data($comment);
 726
 727         # If it's not a clone, we need to know where to start from
 728         if (!$full_import && $n == 1) {
 729                 print STDOUT "from refs/mediawiki/$remotename/master^0\n";
 730         }
 731         if ($content ne DELETED_CONTENT) {
 732                 print STDOUT "M 644 inline $title.mw\n";
 733                 literal_data($content);
 734                 if (%mediafile) {
 735                         print STDOUT "M 644 inline $mediafile{title}\n";
 736                         literal_data_raw($mediafile{content});
 737                 }
 738                 print STDOUT "\n\n";
 739         } else {
 740                 print STDOUT "D $title.mw\n";
 741         }
 742
 743         # mediawiki revision number in the git note
 744         if ($full_import && $n == 1) {
 745                 print STDOUT "reset refs/notes/$remotename/mediawiki\n";
 746         }
 747         print STDOUT "commit refs/notes/$remotename/mediawiki\n";
 748         print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n";
 749         literal_data("Note added by git-mediawiki during import");
 750         if (!$full_import && $n == 1) {
 751                 print STDOUT "from refs/notes/$remotename/mediawiki^0\n";
 752         }
 753         print STDOUT "N inline :$n\n";
 754         literal_data("mediawiki_revision: " . $commit{mw_revision});
 755         print STDOUT "\n\n";
 756 }
 757
 758 # parse a sequence of
 759 # <cmd> <arg1>
 760 # <cmd> <arg2>
 761 # \n
 762 # (like batch sequence of import and sequence of push statements)
 763 sub get_more_refs {
 764         my $cmd = shift;
 765         my @refs;
 766         while (1) {
 767                 my $line = <STDIN>;
 768                 if ($line =~ m/^$cmd (.*)$/) {
 769                         push(@refs, $1);
 770                 } elsif ($line eq "\n") {
 771                         return @refs;
 772                 } else {
 773                         die("Invalid command in a '$cmd' batch: ". $_);
 774                 }
 775         }
 776 }
 777
 778 sub mw_import {
 779         # multiple import commands can follow each other.
 780         my @refs = (shift, get_more_refs("import"));
 781         foreach my $ref (@refs) {
 782                 mw_import_ref($ref);
 783         }
 784         print STDOUT "done\n";
 785 }
 786
 787 sub mw_import_ref {
 788         my $ref = shift;
 789         # The remote helper will call "import HEAD" and
 790         # "import refs/heads/master".
 791         # Since HEAD is a symbolic ref to master (by convention,
 792         # followed by the output of the command "list" that we gave),
 793         # we don't need to do anything in this case.
 794         if ($ref eq "HEAD") {
 795                 return;
 796         }
 797
 798         mw_connect_maybe();
 799
 800         my @pages = get_mw_pages();
 801
 802         print STDERR "Searching revisions...\n";
 803         my $last_local = get_last_local_revision();
 804         my $fetch_from = $last_local + 1;
 805         if ($fetch_from == 1) {
 806                 print STDERR ", fetching from beginning.\n";
 807         } else {
 808                 print STDERR ", fetching from here.\n";
 809         }
 810         my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
 811
 812         # Creation of the fast-import stream
 813         print STDERR "Fetching & writing export data...\n";
 814
 815         $n = 0;
 816         my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined
 817
 818         foreach my $pagerevid (sort {$a->{revid} <=> $b->{revid}} @revisions) {
 819                 # fetch the content of the pages
 820                 my $query = {
 821                         action => 'query',
 822                         prop => 'revisions',
 823                         rvprop => 'content|timestamp|comment|user|ids',
 824                         revids => $pagerevid->{revid},
 825                 };
 826
 827                 my $result = $mediawiki->api($query);
 828
 829                 my $rev = pop(@{$result->{query}->{pages}->{$pagerevid->{pageid}}->{revisions}});
 830
 831                 $n++;
 832
 833                 my $page_title = $result->{query}->{pages}->{$pagerevid->{pageid}}->{title};
 834                 my %commit;
 835                 $commit{author} = $rev->{user} || 'Anonymous';
 836                 $commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*';
 837                 $commit{title} = mediawiki_smudge_filename($page_title);
 838                 $commit{mw_revision} = $pagerevid->{revid};
 839                 $commit{content} = mediawiki_smudge($rev->{'*'});
 840
 841                 if (!defined($rev->{timestamp})) {
 842                         $last_timestamp++;
 843                 } else {
 844                         $last_timestamp = $rev->{timestamp};
 845                 }
 846                 $commit{date} = DateTime::Format::ISO8601->parse_datetime($last_timestamp);
 847
 848                 # Differentiates classic pages and media files.
 849                 my ($namespace, $filename) = $page_title =~ /^([^:]*):(.*)$/;
 850                 my %mediafile;
 851                 if ($namespace && get_mw_namespace_id($namespace) == get_mw_namespace_id("File")) {
 852                         %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp});
 853                 }
 854                 # If this is a revision of the media page for new version
 855                 # of a file do one common commit for both file and media page.
 856                 # Else do commit only for that page.
 857                 print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevid->{revid} of $commit{title}\n";
 858                 import_file_revision(\%commit, ($fetch_from == 1), $n, \%mediafile);
 859         }
 860
 861         if ($fetch_from == 1 && $n == 0) {
 862                 print STDERR "You appear to have cloned an empty MediaWiki.\n";
 863                 # Something has to be done remote-helper side. If nothing is done, an error is
 864                 # thrown saying that HEAD is refering to unknown object 0000000000000000000
 865                 # and the clone fails.
 866         }
 867 }
 868
 869 sub error_non_fast_forward {
 870         my $advice = run_git("config --bool advice.pushNonFastForward");
 871         chomp($advice);
 872         if ($advice ne "false") {
 873                 # Native git-push would show this after the summary.
 874                 # We can't ask it to display it cleanly, so print it
 875                 # ourselves before.
 876                 print STDERR "To prevent you from losing history, non-fast-forward updates were rejected\n";
 877                 print STDERR "Merge the remote changes (e.g. 'git pull') before pushing again. See the\n";
 878                 print STDERR "'Note about fast-forwards' section of 'git push --help' for details.\n";
 879         }
 880         print STDOUT "error $_[0] \"non-fast-forward\"\n";
 881         return 0;
 882 }
 883
 884 sub mw_upload_file {
 885         my $complete_file_name = shift;
 886         my $new_sha1 = shift;
 887         my $extension = shift;
 888         my $file_deleted = shift;
 889         my $summary = shift;
 890         my $newrevid;
 891         my $path = "File:" . $complete_file_name;
 892         my %hashFiles = get_allowed_file_extensions();
 893         if (!exists($hashFiles{$extension})) {
 894                 print STDERR "$complete_file_name is not a permitted file on this wiki.\n";
 895                 print STDERR "Check the configuration of file uploads in your mediawiki.\n";
 896                 return $newrevid;
 897         }
 898         # Deleting and uploading a file requires a priviledged user
 899         if ($file_deleted) {
 900                 mw_connect_maybe();
 901                 my $query = {
 902                         action => 'delete',
 903                         title => $path,
 904                         reason => $summary
 905                 };
 906                 if (!$mediawiki->edit($query)) {
 907                         print STDERR "Failed to delete file on remote wiki\n";
 908                         print STDERR "Check your permissions on the remote site. Error code:\n";
 909                         print STDERR $mediawiki->{error}->{code} . ':' . $mediawiki->{error}->{details};
 910                         exit 1;
 911                 }
 912         } else {
 913                 # Don't let perl try to interpret file content as UTF-8 => use "raw"
 914                 my $content = run_git("cat-file blob $new_sha1", "raw");
 915                 if ($content ne "") {
 916                         mw_connect_maybe();
 917                         $mediawiki->{config}->{upload_url} =
 918                                 "$url/index.php/Special:Upload";
 919                         $mediawiki->edit({
 920                                 action => 'upload',
 921                                 filename => $complete_file_name,
 922                                 comment => $summary,
 923                                 file => [undef,
 924                                          $complete_file_name,
 925                                          Content => $content],
 926                                 ignorewarnings => 1,
 927                         }, {
 928                                 skip_encoding => 1
 929                         } ) || die $mediawiki->{error}->{code} . ':'
 930                                  . $mediawiki->{error}->{details};
 931                         my $last_file_page = $mediawiki->get_page({title => $path});
 932                         $newrevid = $last_file_page->{revid};
 933                         print STDERR "Pushed file: $new_sha1 - $complete_file_name.\n";
 934                 } else {
 935                         print STDERR "Empty file $complete_file_name not pushed.\n";
 936                 }
 937         }
 938         return $newrevid;
 939 }
 940
 941 sub mw_push_file {
 942         my $diff_info = shift;
 943         # $diff_info contains a string in this format:
 944         # 100644 100644 <sha1_of_blob_before_commit> <sha1_of_blob_now> <status>
 945         my @diff_info_split = split(/[ \t]/, $diff_info);
 946
 947         # Filename, including .mw extension
 948         my $complete_file_name = shift;
 949         # Commit message
 950         my $summary = shift;
 951         # MediaWiki revision number. Keep the previous one by default,
 952         # in case there's no edit to perform.
 953         my $oldrevid = shift;
 954         my $newrevid;
 955
 956         my $new_sha1 = $diff_info_split[3];
 957         my $old_sha1 = $diff_info_split[2];
 958         my $page_created = ($old_sha1 eq NULL_SHA1);
 959         my $page_deleted = ($new_sha1 eq NULL_SHA1);
 960         $complete_file_name = mediawiki_clean_filename($complete_file_name);
 961
 962         my ($title, $extension) = $complete_file_name =~ /^(.*)\.([^\.]*)$/;
 963         if (!defined($extension)) {
 964                 $extension = "";
 965         }
 966         if ($extension eq "mw") {
 967                 my $file_content;
 968                 if ($page_deleted) {
 969                         # Deleting a page usually requires
 970                         # special priviledges. A common
 971                         # convention is to replace the page
 972                         # with this content instead:
 973                         $file_content = DELETED_CONTENT;
 974                 } else {
 975                         $file_content = run_git("cat-file blob $new_sha1");
 976                 }
 977
 978                 mw_connect_maybe();
 979
 980                 my $result = $mediawiki->edit( {
 981                         action => 'edit',
 982                         summary => $summary,
 983                         title => $title,
 984                         basetimestamp => $basetimestamps{$oldrevid},
 985                         text => mediawiki_clean($file_content, $page_created),
 986                                   }, {
 987                                           skip_encoding => 1 # Helps with names with accentuated characters
 988                                   });
 989                 if (!$result) {
 990                         if ($mediawiki->{error}->{code} == 3) {
 991                                 # edit conflicts, considered as non-fast-forward
 992                                 print STDERR 'Warning: Error ' .
 993                                     $mediawiki->{error}->{code} .
 994                                     ' from mediwiki: ' . $mediawiki->{error}->{details} .
 995                                     ".\n";
 996                                 return ($oldrevid, "non-fast-forward");
 997                         } else {
 998                                 # Other errors. Shouldn't happen => just die()
 999                                 die 'Fatal: Error ' .
1000                                     $mediawiki->{error}->{code} .
1001                                     ' from mediwiki: ' . $mediawiki->{error}->{details};
1002                         }
1003                 }
1004                 $newrevid = $result->{edit}->{newrevid};
1005                 print STDERR "Pushed file: $new_sha1 - $title\n";
1006         } else {
1007                 $newrevid = mw_upload_file($complete_file_name, $new_sha1,
1008                                            $extension, $page_deleted,
1009                                            $summary);
1010         }
1011         $newrevid = ($newrevid or $oldrevid);
1012         return ($newrevid, "ok");
1013 }
1014
1015 sub mw_push {
1016         # multiple push statements can follow each other
1017         my @refsspecs = (shift, get_more_refs("push"));
1018         my $pushed;
1019         for my $refspec (@refsspecs) {
1020                 my ($force, $local, $remote) = $refspec =~ /^(\+)?([^:]*):([^:]*)$/
1021                     or die("Invalid refspec for push. Expected <src>:<dst> or +<src>:<dst>");
1022                 if ($force) {
1023                         print STDERR "Warning: forced push not allowed on a MediaWiki.\n";
1024                 }
1025                 if ($local eq "") {
1026                         print STDERR "Cannot delete remote branch on a MediaWiki\n";
1027                         print STDOUT "error $remote cannot delete\n";
1028                         next;
1029                 }
1030                 if ($remote ne "refs/heads/master") {
1031                         print STDERR "Only push to the branch 'master' is supported on a MediaWiki\n";
1032                         print STDOUT "error $remote only master allowed\n";
1033                         next;
1034                 }
1035                 if (mw_push_revision($local, $remote)) {
1036                         $pushed = 1;
1037                 }
1038         }
1039
1040         # Notify Git that the push is done
1041         print STDOUT "\n";
1042
1043         if ($pushed && $dumb_push) {
1044                 print STDERR "Just pushed some revisions to MediaWiki.\n";
1045                 print STDERR "The pushed revisions now have to be re-imported, and your current branch\n";
1046                 print STDERR "needs to be updated with these re-imported commits. You can do this with\n";
1047                 print STDERR "\n";
1048                 print STDERR "  git pull --rebase\n";
1049                 print STDERR "\n";
1050         }
1051 }
1052
1053 sub mw_push_revision {
1054         my $local = shift;
1055         my $remote = shift; # actually, this has to be "refs/heads/master" at this point.
1056         my $last_local_revid = get_last_local_revision();
1057         print STDERR ".\n"; # Finish sentence started by get_last_local_revision()
1058         my $last_remote_revid = get_last_remote_revision();
1059         my $mw_revision = $last_remote_revid;
1060
1061         # Get sha1 of commit pointed by local HEAD
1062         my $HEAD_sha1 = run_git("rev-parse $local 2>/dev/null"); chomp($HEAD_sha1);
1063         # Get sha1 of commit pointed by remotes/$remotename/master
1064         my $remoteorigin_sha1 = run_git("rev-parse refs/remotes/$remotename/master 2>/dev/null");
1065         chomp($remoteorigin_sha1);
1066
1067         if ($last_local_revid > 0 &&
1068             $last_local_revid < $last_remote_revid) {
1069                 return error_non_fast_forward($remote);
1070         }
1071
1072         if ($HEAD_sha1 eq $remoteorigin_sha1) {
1073                 # nothing to push
1074                 return 0;
1075         }
1076
1077         # Get every commit in between HEAD and refs/remotes/origin/master,
1078         # including HEAD and refs/remotes/origin/master
1079         my @commit_pairs = ();
1080         if ($last_local_revid > 0) {
1081                 my $parsed_sha1 = $remoteorigin_sha1;
1082                 # Find a path from last MediaWiki commit to pushed commit
1083                 while ($parsed_sha1 ne $HEAD_sha1) {
1084                         my @commit_info =  grep(/^$parsed_sha1/, split(/\n/, run_git("rev-list --children $local")));
1085                         if (!@commit_info) {
1086                                 return error_non_fast_forward($remote);
1087                         }
1088                         my @commit_info_split = split(/ |\n/, $commit_info[0]);
1089                         # $commit_info_split[1] is the sha1 of the commit to export
1090                         # $commit_info_split[0] is the sha1 of its direct child
1091                         push(@commit_pairs, \@commit_info_split);
1092                         $parsed_sha1 = $commit_info_split[1];
1093                 }
1094         } else {
1095                 # No remote mediawiki revision. Export the whole
1096                 # history (linearized with --first-parent)
1097                 print STDERR "Warning: no common ancestor, pushing complete history\n";
1098                 my $history = run_git("rev-list --first-parent --children $local");
1099                 my @history = split('\n', $history);
1100                 @history = @history[1..$#history];
1101                 foreach my $line (reverse @history) {
1102                         my @commit_info_split = split(/ |\n/, $line);
1103                         push(@commit_pairs, \@commit_info_split);
1104                 }
1105         }
1106
1107         foreach my $commit_info_split (@commit_pairs) {
1108                 my $sha1_child = @{$commit_info_split}[0];
1109                 my $sha1_commit = @{$commit_info_split}[1];
1110                 my $diff_infos = run_git("diff-tree -r --raw -z $sha1_child $sha1_commit");
1111                 # TODO: we could detect rename, and encode them with a #redirect on the wiki.
1112                 # TODO: for now, it's just a delete+add
1113                 my @diff_info_list = split(/\0/, $diff_infos);
1114                 # Keep the subject line of the commit message as mediawiki comment for the revision
1115                 my $commit_msg = run_git("log --no-walk --format=\"%s\" $sha1_commit");
1116                 chomp($commit_msg);
1117                 # Push every blob
1118                 while (@diff_info_list) {
1119                         my $status;
1120                         # git diff-tree -z gives an output like
1121                         # <metadata>\0<filename1>\0
1122                         # <metadata>\0<filename2>\0
1123                         # and we've split on \0.
1124                         my $info = shift(@diff_info_list);
1125                         my $file = shift(@diff_info_list);
1126                         ($mw_revision, $status) = mw_push_file($info, $file, $commit_msg, $mw_revision);
1127                         if ($status eq "non-fast-forward") {
1128                                 # we may already have sent part of the
1129                                 # commit to MediaWiki, but it's too
1130                                 # late to cancel it. Stop the push in
1131                                 # the middle, but still give an
1132                                 # accurate error message.
1133                                 return error_non_fast_forward($remote);
1134                         }
1135                         if ($status ne "ok") {
1136                                 die("Unknown error from mw_push_file()");
1137                         }
1138                 }
1139                 unless ($dumb_push) {
1140                         run_git("notes --ref=$remotename/mediawiki add -m \"mediawiki_revision: $mw_revision\" $sha1_commit");
1141                         run_git("update-ref -m \"Git-MediaWiki push\" refs/mediawiki/$remotename/master $sha1_commit $sha1_child");
1142                 }
1143         }
1144
1145         print STDOUT "ok $remote\n";
1146         return 1;
1147 }
1148
1149 sub get_allowed_file_extensions {
1150         mw_connect_maybe();
1151
1152         my $query = {
1153                 action => 'query',
1154                 meta => 'siteinfo',
1155                 siprop => 'fileextensions'
1156                 };
1157         my $result = $mediawiki->api($query);
1158         my @file_extensions= map $_->{ext},@{$result->{query}->{fileextensions}};
1159         my %hashFile = map {$_ => 1}@file_extensions;
1160
1161         return %hashFile;
1162 }
1163
1164 # In memory cache for MediaWiki namespace ids.
1165 my %namespace_id;
1166
1167 # Namespaces whose id is cached in the configuration file
1168 # (to avoid duplicates)
1169 my %cached_mw_namespace_id;
1170
1171 # Return MediaWiki id for a canonical namespace name.
1172 # Ex.: "File", "Project".
1173 sub get_mw_namespace_id {
1174         mw_connect_maybe();
1175         my $name = shift;
1176
1177         if (!exists $namespace_id{$name}) {
1178                 # Look at configuration file, if the record for that namespace is
1179                 # already cached. Namespaces are stored in form:
1180                 # "Name_of_namespace:Id_namespace", ex.: "File:6".
1181                 my @temp = split(/[ \n]/, run_git("config --get-all remote."
1182                                                 . $remotename .".namespaceCache"));
1183                 chomp(@temp);
1184                 foreach my $ns (@temp) {
1185                         my ($n, $id) = split(/:/, $ns);
1186                         $namespace_id{$n} = $id;
1187                         $cached_mw_namespace_id{$n} = 1;
1188                 }
1189         }
1190
1191         if (!exists $namespace_id{$name}) {
1192                 print STDERR "Namespace $name not found in cache, querying the wiki ...\n";
1193                 # NS not found => get namespace id from MW and store it in
1194                 # configuration file.
1195                 my $query = {
1196                         action => 'query',
1197                         meta => 'siteinfo',
1198                         siprop => 'namespaces'
1199                 };
1200                 my $result = $mediawiki->api($query);
1201
1202                 while (my ($id, $ns) = each(%{$result->{query}->{namespaces}})) {
1203                         if (defined($ns->{id}) && defined($ns->{canonical})) {
1204                                 $namespace_id{$ns->{canonical}} = $ns->{id};
1205                                 if ($ns->{'*'}) {
1206                                         # alias (e.g. french Fichier: as alias for canonical File:)
1207                                         $namespace_id{$ns->{'*'}} = $ns->{id};
1208                                 }
1209                         }
1210                 }
1211         }
1212
1213         my $id = $namespace_id{$name};
1214
1215         if (defined $id) {
1216                 # Store explicitely requested namespaces on disk
1217                 if (!exists $cached_mw_namespace_id{$name}) {
1218                         run_git("config --add remote.". $remotename
1219                                 .".namespaceCache \"". $name .":". $id ."\"");
1220                         $cached_mw_namespace_id{$name} = 1;
1221                 }
1222                 return $id;
1223         } else {
1224                 die "No such namespace $name on MediaWiki.";
1225         }
1226 }