bin/lo-pack-sources

   1 #!/usr/bin/perl
   2     eval 'exec /usr/bin/perl -S $0 ${1+"$@"}'
   3         if $running_under_some_shell;
   4 #!/usr/bin/perl
   5
   6 use strict;
   7 use File::Copy;
   8 use File::Temp qw/ tempfile tempdir /;
   9
  10 # get libreoffice-build version from the given libreoffice-build sources
  11 sub get_config_version($)
  12 {
  13     my ($lo_build_dir) = @_;
  14     my $version;
  15
  16     open (CONFIGURE, "$lo_build_dir/configure.in") ||
  17         die "can't open \"$lo_build_dir/configure.in\" for reading: $!\n";
  18
  19     while (my $line = <CONFIGURE>) {
  20         chomp $line;
  21
  22         if ($line =~ /AC_INIT\s*\(\s*libreoffice-build\s*,\s*([\w\.]*)\)/) {
  23             $version="$1";
  24         }
  25     }
  26     close (CONFIGURE);
  27     return $version;
  28 }
  29
  30 # increment the version for a test build:
  31 #        + add 'a' if the version ended with a number
  32 #       + bump the letter otherwise
  33 sub inc_test_version($)
  34 {
  35     my ($version) = @_;
  36
  37     my $lastchar = chop $version;
  38     my $new_version;
  39
  40     if ($lastchar =~ /\d/) {
  41         return "$version" . "$lastchar" . "a";
  42     } elsif ($lastchar =~ /\w/) {
  43         # select next letter alhabeticaly: a->b, b->c, ...
  44         $lastchar =~ tr/0a-zA-Z/a-zA-Z0/;
  45         return "$version" . "$lastchar";
  46     } else {
  47         die "Can't generate test version from \"$version$lastchar\n";
  48     }
  49 }
  50
  51 sub get_release_version($$$$)
  52 {
  53     my ($config_version, $state_config_version, $state_release_version, $inc_version) = @_;
  54     my $release_version;
  55
  56     if (defined $state_config_version &&
  57         defined $state_release_version &&
  58         "$state_config_version" eq "$config_version") {
  59         $release_version = "$state_release_version";
  60     } else {
  61         $release_version = "$config_version";
  62     }
  63
  64     if ( defined $inc_version ) {
  65         $release_version = inc_test_version($release_version);
  66     }
  67
  68     return $release_version;
  69 }
  70
  71
  72 sub generate_lo_piece_blacklist($)
  73 {
  74     my ($blacklist) = @_;
  75
  76     # FIXME: crazy hacks to copy libreoffice-build without too big and useless subdirectories and to show a progress
  77     open (BLACKLIST, ">$blacklist") || die "Can't open $blacklist: $!\n";
  78
  79     # IMPORTANT: Do not remove .git directories because "git log" is called during "make dist"
  80     print BLACKLIST ".git\n";
  81     print BLACKLIST ".gitignore\n";
  82     print BLACKLIST ".gitattributes\n";
  83     print BLACKLIST "autom4te.cache/*\n";
  84     print BLACKLIST "autom4te.cache\n";
  85     print BLACKLIST "autogen.lastrun\n";
  86     print BLACKLIST "clone/*\n";
  87     print BLACKLIST "config.log\n";
  88     print BLACKLIST "git-hooks\n";
  89     print BLACKLIST "Env.Host.sh\n";
  90     print BLACKLIST "src/tmp*\n";
  91     print BLACKLIST "src/fetch.log\n";
  92     print BLACKLIST "src/libreoffice-*.tar.bz2\n";
  93     print BLACKLIST "src/????????????????????????????????-*\n";
  94     print BLACKLIST "warn\n";
  95
  96     close BLACKLIST;
  97 }
  98
  99 # remove symlinks to clone subdir
 100 sub remove_module_symlinks($$)
 101 {
 102     my ($tempdir, $piece_tarball_name) = @_;
 103
 104     print STDERR "find $tempdir/$piece_tarball_name -mindepth 1 -maxdepth 1 -type l -exec rm {} \\;\n";
 105     system ("find $tempdir/$piece_tarball_name -mindepth 1 -maxdepth 1 -type l -exec rm {} \\;") &&
 106         die "Error: can't delete symlinks to clone dir in $tempdir: $!\n";
 107 }
 108
 109
 110 # copy files to temp dir; showing a progress; using a black list
 111 sub copy_dir_filter_and_show_progress($$$)
 112 {
 113     my ($source_dir, $target_dir, $blacklist) = @_;
 114
 115     print "Copying \"$source_dir\" -> \"$target_dir\"...";
 116     # FIXME: crazy hacks to copy dir with a blacklist and showing a progress
 117     system ("tar -cf - -C $source_dir -X $blacklist \.\/ | " .
 118             "tar -xf - -C $target_dir --checkpoint 2>&1 | " .
 119             "awk '{ ORS=\"\" ; if (++nlines\%50 == 0) printf \".\"; fflush() }'") &&
 120         die "Error: copying failed: $!\n";
 121     print "\n";
 122 }
 123
 124 # copy the piece lo source directory into a tmp directory
 125 # omit the .git subdirectories
 126 sub copy_lo_piece_to_tempdir($$$)
 127 {
 128     my ($piece_dir, $piece, $piece_tarball_name) = @_;
 129
 130
 131     my $tempdir = tempdir( 'libreoffice-XXXXXX', DIR => File::Spec->tmpdir );
 132     my $blacklist = "$tempdir/libreoffice-$piece.copy.blacklist";
 133
 134     mkdir "$tempdir/$piece_tarball_name" || die "Can't create directory \"$tempdir/$piece_tarball_name\": $!\n";
 135
 136     generate_lo_piece_blacklist($blacklist);
 137     copy_dir_filter_and_show_progress("$piece_dir", "$tempdir/$piece_tarball_name", $blacklist);
 138     remove_module_symlinks($tempdir, $piece_tarball_name);
 139
 140     unlink $blacklist;
 141
 142     return $tempdir;
 143 }
 144
 145 sub generate_lo_piece_changelog($$$)
 146 {
 147     my ($lo_piece_clone, $lo_piece_release_dir, $piece) = @_;
 148     print "Generating changelog for $piece...\n";
 149     print "1:$lo_piece_clone, 2:$lo_piece_release_dir, 3:$piece\n";
 150     # FIXME: crazy hacks to copy dir with a blacklist and showing a progress
 151     system ("cd $lo_piece_clone && " .
 152             "git log --date=short --pretty='format:@%cd  %an  <%ae>  [%H]%n%n%s%n%n%e%b' | " .
 153             "  sed -e 's|^\([^@]\)|\t\1|' -e 's|^@||' >$lo_piece_release_dir/ChangeLog" ) &&
 154         die "Error: generating failed: $!\n";
 155 }
 156
 157 sub run_autoreconf($$)
 158 {
 159     my ($dir, $piece) = @_;
 160
 161     print "Running autoreconf for $piece...\n";
 162     system ("cd $dir && " .
 163             "autoreconf -f -i && " .
 164             "rm -rf autom4te.cache && " .
 165             "cd - >/dev/null 2>&1") && die "Error: autoreconf failed: $!\n";
 166 }
 167
 168 sub generate_version_file($$$)
 169 {
 170     my ($dir, $piece, $release_version) = @_;
 171
 172     # FIXME: crazy hacks to copy libreoffice-build without too big and useless subdirectories and to show a progress
 173     open (VERFILE, ">$dir/$piece.ver") || die "Can't open $dir/lo-$piece.ver: $!\n";
 174
 175     print VERFILE "lo_core_ver=$release_version\n";
 176
 177     close VERFILE;
 178 }
 179
 180 sub release_lo_piece($$)
 181 {
 182     my ($lo_piece_dir, $piece_tarball_name) = @_;
 183
 184     print "Creating $piece_tarball_name.tar.bz2...";
 185     system ("cd $lo_piece_dir && " .
 186             "tar -cjf $piece_tarball_name.tar.bz2 --checkpoint * 2>&1 | awk '{ ORS=\"\" ; if (++nlines\%50 == 0) printf \".\"; fflush() }' && " .
 187             "cd - >/dev/null 2>&1") && die "Error: releasing failed: $!\n";
 188     print "\n";
 189 }
 190
 191 sub generate_md5($$$)
 192 {
 193     my ($dir, $tarball_name, $tarball_suffix) = @_;
 194
 195     print "Generating MD5...\n";
 196     system ("cd $dir && " .
 197             "md5sum $tarball_name$tarball_suffix >$tarball_name$tarball_suffix.md5 && " .
 198             "cd - >/dev/null 2>&1") && die "Error: releasing failed: $!\n";
 199 }
 200
 201 sub default_releases_state_file($)
 202 {
 203     my ($lo_build_dir) = @_;
 204
 205     my $rootdir = $lo_build_dir;
 206     $rootdir =~ s/^(.*?)\/?[^\/]+\/?$/$1/;
 207
 208     my $releases_state_file;
 209     if ($rootdir) {
 210         $releases_state_file = "$rootdir/.releases";
 211     } else {
 212         $releases_state_file = ".releases";
 213     }
 214
 215     return "$releases_state_file";
 216 }
 217
 218 sub default_releases_archive($)
 219 {
 220     my ($lo_build_dir) = @_;
 221
 222     my $rootdir = $lo_build_dir;
 223     $rootdir =~ s/^(.*?)\/?[^\/]+\/?$/$1/;
 224
 225     my $releases_archive_dir;
 226     if ($rootdir) {
 227         $releases_archive_dir = "$rootdir/archive";
 228     } else {
 229         $releases_archive_dir = "archive";
 230     }
 231
 232     return "$releases_archive_dir";
 233 }
 234
 235 sub load_releases_state($)
 236 {
 237     my ($releases_state_file) = @_;
 238
 239     my $state_config_version;
 240     my $state_release_version;
 241
 242     if (open (STATE, "$releases_state_file")) {
 243
 244         while (my $line = <STATE>) {
 245             chomp $line;
 246
 247             if ($line =~ /^\s*configure_version\s*=\s*(.*)$/) {
 248                 $state_config_version = "$1";
 249             } elsif ($line =~ /^\s*released_version\s*=\s*(.*)$/) {
 250                 $state_release_version = "$1";
 251             }
 252         }
 253             close (STATE);
 254     }
 255
 256     return $state_config_version, $state_release_version;
 257 }
 258
 259 sub save_releases_state($$$)
 260 {
 261     my ($releases_state_file, $config_version, $release_version) = @_;
 262
 263     open (STATE, '>', "$releases_state_file") ||
 264         die "Can't open \"$releases_state_file\" for writing: $!\n";
 265
 266     print STATE "configure_version = $config_version\n";
 267     print STATE "released_version  = $release_version\n";
 268
 269     close (STATE);
 270 }
 271
 272 sub remove_tempdir($)
 273 {
 274     my ($tempdir) = @_;
 275
 276 #    print "Cleaning $tempdir...\n";
 277     system ("rm -rf $tempdir") && die "Error: rm failed: $!\n";
 278 }
 279
 280 sub save_file($$$)
 281 {
 282     my ($source_dir, $target_dir, $file) = @_;
 283
 284     unless ( -d "$target_dir" ) {
 285         mkdir ("$target_dir") ||
 286             die "Can't create directory \"$target_dir\": $!\n";
 287     }
 288
 289     if ( -f "$target_dir/$file" ) {
 290         print "Warning: $target_dir/$file already exists and will be replaced\n";
 291         unlink ("$target_dir/$file");
 292     }
 293
 294     print "Copying into archive: $target_dir/$file ...\n";
 295     copy ("$source_dir/$file", "$target_dir/$file") ||
 296         die "Error: Can't copy $source_dir/$file to $target_dir/$file: $!\n";
 297 }
 298
 299 sub check_if_file_exists($$)
 300 {
 301     my ($file, $force) = @_;
 302
 303     if (-e $file) {
 304         if (defined $force) {
 305             print "Warning: $file already exists and will be replaced!\n";
 306         } else {
 307             die "Error: $file alrady exists.\n".
 308                 "       Use --force if you want to replace it.\n";
 309         }
 310     }
 311 }
 312
 313 sub check_if_tarball_already_released($$$)
 314 {
 315     my ($tarball, $releases_archive_dir, $force) = @_;
 316
 317     check_if_file_exists($tarball, $force);
 318     check_if_file_exists("$releases_archive_dir/$tarball", $force) if (defined $releases_archive_dir);
 319 }
 320
 321 sub check_if_already_released($$$$$$)
 322 {
 323     my ($lo_build_tarball_name, $p_piece_tarball_name, $releases_archive_dir, $force, $pack_lo_build, $pack_lo_pieces) = @_;
 324
 325     check_if_tarball_already_released("$lo_build_tarball_name.tar.gz", $releases_archive_dir, $force) if ($pack_lo_build);
 326
 327     if ($pack_lo_pieces) {
 328         foreach my $tarball_name ( values %{$p_piece_tarball_name} ) {
 329             check_if_tarball_already_released("$tarball_name.tar.bz2", $releases_archive_dir, $force);
 330         }
 331     }
 332 }
 333
 334 sub prepare_lo_piece_tarball($$$$$)
 335 {
 336     my ($piece_dir, $release_version, $md5, $piece, $piece_tarball_name) = @_;
 337
 338     my $temp_dir = copy_lo_piece_to_tempdir($piece_dir, $piece, $piece_tarball_name);
 339     generate_lo_piece_changelog($piece_dir, "$temp_dir/$piece_tarball_name", $piece);
 340     run_autoreconf("$temp_dir/$piece_tarball_name", $piece) if ($piece eq 'core');
 341     generate_version_file("$temp_dir/$piece_tarball_name", $piece, $release_version) if ($piece eq 'core');
 342     release_lo_piece($temp_dir, $piece_tarball_name);
 343     generate_md5($temp_dir, $piece_tarball_name, ".tar.bz2") if (defined $md5);
 344
 345     return $temp_dir;
 346 }
 347
 348 sub move_tarball_to_final_location($$$$)
 349 {
 350     my ($temp_dir, $releases_archive_dir, $md5, $tarball) = @_;
 351
 352     save_file($temp_dir, ".", "$tarball");
 353     save_file($temp_dir, ".", "$tarball.md5") if (defined $md5);
 354     if ( defined $releases_archive_dir ) {
 355         save_file($temp_dir, $releases_archive_dir, "$tarball");
 356         save_file($temp_dir, $releases_archive_dir, "$tarball.md5") if (defined $md5);
 357     }
 358
 359     remove_tempdir($temp_dir);
 360 }
 361
 362
 363 sub generate_tarballs($$$$$$$$$)
 364 {
 365     my ($source_dir, $releases_archive_dir, $release_version, $md5, $lo_build_tarball_name, $p_piece_tarball_name, $pack_lo_build, $pack_lo_pieces, $is_lo_build_dir) = @_;
 366
 367     if ($pack_lo_build) {
 368         my $temp_dir=prepare_lo_piece_tarball($source_dir, $release_version, $md5, "core", $lo_build_tarball_name);
 369         move_tarball_to_final_location($temp_dir, $releases_archive_dir, $md5, "$lo_build_tarball_name.tar.bz2");
 370     }
 371
 372     if ($pack_lo_pieces) {
 373         my $piece_dir = $source_dir;
 374         foreach my $piece ( keys %{$p_piece_tarball_name} ) {
 375             print "\n--- Generating $piece ---\n";
 376             $piece_dir = "$source_dir/clone/$piece" if ($is_lo_build_dir);
 377             my $temp_dir=prepare_lo_piece_tarball($piece_dir, $release_version, $md5, $piece, $p_piece_tarball_name->{$piece});
 378             move_tarball_to_final_location($temp_dir, $releases_archive_dir, $md5, "$p_piece_tarball_name->{$piece}.tar.bz2");
 379         }
 380     }
 381
 382 }
 383
 384
 385 sub usage()
 386 {
 387     print "This tool helps to pack the libreoffice-build and piece sources\n\n" .
 388
 389           "Usage:\n".
 390           "\tlo-pack-sources [--help] [--force] [--version]\n" .
 391           "\t [--set-version=<ver>] [--inc-version] [--md5]\n" .
 392           "\t [--no-lo-build] [--no-lo-pieces] [--piece=<piece>]\n" .
 393           "\t [dir]\n\n" .
 394
 395           "Options:\n\n" .
 396           "\t--help: print this help\n" .
 397           "\t--force: replace an already existing release of the same version\n" .
 398           "\t--version: just print version of the released package but do not\n" .
 399           "\t\trelease it; the version is affected by the other options, e.g.\n" .
 400           "\t\t--inc-version\n" .
 401           "\t--set-version: force another version\n" .
 402           "\t--inc-version: increment the latest version; there is a difference\n" .
 403           "\t\tbetween test release (default) and final (not yet supported)\n" .
 404           "\t--md5: generate md5 sum for the final tarball\n" .
 405           "\t--no-lo-build: do not pack the libreoffice-build tarball\n" .
 406           "\t--no-lo-pieces: do not pack the libreoffice-build piece sources\n" .
 407           "\t--piece=<piece>: pack just a single piece, .e.g. \"writer\",\n" .
 408           "\tdir: path of the source directory, either libreoffice-build or piece\n";
 409 }
 410
 411
 412 my $ptf;
 413 my $md5;
 414 my $inc_version;
 415 my $config_version;
 416 my $set_version;
 417 my $get_config_version;
 418 my $release_version;
 419 my $pack_lo_build=1;
 420 my $pack_lo_pieces=1;
 421 my $source_dir;
 422 my $releases_archive_dir;
 423 my $releases_state_file;
 424 my $state_config_version;
 425 my $state_release_version;
 426 my $lo_build_tarball_name;
 427 my $lo_build_tempdir;
 428 my $force;
 429 my $verbose=1;
 430 my $is_lo_build_dir=0;
 431 my @pieces=("binfilter", "dictionaries", "help", "translations");
 432 my %piece_tarball_name;
 433
 434 ###################
 435 # Arguments parsing
 436 ###################
 437
 438 for my $arg (@ARGV) {
 439     if ($arg eq '--help' || $arg eq '-h') {
 440         usage;
 441         exit 0;
 442     } elsif ($arg eq '--force') {
 443         $force=1;
 444     } elsif ($arg eq '--md5') {
 445         $md5=1;
 446     } elsif ($arg eq '--version') {
 447         $get_config_version=1;
 448         $verbose = undef;
 449     } elsif ($arg eq '--inc-version') {
 450         $inc_version=1
 451     } elsif ($arg =~ m/--set-version=(.*)/) {
 452           $set_version="$1";
 453     } elsif ($arg eq '--no-lo-build') {
 454         $pack_lo_build=0;
 455     } elsif ($arg eq '--no-lo-pieces') {
 456         $pack_lo_pieces=0;
 457     } elsif ($arg =~ m/--piece=(.*)/) {
 458           # process just one piece and do not pack libreoffice-build
 459           @pieces=();
 460           push @pieces, "$1";
 461           $pack_lo_build=0;
 462     } elsif ($arg =~ /^-/ ) {
 463         die "Error: unknown option: $arg\n";
 464     } else {
 465         if (! defined $source_dir) {
 466             $source_dir = $arg;
 467         } else {
 468             die "Error: Too many arguments $arg\n";
 469         }
 470     }
 471 }
 472
 473 ###################
 474 # Initial checks
 475 ###################
 476
 477 unless ( defined $source_dir ) {
 478     die "Error: undefined source directory, try --help\n";
 479 }
 480
 481 unless ( -d "$source_dir" ) {
 482     die "Error: is not a directory: $source_dir\n";
 483 }
 484
 485 # check if it is a valid libreoffice-core directory
 486 $is_lo_build_dir=1 if (-f "$source_dir/autogen.sh" && -f "$source_dir/set_soenv.in");
 487
 488 # all tarballs are generated from the libreoffice-core directory
 489 if (@pieces > 1 && $is_lo_build_dir == 0 ) {
 490     die "Error: \"$source_dir\" is not a valid libreoffice-core directory\n";
 491 }
 492
 493 # just a single piece tarball can be generated from piece directory; version must be explicitly set in this case
 494 if (@pieces == 1  &&  $is_lo_build_dir == 0  &&  ! defined $set_version ) {
 495     die "Error: version must be set using the --set-version=<version> option\n" unless (defined $set_version);
 496 }
 497
 498 if (defined $set_version && defined $inc_version) {
 499     die "Error: --set-version and --inc-version options can't be used together\n";
 500 }
 501
 502
 503 ###################
 504 # Main logic
 505 ###################
 506
 507
 508 print "Source: $source_dir\n" if ($verbose);
 509
 510 if ($is_lo_build_dir) {
 511     # detect some paths
 512     $releases_state_file = default_releases_state_file($source_dir) unless (defined $releases_state_file);
 513     $releases_archive_dir = default_releases_archive($source_dir) unless (defined $releases_archive_dir);
 514
 515     # detect versions
 516     $config_version = get_config_version($source_dir);
 517     ($state_config_version, $state_release_version) = load_releases_state($releases_state_file);
 518     if (defined $set_version) {
 519         $release_version = "$set_version";
 520     } else {
 521         $release_version = get_release_version($config_version, $state_config_version, $state_release_version, $inc_version);
 522     }
 523 } else {
 524     # must be single piece release with predefined version
 525     $release_version = "$set_version";
 526 }
 527
 528 # define tarball names
 529 $lo_build_tarball_name = "libreoffice-core-$release_version";
 530 foreach my $piece (@pieces) {
 531     if (-d "$source_dir/clone/$piece") {
 532         $piece_tarball_name{$piece} = "libreoffice-$piece-$release_version";
 533     } else {
 534         print "Warning: $piece sources are not available -> skipping\n";
 535     }
 536 }
 537
 538 print "Default version   : $config_version\n" if ($verbose && defined $config_version);
 539 print "Last used version : $state_release_version\n" if ($verbose && defined $state_release_version);
 540 print "New version       : $release_version\n" if ($verbose);
 541
 542 # do the real job
 543 if ( defined $get_config_version ) {
 544     print "$release_version\n";
 545 } else {
 546     check_if_already_released($lo_build_tarball_name, \%piece_tarball_name, $releases_archive_dir, $force, $pack_lo_build, $pack_lo_pieces);
 547
 548     # give a chance to stop the process
 549     print ("\nWaiting 3 seconds...\n");
 550     sleep 3;
 551
 552     generate_tarballs($source_dir, $releases_archive_dir, $release_version, $md5, $lo_build_tarball_name, \%piece_tarball_name, $pack_lo_build, $pack_lo_pieces, $is_lo_build_dir);
 553
 554     if ( defined $releases_state_file ) {
 555         save_releases_state($releases_state_file, $config_version, $release_version);
 556     }
 557 }