xapian-maintainer-tools/make-xapian-git-snapshot-tarballs

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright (C) 2011,2012,2013,2015 Olly Betts
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to
   7 # deal in the Software without restriction, including without limitation the
   8 # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
   9 # sell copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22
  23 use strict;
  24 use Fcntl ':flock';
  25 use File::Find;
  26 use File::Path;
  27 use Sys::Hostname;
  28 use POSIX;
  29
  30 my $MAXLOADPERCPU = 1.5;
  31
  32 sub run_command {
  33     my $cmd = shift;
  34     my $out = `$cmd`;
  35     chomp $out;
  36     return $out;
  37 }
  38
  39 my $force = 0;
  40 if (scalar @ARGV && $ARGV[0] eq '--force') {
  41     shift @ARGV;
  42     $force = 1;
  43 }
  44
  45 # Configuration:
  46 my $work = "/home/olly/tmp/xapian-git-snapshot";
  47 # Add ccache's directory first so we'll use it if it is installed.
  48 $ENV{PATH} = "/usr/lib/ccache:/home/olly/install/bin:/usr/bin:/bin";
  49 my $repourl = '/home/xapian-git/xapian/';
  50 my $webbase = '/srv/www';
  51
  52 # Create the work directory first, since we need it to exist so we can create
  53 # the lockfile.
  54 mkpath($work, 0, 0755);
  55 chdir $work or die $!;
  56
  57 # Prevent multiple instances of this script from running at once.
  58 # Use the same lockfile that fetch does.
  59 open LOCK, '>', 'flockme' or die "Couldn't open 'flockme' for writing: $!\n";
  60 unless (flock LOCK, LOCK_EX|LOCK_NB) {
  61     # If we're building a tagged release, we want to wait rather than exit.
  62     unless (scalar @ARGV && $ARGV[0] =~ m!^tags/! && flock LOCK, LOCK_EX) {
  63         # Work directory already in use.  Don't print anything unless STDOUT
  64         # is a tty - the cron job will send it as mail, which we don't really
  65         # want.
  66         print STDERR "'flockme' is already locked, can't build '$ARGV[0]' right now\n"
  67             if -t STDERR;
  68         exit 1;
  69     }
  70 }
  71
  72 # Check the load average AFTER getting the lock, since we generate output if
  73 # the load is too high, and it will probably be too high if we're already
  74 # running.
  75 my $HOSTNAME = Sys::Hostname::hostname();
  76 # Check the load average isn't too high.
  77 if (!$force) {
  78     if (-e "/var/run/dobackup") {
  79         print STDERR "$HOSTNAME: Backup running (/var/run/dobackup exists)\n"
  80             if -t STDERR;
  81         exit(1);
  82     }
  83     if (((`uptime 2>/dev/null`)[0] =~ /.*: (\d+(?:\.\d+)?),/) &&
  84         ($1 > $MAXLOADPERCPU)) {
  85         my $loadavg = $1;
  86         # `getconf _NPROCESSORS_ONLN` on linux gives e.g. 2
  87         # `sysctl hw.ncpu` on openbsd (and prob. freebsd & darwin) gives e.g. hw.ncpu=2
  88         # `psrinfo|grep -c on-line` on Solaris or OSF/1 gives e.g. 2
  89         my $ncpu;
  90         # Works on Linux, at least back to kernel 2.2.26.
  91         $ncpu ||= run_command("getconf _NPROCESSORS_ONLN 2>/dev/null|grep -v '[^0-9]'");
  92         # Works on OpenBSD (and apparently FreeBSD and Darwin).
  93         $ncpu ||= run_command("sysctl hw.ncpu 2>/dev/null|sed 's/.*=//'");
  94         # Works on Solaris and OSF/1.
  95         $ncpu ||= run_command("PATH=/usr/sbin:\$PATH psrinfo 2>/dev/null|grep -c on-line");
  96         # Works on Linux, just in case the getconf version doesn't.  Different
  97         # architectures have different formats for /proc/cpuinfo so this won't
  98         # work as widely as getconf _NPROCESSORS_ONLN will.
  99         $ncpu ||= run_command("grep -c processor /proc/cpuinfo 2>/dev/null");
 100         $ncpu ||= 1;
 101         if ($loadavg > $ncpu * $MAXLOADPERCPU) {
 102             $ncpu ||= "unknown";
 103             print STDERR "$HOSTNAME: High load average: $loadavg ($ncpu CPUs)\n";
 104             exit(1);
 105         }
 106     }
 107 }
 108
 109 # If no tags specified, default to those which there are directories for.
 110 # Don't bother to build tags/* since those (generally) don't change.
 111 if (scalar @ARGV == 0) {
 112     @ARGV = grep /\/[A-Za-z0-9][-A-Za-z0-9_.]*$/ && -d $_,
 113         glob('branches/[A-Za-z0-9]*');
 114     -d "trunk" && unshift @ARGV, "trunk";
 115     -d "master" && unshift @ARGV, "master";
 116 }
 117
 118 # Or if there are no directories, default to the trunk.
 119 if (scalar @ARGV == 0) {
 120     @ARGV = 'trunk';
 121 }
 122
 123 my $status = 0;
 124 foreach my $tag (@ARGV) {
 125     chdir $work or die $!;
 126     # Restrict tags to sane characters.
 127     next if $tag !~ /^[-A-Za-z0-9_.\/]+$/;
 128     if (! -d $tag) {
 129         print "*** No directory for '$tag'\n";
 130         $status = 1;
 131         next;
 132     }
 133
 134     my $old_cwd = `pwd`;
 135     chomp($old_cwd);
 136
 137     my $logfile = "$tag/snapshot.log";
 138     my $log = '';
 139     my $co_dir = "$tag/xapian";
 140     if (! -d "$co_dir/.git") {
 141         system "chmod", "-R", "+w", $co_dir if -d $co_dir;
 142         system "rm", "-rf", $co_dir;
 143         # Check out the tag into a directory of the same name.
 144         my $branch = $tag;
 145         $branch =~ s!^branches/!svn/! || $branch =~ s!^tags/!! ||
 146             ($branch =~ s/^trunk$/master/);
 147         open CMD, "git clone --branch \Q$branch\E \Q$repourl\E \Q$co_dir\E 2>&1|" or die $!;
 148         $log = join '', <CMD>;
 149         close CMD or do { print $log; die $!; };
 150         chdir $co_dir or die $!;
 151     } else {
 152         # Revert any local changes.
 153         chdir $co_dir or die $!;
 154         $log = "git reset --hard:\n".`git reset --hard 2>&1`."git pull --ff-only:\n";
 155         open CMD, "git pull --ff-only 2>&1|" or die $!;
 156         my $changed = 1;
 157         while (<CMD>) {
 158             $log .= $_;
 159             if ($changed && !$force) {
 160                 if (/^Already up-to-date/) {
 161                     $changed = 0;
 162                 }
 163             }
 164         }
 165         close CMD or die $!;
 166         if (!$changed) {
 167             chdir $old_cwd or die $!;
 168             $ENV{'VERBOSE'} and print "No changes\n";
 169             next;
 170         }
 171     }
 172     my ($revision) = `git describe --always`;
 173     chomp($revision);
 174     my ($revcount) = ($revision =~ /-([0-9]+)-/);
 175     if (!defined $revcount) {
 176         # Workaround until we have git tags - count commits since 1.3.1.
 177         $revcount = scalar(@{[`git log --format='%H' 1daf0071342d883ca308762f269a63f6ec5df981..`]});
 178         $revision = "v1.3.1-$revcount-g$revision";
 179     }
 180
 181     chdir $old_cwd or die $!;
 182
 183     # Don't repeat a build for the same revision.
 184     next if -f "$logfile.$revision";
 185
 186     open LOG, ">", "$logfile.$revision" or die $!;
 187     # Flush output after every print.
 188     my $old_fh = select(LOG);
 189     $| = 1;
 190     select($old_fh);
 191
 192     print LOG $log;
 193     $log = undef;
 194
 195     if ($tag !~ m!^tags/!) {
 196         # Modify configure.ac files to insert $revision into version string.
 197         foreach my $configure_ac
 198             (glob("\Q$co_dir\E/xapian*/configure.ac"),
 199              glob("\Q$co_dir\E/xapian*/*/configure.ac")) {
 200             open OUT, ">", "tmp.out" or die $!;
 201             open IN, "<", $configure_ac or die $!;
 202             while (<IN>) {
 203                 s/(^(?:AC_INIT\([^,]*|m4_define\(\[project_version\]),.*?[0-9])(\s*[),\]])/$1_git$revcount$2/g;
 204                 print OUT;
 205             }
 206             close IN or die $!;
 207             close OUT or die $!;
 208             rename "tmp.out", $configure_ac or die $!;
 209         }
 210         if (-f  "$co_dir/search-xapian/Makefile.PL") {
 211             my $snap_version;
 212             my $fnm = "$co_dir/search-xapian/Xapian.pm";
 213             open OUT, ">", "tmp.out" or die $!;
 214             open IN, "<", $fnm or die $!;
 215             while (<IN>) {
 216                 if (s/^(our \$VERSION = ')(\d+\.\d+\.\d+\.)\d+(.*)/$1$2$revcount$3 # git snapshot/) {
 217                     $snap_version = $2 . $revcount;
 218                 }
 219                 print OUT;
 220             }
 221             close IN or die $!;
 222             close OUT or die $!;
 223             rename "tmp.out", $fnm or die $!;
 224
 225             $fnm = "$co_dir/search-xapian/README";
 226             open OUT, ">", "tmp.out" or die $!;
 227             open IN, "<", $fnm or die $!;
 228             $_ = <IN>;
 229             s/(\d+\.\d+\.\d+\.)\d+/$1$revcount (git snapshot)/;
 230             print OUT;
 231             while (<IN>) {
 232                 print OUT;
 233             }
 234             close IN or die $!;
 235             close OUT or die $!;
 236             rename "tmp.out", $fnm or die $!;
 237
 238             $fnm = "$co_dir/search-xapian/Changes";
 239             open OUT, ">", "tmp.out" or die $!;
 240             open IN, "<", $fnm or die $!;
 241             while (<IN>) {
 242                 print OUT;
 243                 last if /^\s*$/;
 244             }
 245             print OUT $snap_version.strftime("  %a %b %e %H:%M:%S %Z %Y\n",gmtime());
 246             print OUT "\t- git snapshot of revision $revision.\n\n";
 247             while (<IN>) {
 248                 print OUT;
 249             }
 250             close IN or die $!;
 251             close OUT or die $!;
 252             rename "tmp.out", $fnm or die $!;
 253         }
 254     }
 255
 256     system "chmod", "-R", "+w", "$tag/build" if -d "$tag/build";
 257     system "rm", "-rf", "$tag/build";
 258     mkpath("$tag/build", 0, 0755) or die $!;
 259     chdir "$tag/build" or die $!;
 260
 261     # Note the current time so we can find sources which weren't used during
 262     # the build.  Sleep for a couple of seconds first to avoid needing to
 263     # worry about timestamps equal to $timestamp.
 264     sleep 2;
 265
 266     # Skip xapian-letor for now, as atreus lacks libsvm-dev, or the ability
 267     # to install a new enough one from a package.
 268     open TOUCH, '>>', '../xapian/xapian-letor/.nobootstrap' and close TOUCH;
 269
 270     open F, '<', '../xapian/bootstrap' or die $!;
 271     <F>;
 272     my $timestamp = (stat F)[8];
 273     close F;
 274
 275     $log = `../xapian/bootstrap 2>&1`;
 276     print LOG $log;
 277     if ($?) {
 278         print "*** bootstrap failed for '$tag':";
 279         print $log;
 280         $status = 1;
 281         next;
 282     }
 283     $log = undef;
 284
 285     # On atreus, javac keeps going into memory eating loops, so don't try to
 286     # build the java bindings.
 287     $log = `../xapian/configure --enable-quiet --enable-maintainer-mode --without-java 2>&1`;
 288     print LOG $log;
 289     if ($?) {
 290         print "*** configure failed for '$tag':";
 291         print $log;
 292         $status = 1;
 293         next;
 294     }
 295     $log = undef;
 296
 297     $log = `make -s 2>&1`;
 298     print LOG $log;
 299     if ($?) {
 300         print "*** make failed for '$tag':";
 301         print $log;
 302         $status = 1;
 303         next;
 304     }
 305     $log = undef;
 306
 307     my %unused_files = ();
 308     sub check_unused_files_from_build {
 309         return if $File::Find::name eq '../xapian';
 310         my $f = substr($File::Find::name, length('../xapian/'));
 311         if ($_ eq 'autom4te.cache' ||
 312             $_ eq 'debian' ||
 313             $f eq 'search-xapian/blib' ||
 314             $f eq 'swig' ||
 315             $f eq 'xapian-applications/omega/common' ||
 316             $f eq 'xapian-data' || # FIXME: make check should use this
 317             $f eq 'xapian-maintainer-tools' ||
 318             $f eq 'BUILD' ||
 319             $f eq 'INST' ||
 320             $f eq '.git' ||
 321             /^Search-Xapian-\d+\.\d+\.\d+\.\d+$/) {
 322             if (-d $File::Find::name) {
 323                 # Don't descend into these subdirectories.
 324                 $File::Find::prune = 1;
 325                 return;
 326             }
 327         }
 328         return unless -f $File::Find::name and (stat _)[8] < $timestamp;
 329         return if $_ eq '.gitignore';
 330         return if $_ eq 'config.h.in~';
 331         return if $_ eq 'NEWS.SKELETON';
 332         return if $f eq 'README';
 333         return if $f eq 'Vagrantfile';
 334         return if $f eq '.travis.yml';
 335         return if /^Search-Xapian-\d+\.\d+\.\d+\.\d+\.tar\..z$/;
 336         ++$unused_files{$f};
 337         print "Unused during make: $f\n";
 338     }
 339     find(\&check_unused_files_from_build, '../xapian');
 340
 341     my $lib = (<xapian-core/.libs/libxapian*.so>)[0];
 342     my $unstripped_so = -s $lib;
 343     $log = `strip \Q$lib\E`;
 344     print LOG $log;
 345     $log = undef;
 346     my $stripped_so = -s $lib;
 347
 348     open SIZELOG, ">>/home/olly/xapian-autobuild-stats.log";
 349     print SIZELOG "$tag\trev=$revision\tunstripped_so=$unstripped_so\tstripped_so=$stripped_so\n";
 350     close SIZELOG;
 351
 352     $log = `make -s distcheck VALGRIND= 2>&1`;
 353     print LOG $log;
 354     if ($?) {
 355         print "*** make distcheck failed for '$tag':";
 356         print $log;
 357         $status = 1;
 358         next;
 359     }
 360     $log = undef;
 361
 362     # This finds files we don't ship or use to get to what we ship:
 363     find(\&check_unused_files_from_build, '../xapian');
 364
 365     # MSVC build files aren't up to date, so don't snapshot them for now.
 366     if (0) {
 367         # Snapshots of MSVC build files.
 368         my $win32dir = "win32msvc_$revision";
 369         unlink $win32dir;
 370         symlink "../xapian/xapian-maintainer-tools/win32msvc", $win32dir;
 371         $log = `tar --dereference --exclude .git -acvf \Q$win32dir\E.tar.xz \Q$win32dir\E 2>&1`;
 372         unlink $win32dir;
 373         print LOG $log;
 374         if ($?) {
 375             print "*** tar win32msvc failed for '$tag':";
 376             print $log;
 377             $status = 1;
 378             next;
 379         }
 380     }
 381
 382     my $d = "$webbase/oligarchy.co.uk/xapian/$tag";
 383     if ($tag =~ m!^tags/v?([\d.]+)$!) {
 384         $d = "$webbase/oligarchy.co.uk/xapian/$1";
 385     }
 386     if (! -d $d) {
 387         mkpath($d, 0, 0755) or die $!;
 388         open HTACCESS, ">", "$d/.htaccess" or die $!;
 389         print HTACCESS "IndexOptions NameWidth=*\n";
 390         close HTACCESS or die $!;
 391     } else {
 392         if (-d "$d/old") {
 393             # Delete snapshots more than a week old, but leave at least one.
 394             my $num_of_days_to_keep = 7;
 395             my @o = glob "$d/old/*.tar.?z";
 396             my $n = scalar @o;
 397             @o = grep {-M $_ > $num_of_days_to_keep} @o;
 398             $n -= scalar @o;
 399             unlink @o if $n > 0;
 400         } else {
 401             mkdir "$d/old", 0755 or die $!;
 402             open HTACCESS, ">", "$d/old/.htaccess" or die $!;
 403             print HTACCESS "IndexOptions NameWidth=*\n";
 404             close HTACCESS or die $!;
 405         }
 406         for (glob "$d/*.tar.?z") {
 407             my ($leaf) = m!([^/]*)$!;
 408             rename $_, "$d/old/$leaf";
 409         }
 410     }
 411     for (glob("*/*.tar.?z"), glob("xapian-applications/*/*.tar.?z"), glob("*.tar.?z")) {
 412         print LOG "Moving '$_' to '$d'\n";
 413         system("mv", $_, $d);
 414         if ($?) {
 415             print LOG "Failed with exit code $?\n";
 416         } else {
 417             print LOG "OK\n";
 418         }
 419     }
 420     for (glob("search-xapian/*.tar.?z")) {
 421         print LOG2 "Moving '$_' to '$d'\n";
 422         system("mv", $_, $d);
 423         if ($?) {
 424             print LOG2 "Failed with exit code $?\n";
 425         } else {
 426             print LOG2 "OK\n";
 427         }
 428     }
 429     chdir("..");
 430     system("rm -rf xapian/search-xapian/Search-Xapian-*");
 431     close LOG;
 432     # Expire logs more than 10 days old
 433     unlink grep {-M $_ > 10} glob 'snapshot.log.*';
 434 }
 435
 436 system("/home/olly/bin/plot-sizes");
 437 exit($status);