Update to latest config.guess and config.sub
[xapian.git] / xapian-maintainer-tools / make-xapian-git-snapshot-tarballs
blob70400afc7506f1269328f0fee84c546c25c6002f
1 #!/usr/bin/perl -w
3 # Copyright (C) 2011,2012,2013,2015 Olly Betts
4 #
5 # Permission is hereby granted, free of charge, to any person obtaining a copy
6 # of this software and associated documentation files (the "Software"), to
7 # deal in the Software without restriction, including without limitation the
8 # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
9 # sell copies of the Software, and to permit persons to whom the Software is
10 # furnished to do so, subject to the following conditions:
12 # The above copyright notice and this permission notice shall be included in
13 # all copies or substantial portions of the Software.
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 # IN THE SOFTWARE.
23 use strict;
24 use Fcntl ':flock';
25 use File::Find;
26 use File::Path;
27 use Sys::Hostname;
28 use POSIX;
30 my $MAXLOADPERCPU = 1.5;
32 sub run_command {
33 my $cmd = shift;
34 my $out = `$cmd`;
35 chomp $out;
36 return $out;
39 my $force = 0;
40 if (scalar @ARGV && $ARGV[0] eq '--force') {
41 shift @ARGV;
42 $force = 1;
45 # Configuration:
46 my $work = "/home/olly/tmp/xapian-git-snapshot";
47 # Add ccache's directory first so we'll use it if it is installed.
48 $ENV{PATH} = "/usr/lib/ccache:/home/olly/install/bin:/usr/bin:/bin";
49 my $repourl = '/home/xapian-git/xapian/';
50 my $webbase = '/srv/www';
52 # Create the work directory first, since we need it to exist so we can create
53 # the lockfile.
54 mkpath($work, 0, 0755);
55 chdir $work or die $!;
57 # Prevent multiple instances of this script from running at once.
58 # Use the same lockfile that fetch does.
59 open LOCK, '>', 'flockme' or die "Couldn't open 'flockme' for writing: $!\n";
60 unless (flock LOCK, LOCK_EX|LOCK_NB) {
61 # If we're building a tagged release, we want to wait rather than exit.
62 unless (scalar @ARGV && $ARGV[0] =~ m!^tags/! && flock LOCK, LOCK_EX) {
63 # Work directory already in use. Don't print anything unless STDOUT
64 # is a tty - the cron job will send it as mail, which we don't really
65 # want.
66 print STDERR "'flockme' is already locked, can't build '$ARGV[0]' right now\n"
67 if -t STDERR;
68 exit 1;
72 # Check the load average AFTER getting the lock, since we generate output if
73 # the load is too high, and it will probably be too high if we're already
74 # running.
75 my $HOSTNAME = Sys::Hostname::hostname();
76 # Check the load average isn't too high.
77 if (!$force) {
78 if (-e "/var/run/dobackup") {
79 print STDERR "$HOSTNAME: Backup running (/var/run/dobackup exists)\n"
80 if -t STDERR;
81 exit(1);
83 if (((`uptime 2>/dev/null`)[0] =~ /.*: (\d+(?:\.\d+)?),/) &&
84 ($1 > $MAXLOADPERCPU)) {
85 my $loadavg = $1;
86 # `getconf _NPROCESSORS_ONLN` on linux gives e.g. 2
87 # `sysctl hw.ncpu` on openbsd (and prob. freebsd & darwin) gives e.g. hw.ncpu=2
88 # `psrinfo|grep -c on-line` on Solaris or OSF/1 gives e.g. 2
89 my $ncpu;
90 # Works on Linux, at least back to kernel 2.2.26.
91 $ncpu ||= run_command("getconf _NPROCESSORS_ONLN 2>/dev/null|grep -v '[^0-9]'");
92 # Works on OpenBSD (and apparently FreeBSD and Darwin).
93 $ncpu ||= run_command("sysctl hw.ncpu 2>/dev/null|sed 's/.*=//'");
94 # Works on Solaris and OSF/1.
95 $ncpu ||= run_command("PATH=/usr/sbin:\$PATH psrinfo 2>/dev/null|grep -c on-line");
96 # Works on Linux, just in case the getconf version doesn't. Different
97 # architectures have different formats for /proc/cpuinfo so this won't
98 # work as widely as getconf _NPROCESSORS_ONLN will.
99 $ncpu ||= run_command("grep -c processor /proc/cpuinfo 2>/dev/null");
100 $ncpu ||= 1;
101 if ($loadavg > $ncpu * $MAXLOADPERCPU) {
102 $ncpu ||= "unknown";
103 print STDERR "$HOSTNAME: High load average: $loadavg ($ncpu CPUs)\n";
104 exit(1);
109 # If no tags specified, default to those which there are directories for.
110 # Don't bother to build tags/* since those (generally) don't change.
111 if (scalar @ARGV == 0) {
112 @ARGV = grep /\/[A-Za-z0-9][-A-Za-z0-9_.]*$/ && -d $_,
113 glob('branches/[A-Za-z0-9]*');
114 -d "trunk" && unshift @ARGV, "trunk";
115 -d "master" && unshift @ARGV, "master";
118 # Or if there are no directories, default to the trunk.
119 if (scalar @ARGV == 0) {
120 @ARGV = 'trunk';
123 my $status = 0;
124 foreach my $tag (@ARGV) {
125 chdir $work or die $!;
126 # Restrict tags to sane characters.
127 next if $tag !~ /^[-A-Za-z0-9_.\/]+$/;
128 if (! -d $tag) {
129 print "*** No directory for '$tag'\n";
130 $status = 1;
131 next;
134 my $old_cwd = `pwd`;
135 chomp($old_cwd);
137 my $logfile = "$tag/snapshot.log";
138 my $log = '';
139 my $co_dir = "$tag/xapian";
140 if (! -d "$co_dir/.git") {
141 system "chmod", "-R", "+w", $co_dir if -d $co_dir;
142 system "rm", "-rf", $co_dir;
143 # Check out the tag into a directory of the same name.
144 my $branch = $tag;
145 $branch =~ s!^branches/!svn/! || $branch =~ s!^tags/!! ||
146 ($branch =~ s/^trunk$/master/);
147 open CMD, "git clone --branch \Q$branch\E \Q$repourl\E \Q$co_dir\E 2>&1|" or die $!;
148 $log = join '', <CMD>;
149 close CMD or do { print $log; die $!; };
150 chdir $co_dir or die $!;
151 } else {
152 # Revert any local changes.
153 chdir $co_dir or die $!;
154 $log = "git reset --hard:\n".`git reset --hard 2>&1`."git pull --ff-only:\n";
155 open CMD, "git pull --ff-only 2>&1|" or die $!;
156 my $changed = 1;
157 while (<CMD>) {
158 $log .= $_;
159 if ($changed && !$force) {
160 if (/^Already up-to-date/) {
161 $changed = 0;
165 close CMD or die $!;
166 if (!$changed) {
167 chdir $old_cwd or die $!;
168 $ENV{'VERBOSE'} and print "No changes\n";
169 next;
172 my ($revision) = `git describe --always`;
173 chomp($revision);
174 my ($revcount) = ($revision =~ /-([0-9]+)-/);
175 if (!defined $revcount) {
176 # Workaround until we have git tags - count commits since 1.3.1.
177 $revcount = scalar(@{[`git log --format='%H' 1daf0071342d883ca308762f269a63f6ec5df981..`]});
178 $revision = "v1.3.1-$revcount-g$revision";
181 chdir $old_cwd or die $!;
183 # Don't repeat a build for the same revision.
184 next if -f "$logfile.$revision";
186 open LOG, ">", "$logfile.$revision" or die $!;
187 # Flush output after every print.
188 my $old_fh = select(LOG);
189 $| = 1;
190 select($old_fh);
192 print LOG $log;
193 $log = undef;
195 if ($tag !~ m!^tags/!) {
196 # Modify configure.ac files to insert $revision into version string.
197 foreach my $configure_ac
198 (glob("\Q$co_dir\E/xapian*/configure.ac"),
199 glob("\Q$co_dir\E/xapian*/*/configure.ac")) {
200 open OUT, ">", "tmp.out" or die $!;
201 open IN, "<", $configure_ac or die $!;
202 while (<IN>) {
203 s/(^(?:AC_INIT\([^,]*|m4_define\(\[project_version\]),.*?[0-9])(\s*[),\]])/$1_git$revcount$2/g;
204 print OUT;
206 close IN or die $!;
207 close OUT or die $!;
208 rename "tmp.out", $configure_ac or die $!;
210 if (-f "$co_dir/search-xapian/Makefile.PL") {
211 my $snap_version;
212 my $fnm = "$co_dir/search-xapian/Xapian.pm";
213 open OUT, ">", "tmp.out" or die $!;
214 open IN, "<", $fnm or die $!;
215 while (<IN>) {
216 if (s/^(our \$VERSION = ')(\d+\.\d+\.\d+\.)\d+(.*)/$1$2$revcount$3 # git snapshot/) {
217 $snap_version = $2 . $revcount;
219 print OUT;
221 close IN or die $!;
222 close OUT or die $!;
223 rename "tmp.out", $fnm or die $!;
225 $fnm = "$co_dir/search-xapian/README";
226 open OUT, ">", "tmp.out" or die $!;
227 open IN, "<", $fnm or die $!;
228 $_ = <IN>;
229 s/(\d+\.\d+\.\d+\.)\d+/$1$revcount (git snapshot)/;
230 print OUT;
231 while (<IN>) {
232 print OUT;
234 close IN or die $!;
235 close OUT or die $!;
236 rename "tmp.out", $fnm or die $!;
238 $fnm = "$co_dir/search-xapian/Changes";
239 open OUT, ">", "tmp.out" or die $!;
240 open IN, "<", $fnm or die $!;
241 while (<IN>) {
242 print OUT;
243 last if /^\s*$/;
245 print OUT $snap_version.strftime(" %a %b %e %H:%M:%S %Z %Y\n",gmtime());
246 print OUT "\t- git snapshot of revision $revision.\n\n";
247 while (<IN>) {
248 print OUT;
250 close IN or die $!;
251 close OUT or die $!;
252 rename "tmp.out", $fnm or die $!;
256 system "chmod", "-R", "+w", "$tag/build" if -d "$tag/build";
257 system "rm", "-rf", "$tag/build";
258 mkpath("$tag/build", 0, 0755) or die $!;
259 chdir "$tag/build" or die $!;
261 # Note the current time so we can find sources which weren't used during
262 # the build. Sleep for a couple of seconds first to avoid needing to
263 # worry about timestamps equal to $timestamp.
264 sleep 2;
266 # Skip xapian-letor for now, as atreus lacks libsvm-dev, or the ability
267 # to install a new enough one from a package.
268 open TOUCH, '>>', '../xapian/xapian-letor/.nobootstrap' and close TOUCH;
270 open F, '<', '../xapian/bootstrap' or die $!;
271 <F>;
272 my $timestamp = (stat F)[8];
273 close F;
275 $log = `../xapian/bootstrap 2>&1`;
276 print LOG $log;
277 if ($?) {
278 print "*** bootstrap failed for '$tag':";
279 print $log;
280 $status = 1;
281 next;
283 $log = undef;
285 # On atreus, javac keeps going into memory eating loops, so don't try to
286 # build the java bindings.
287 $log = `../xapian/configure --enable-quiet --enable-maintainer-mode --without-java 2>&1`;
288 print LOG $log;
289 if ($?) {
290 print "*** configure failed for '$tag':";
291 print $log;
292 $status = 1;
293 next;
295 $log = undef;
297 $log = `make -s 2>&1`;
298 print LOG $log;
299 if ($?) {
300 print "*** make failed for '$tag':";
301 print $log;
302 $status = 1;
303 next;
305 $log = undef;
307 my %unused_files = ();
308 sub check_unused_files_from_build {
309 return if $File::Find::name eq '../xapian';
310 my $f = substr($File::Find::name, length('../xapian/'));
311 if ($_ eq 'autom4te.cache' ||
312 $_ eq 'debian' ||
313 $f eq 'search-xapian/blib' ||
314 $f eq 'swig' ||
315 $f eq 'xapian-applications/omega/common' ||
316 $f eq 'xapian-data' || # FIXME: make check should use this
317 $f eq 'xapian-maintainer-tools' ||
318 $f eq 'BUILD' ||
319 $f eq 'INST' ||
320 $f eq '.git' ||
321 /^Search-Xapian-\d+\.\d+\.\d+\.\d+$/) {
322 if (-d $File::Find::name) {
323 # Don't descend into these subdirectories.
324 $File::Find::prune = 1;
325 return;
328 return unless -f $File::Find::name and (stat _)[8] < $timestamp;
329 return if $_ eq '.gitignore';
330 return if $_ eq 'config.h.in~';
331 return if $_ eq 'NEWS.SKELETON';
332 return if $f eq 'README';
333 return if $f eq 'Vagrantfile';
334 return if $f eq '.travis.yml';
335 return if /^Search-Xapian-\d+\.\d+\.\d+\.\d+\.tar\..z$/;
336 ++$unused_files{$f};
337 print "Unused during make: $f\n";
339 find(\&check_unused_files_from_build, '../xapian');
341 my $lib = (<xapian-core/.libs/libxapian*.so>)[0];
342 my $unstripped_so = -s $lib;
343 $log = `strip \Q$lib\E`;
344 print LOG $log;
345 $log = undef;
346 my $stripped_so = -s $lib;
348 open SIZELOG, ">>/home/olly/xapian-autobuild-stats.log";
349 print SIZELOG "$tag\trev=$revision\tunstripped_so=$unstripped_so\tstripped_so=$stripped_so\n";
350 close SIZELOG;
352 $log = `make -s distcheck VALGRIND= 2>&1`;
353 print LOG $log;
354 if ($?) {
355 print "*** make distcheck failed for '$tag':";
356 print $log;
357 $status = 1;
358 next;
360 $log = undef;
362 # This finds files we don't ship or use to get to what we ship:
363 find(\&check_unused_files_from_build, '../xapian');
365 # MSVC build files aren't up to date, so don't snapshot them for now.
366 if (0) {
367 # Snapshots of MSVC build files.
368 my $win32dir = "win32msvc_$revision";
369 unlink $win32dir;
370 symlink "../xapian/xapian-maintainer-tools/win32msvc", $win32dir;
371 $log = `tar --dereference --exclude .git -acvf \Q$win32dir\E.tar.xz \Q$win32dir\E 2>&1`;
372 unlink $win32dir;
373 print LOG $log;
374 if ($?) {
375 print "*** tar win32msvc failed for '$tag':";
376 print $log;
377 $status = 1;
378 next;
382 my $d = "$webbase/oligarchy.co.uk/xapian/$tag";
383 if ($tag =~ m!^tags/v?([\d.]+)$!) {
384 $d = "$webbase/oligarchy.co.uk/xapian/$1";
386 if (! -d $d) {
387 mkpath($d, 0, 0755) or die $!;
388 open HTACCESS, ">", "$d/.htaccess" or die $!;
389 print HTACCESS "IndexOptions NameWidth=*\n";
390 close HTACCESS or die $!;
391 } else {
392 if (-d "$d/old") {
393 # Delete snapshots more than a week old, but leave at least one.
394 my $num_of_days_to_keep = 7;
395 my @o = glob "$d/old/*.tar.?z";
396 my $n = scalar @o;
397 @o = grep {-M $_ > $num_of_days_to_keep} @o;
398 $n -= scalar @o;
399 unlink @o if $n > 0;
400 } else {
401 mkdir "$d/old", 0755 or die $!;
402 open HTACCESS, ">", "$d/old/.htaccess" or die $!;
403 print HTACCESS "IndexOptions NameWidth=*\n";
404 close HTACCESS or die $!;
406 for (glob "$d/*.tar.?z") {
407 my ($leaf) = m!([^/]*)$!;
408 rename $_, "$d/old/$leaf";
411 for (glob("*/*.tar.?z"), glob("xapian-applications/*/*.tar.?z"), glob("*.tar.?z")) {
412 print LOG "Moving '$_' to '$d'\n";
413 system("mv", $_, $d);
414 if ($?) {
415 print LOG "Failed with exit code $?\n";
416 } else {
417 print LOG "OK\n";
420 for (glob("search-xapian/*.tar.?z")) {
421 print LOG2 "Moving '$_' to '$d'\n";
422 system("mv", $_, $d);
423 if ($?) {
424 print LOG2 "Failed with exit code $?\n";
425 } else {
426 print LOG2 "OK\n";
429 chdir("..");
430 system("rm -rf xapian/search-xapian/Search-Xapian-*");
431 close LOG;
432 # Expire logs more than 10 days old
433 unlink grep {-M $_ > 10} glob 'snapshot.log.*';
436 system("/home/olly/bin/plot-sizes");
437 exit($status);