3 # jobd - perform Girocco maintenance jobs
5 # Run with --help for details
12 use POSIX
":sys_wait_h";
20 BEGIN {noFatalsToBrowser
}
25 my $cpus = online_cpus
;
27 my $max_par = $cpus ?
$cpus * 2 : 8;
28 my $max_par_intensive = 1;
29 my $load_triggers = $cpus ?
sprintf("%g,%g", $cpus * 1.5, $cpus * 0.75) : "6,3";
30 my $lockfile = "/tmp/jobd-$Girocco::Config::tmpsuffix.lock";
31 my $restart_delay = 300;
35 my ($load_trig, $load_untrig);
41 my $p = $job->{'project'};
42 check_project_exists
($job) || return;
43 if (-e get_project_path
($p).".nofetch" || -e get_project_path
($p).".bypass" ||
44 -e get_project_path
($p).".bypass_fetch") {
46 return setup_gc
($job);
48 if (-e get_project_path
($p).".clone_in_progress" && ! -e get_project_path
($p).".clone_failed") {
49 job_skip
($job, "initial mirroring not complete yet");
52 if (-e get_project_path
($p).".clone_failed") {
53 job_skip
($job, "initial mirroring failed");
54 # Still need to gc non top-level clones even if they've failed
55 # otherwise the objects copied into them from the parent will
56 # just accumulate without bound
57 setup_gc
($job) if $p =~ m
,/,;
60 if (my $ts = is_operation_uptodate
($p, 'lastrefresh', rand_adjust
($Girocco::Config
::min_mirror_interval
))) {
61 job_skip
($job, "not needed right now, last run at $ts");
65 if (is_svn_clone
($p)) {
66 # git svn can be very, very slow at times
67 $job->{'timeout_factor'} = 3;
69 exec_job_command
($job, ["$Girocco::Config::basedir/jobd/update.sh", $p], $quiet);
74 my $p = $job->{'project'};
75 check_project_exists
($job) || return;
76 my $projpath = get_project_path
($p);
77 if (-e
"$projpath.nogc" || -e
"$projpath.bypass" ||
78 (-e
"$projpath.delaygc" && ! -e
"$projpath.allowgc")) {
82 if (my $ts = is_operation_uptodate
($p, 'lastgc', rand_adjust
($Girocco::Config
::min_gc_interval
))) {
83 job_skip
($job, "not needed right now, last run at $ts");
86 # allow garbage collection to run for longer than an update
87 $job->{'timeout_factor'} = 2;
88 exec_job_command
($job, ["$Girocco::Config::basedir/jobd/gc.sh", $p], $quiet);
94 project
=> $job->{'project'},
96 command
=> \
&gc_project
,
101 sub check_project_exists
{
103 my $p = $job->{'project'};
104 if (!-d get_project_path
($p)) {
105 job_skip
($job, "non-existent project");
111 sub get_project_path
{
112 "$Girocco::Config::reporoot/".shift().".git/";
115 sub is_operation_uptodate
{
116 my ($project, $which, $threshold) = @_;
117 my $path = get_project_path
($project);
118 my $timestamp = get_git
("--git-dir=$path", 'config', "gitweb.$which");
119 defined($timestamp) or $timestamp = '';
121 my $unix_ts = parse_rfc2822_date
($timestamp) || 0;
122 (time - $unix_ts) <= $threshold ?
$timestamp : undef;
127 my $path = get_project_path
($project);
128 my $baseurl = get_git
("--git-dir=$path", 'config', 'gitweb.baseurl');
129 defined($baseurl) or $baseurl = '';
131 my $svnurl = get_git
("--git-dir=$path", 'config', 'svn-remote.svn.url');
132 defined($svnurl) or $svnurl = '';
134 return $baseurl =~ /^svn[:+]/i && $svnurl;
142 command
=> \
&update_project
,
143 on_success
=> \
&setup_gc
,
144 on_error
=> \
&setup_gc
,
149 queue_one
($_) for (Girocco
::Project
->get_full_list());
152 ######### Daemon operation {{{1
162 # Kills and reaps the specified pid. Returns exit status ($?) on success
163 # otherwise undef if process could not be killed or reaped
164 # First sends SIGTERM and if process does not exit within 15 seconds then SIGKILL
165 # Usage: my $exitcode = kill_gently($pid, $kill_process_group = 0);
168 my $use_pg = shift || 0;
169 # Note that the docs for Perl's kill state that a negative signal
170 # number should be used to kill process groups and that while a
171 # a negative process id (and positive signal number) may also do that
172 # on some platforms, that's not portable.
173 my $pg = $use_pg ?
-1 : 1;
174 my $harsh = time() + 15; # SIGKILL after this delay
175 my $count = kill(15*$pg, $targ); # SIGTERM is 15
176 my $reaped = waitpid($targ, WNOHANG
);
177 return undef if $reaped < 0;
178 return $?
if $reaped == $targ;
179 while ($count && time() < $harsh) {
180 select(undef, undef, undef, 0.2);
181 $reaped = waitpid($targ, WNOHANG
);
182 return undef if $reaped < 0;
183 return $?
if $reaped == $targ;
186 $count = kill(9*$pg, $targ); # SIGKILL is 9
187 $reaped = waitpid($targ, WNOHANG
);
188 return undef if $reaped < 0;
189 return $?
if $reaped == $targ;
190 # We should not need to wait to reap a SIGKILL, however, just in case
191 # the system doesn't make a SIGKILL'd process immediately reapable
192 # (perhaps under extremely heavy load) we accomodate a brief delay
193 while ($count && time() < $harsh) {
194 select(undef, undef, undef, 0.2);
195 $reaped = waitpid($targ, WNOHANG
);
196 return undef if $reaped < 0;
197 return $?
if $reaped == $targ;
202 sub handle_softexit
{
203 error
("Waiting for outstanding jobs to finish... ".
204 "^C again to exit immediately");
207 $SIG{'INT'} = \
&handle_exit
;
211 error
("Killing outstanding jobs, please be patient...");
212 $SIG{'TERM'} = 'IGNORE';
214 kill_gently
($_->{'pid'}, 1);
216 unlink $lockfile if ($locked);
222 $opts{'queued_at'} = time;
223 $opts{'dont_run'} = 0;
224 $opts{'intensive'} = 0 unless exists $opts{'intensive'};
232 $job->{'command'}->($job);
233 if ($job->{'dont_run'}) {
242 "[".$job->{'type'}."::".$job->{'project'}."]";
245 # Only one of those per job!
246 sub exec_job_command
{
247 my ($job, $command, $err_only) = @_;
250 if (!defined($pid = fork)) {
251 error
(_job_name
($job) ." Can't fork job: $!");
252 $job->{'finished'} = 1;
256 open STDIN
, '<', '/dev/null' || do {
257 error
(_job_name
($job) ."Can't read from /dev/null: $!");
258 $job->{'finished'} = 1;
262 open STDOUT
, '>', '/dev/null' || do {
263 error
(_job_name
($job) ." Can't write to /dev/null: $!");
264 $job->{'finished'} = 1;
268 # New process group so we can keep track of all of its children
269 if (!defined(POSIX
::setpgid
(0, 0))) {
270 error
(_job_name
($job) ." Can't create process group: $!");
271 $job->{'finished'} = 1;
275 select(undef, undef, undef, 0.1);
277 # Stop perl from complaining
280 $job->{'pid'} = $pid;
281 $job->{'finished'} = 0;
282 $job->{'started_at'} = time;
286 my ($job, $msg) = @_;
287 $job->{'dont_run'} = 1;
288 error
(_job_name
($job) ." Skipping job: $msg") unless $quiet || !$msg;
291 sub reap_hanging_jobs
{
293 my $factor = $_->{'timeout_factor'} || 1;
294 if (defined($_->{'started_at'}) && (time - $_->{'started_at'}) > ($kill_after * $factor)) {
295 $_->{'finished'} = 1;
296 my $exitcode = kill_gently
($_->{'pid'}, 1);
299 error
(_job_name
($_) ." KILLED due to timeout" .
300 (($exitcode & 0x7f) == 9 ?
" with SIGKILL": ""));
301 push @jobs_killed, _job_name
($_);
308 if (!$job->{'finished'}) {
309 $job->{'on_success'}->($job) if defined($job->{'on_success'});
310 $job->{'finished'} = 1;
313 $job->{'on_error'}->($job) if defined($job->{'on_error'});
317 sub reap_finished_jobs
{
319 my $finished_any = 0;
320 foreach my $child (grep { !$_->{'pid'} && $_->{'killed'} } @running) {
321 delete $child->{'killed'};
322 reap_one_job
($child);
326 $pid = waitpid(-1, WNOHANG
);
330 my @child = grep { $_->{'pid'} && $_->{'pid'} == $pid } @running;
332 # XXX- we currently don't care
335 delete $child[0]->{'pid'};
336 reap_one_job
($child[0]);
339 @running = grep { $_->{'finished'} == 0 } @running;
343 sub have_intensive_jobs
{
344 grep { $_->{'intensive'} == 1 } @running;
348 "[". scalar(localtime) ."] ";
352 if ($^O
eq "linux") {
353 # Read /proc/loadavg on Linux
354 open(LOADAV
, '<', '/proc/loadavg') or return undef;
355 my $loadinfo = <LOADAV
>;
357 return (split(/\s/, $loadinfo, 4))[0..2];
359 # Read the output of uptime everywhere else (works on Linux too)
360 open(LOADAV
, '-|', 'uptime') or return undef;
361 my $loadinfo = <LOADAV
>;
363 $loadinfo =~ /load average[^0-9.]*([0-9.]+)[^0-9.]+([0-9.]+)[^0-9.]+([0-9.]+)/iso or return undef;
369 my $last_progress = time;
370 my $last_checkload = time - 5;
371 my $current_load = $load_trig;
378 my $s = @queue == 1 ?
'' : 's';
379 ferror
("--- Processing %d queued job$s", scalar(@queue));
381 $SIG{'INT'} = \
&handle_softexit
;
382 $SIG{'TERM'} = \
&handle_exit
;
383 while (@queue || @running) {
385 my $proceed_immediately = reap_finished_jobs
();
386 # Check current system load
387 if ($load_trig && (time - $last_checkload) >= 5 && defined((my @loadinfo = get_load_info
())[0])) {
388 my $current_load = $loadinfo[0];
389 if ($current_load > $load_trig && !$overloaded) {
391 error
("PAUSE: system load is at $current_load > $load_trig") if $progress;
392 } elsif ($current_load < $load_untrig && $overloaded) {
394 error
("RESUME: system load is at $current_load < $load_untrig") if $progress;
397 $load_info = ', paused (load '. $current_load .')';
399 $load_info = ', load '. $current_load;
401 $last_checkload = time;
404 if ($progress && (time - $last_progress) >= 60) {
405 ferror
("STATUS: %d queued, %d running, %d finished, %d skipped, %d killed$load_info", scalar(@queue), scalar(@running), $jobs_executed, $jobs_skipped, scalar(@jobs_killed));
409 push @run_status, _job_name
($_)." ". (time - $_->{'started_at'}) ."s";
411 error
("STATUS: currently running: ". join(', ', @run_status));
413 $last_progress = time;
415 # Back off if we're too busy
416 if (@running >= $max_par || have_intensive_jobs
() >= $max_par_intensive || !@queue || $overloaded) {
417 sleep 1 unless $proceed_immediately;
421 run_job
(shift(@queue)) if @queue;
424 my $s = $jobs_executed == 1 ?
'' : 's';
425 ferror
("--- Queue processed. %d job$s executed, %d skipped, %d killed.", $jobs_executed, $jobs_skipped, scalar(@jobs_killed));
429 sub run_perpetually
{
431 die "Lockfile '$lockfile' exists. Please make sure no other instance of jobd is running.";
433 open LOCK
, '>', $lockfile || die "Cannot create lockfile '$lockfile': $!";
439 # touch ctime of lockfile to prevent it from being removed by /tmp cleaning
440 chmod 0444, $lockfile;
441 chmod 0644, $lockfile;
444 sleep($restart_delay) if $perpetual; # Let the system breathe for a moment
449 ######### Helpers {{{1
452 print STDERR ts
().shift()."\n";
455 error
(sprintf($_[0], @_[1..$#_]));
464 close(DATA
) if fileno(DATA
);
466 Getopt
::Long
::Configure
('bundling');
467 my $parse_res = GetOptions
(
468 'help|?|h' => sub { pod2usage
(-verbose
=> 2, -exitval
=> 0); },
469 'quiet|q' => \
$quiet,
470 'progress|P' => \
$progress,
471 'kill-after|k=i' => \
$kill_after,
472 'max-parallel|p=i' => \
$max_par,
473 'max-intensive-parallel|i=i' => \
$max_par_intensive,
474 'load-triggers=s' => \
$load_triggers,
475 'restart-delay|d=i' => \
$restart_delay,
476 'lockfile|l=s' => \
$lockfile,
477 'all-once|a' => \
$all_once,
480 fatal
("Error: can only use one out of --all-once and --one")
481 if ($all_once && $one);
484 $ENV{'show_progress'} = '1';
488 $load_triggers = '0,0' unless defined((get_load_info
())[0]);
489 ($load_trig, $load_untrig) = split(/,/, $load_triggers);
505 ########## Documentation {{{1
511 jobd.pl - Perform Girocco maintenance jobs
518 -h | --help detailed instructions
519 -q | --quiet run quietly
520 -P | --progress show occasional status updates
521 -k SECONDS | --kill-after SECONDS how long to wait before killing jobs
522 -p NUM | --max-parallel NUM how many jobs to run at the same time
523 -i NUM | --max-intensive-parallel NUM how many resource-hungry jobs to run
525 --load-triggers TRIG,UNTRIG stop queueing jobs at load above
526 TRIG and resume at load below UNTRIG
527 -d NUM | --restart-delay SECONDS wait for this many seconds between
529 -l FILE | --lockfile FILE create a lockfile in the given
531 -a | --all-once process the list only once
532 -o PRJNAME | --one PRJNAME process only one project
540 Print the full description of jobd.pl's options.
544 Suppress non-error messages, e.g. for use when running this task as a cronjob.
548 Show information about the current status of the job queue occasionally. This
549 is automatically enabled if --quiet is not given.
551 =item B<--kill-after SECONDS>
553 Kill supervised jobs after a certain time to avoid hanging the daemon.
555 =item B<--max-parallel NUM>
557 Run no more than that many jobs at the same time. The default is the number
558 of cpus * 2. If the number of cpus cannot be determined, the default is 8.
560 =item B<--max-intensive-parallel NUM>
562 Run no more than that many resource-hungry jobs at the same time. Right now,
563 this refers to repacking jobs. The default is 1.
565 =item B<--load-triggers TRIG,UNTRIG>
567 If the first system load average (1 minute average) exceeds TRIG, don't queue
568 any more jobs until it goes below UNTRIG. This is currently only supported on
569 Linux and any other platforms that provide an uptime command with load average
572 If both values are zero, load checks are disabled. The default is the number
573 of cpus * 1.5 for TRIG and half that for UNTRIG. If the number of cpus cannot
574 be determined, the default is 6,3.
576 =item B<--restart-delay NUM>
578 After processing the queue, wait this many seconds until the queue is
579 restarted. The default is 300 seconds.
581 =item B<--lockfile FILE>
583 For perpetual operation, specify the full path to a lock file to create and
584 then remove after finishing/aborting. The default is /tmp/jobd-$suffix.lock
585 where $suffix is a 6-character string uniquely determined by the name and
586 nicknme of this Girocco instance. The pid of the running jobd instance will
587 be written to the lock file.
591 Instead of perpetually processing all projects over and over again, process
592 them just once and then exit.
594 =item B<--one PRJNAME>
596 Process only the given project (given as just the project name without C<.git>
597 suffix) and then exit.
603 jobd.pl is Girocco's repositories maintenance servant; it periodically checks
604 all the repositories and updates mirrored repositories and repacks push-mode
605 repositories when needed.