3 # jobd - perform Girocco maintenance jobs
5 # Run with --help for details
12 use POSIX
":sys_wait_h";
20 BEGIN {noFatalsToBrowser
}
25 my $cpus = online_cpus
;
27 my $max_par = $cpus ?
$cpus * 2 : 8;
28 my $max_par_intensive = 1;
29 my $load_triggers = $cpus ?
sprintf("%g,%g", $cpus * 1.5, $cpus * 0.75) : "6,3";
30 my $lockfile = "/tmp/jobd-$Girocco::Config::tmpsuffix.lock";
31 my $restart_delay = 300;
35 my ($load_trig, $load_untrig);
41 my $p = $job->{'project'};
42 check_project_exists
($job) || return;
43 if (-e get_project_path
($p).".nofetch" || -e get_project_path
($p).".bypass" ||
44 -e get_project_path
($p).".bypass_fetch") {
46 return setup_gc
($job);
48 if (-e get_project_path
($p).".clone_in_progress" && ! -e get_project_path
($p).".clone_failed") {
49 job_skip
($job, "initial mirroring not complete yet");
52 if (-e get_project_path
($p).".clone_failed") {
53 job_skip
($job, "initial mirroring failed");
54 # Still need to gc non top-level clones even if they've failed
55 # otherwise the objects copied into them from the parent will
56 # just accumulate without bound
57 setup_gc
($job) if $p =~ m
,/,;
60 if (my $ts = is_operation_uptodate
($p, 'lastrefresh', rand_adjust
($Girocco::Config
::min_mirror_interval
))) {
61 job_skip
($job, "not needed right now, last run at $ts");
65 if (is_svn_clone
($p)) {
66 # git svn can be very, very slow at times
67 $job->{'timeout_factor'} = 3;
69 exec_job_command
($job, ["$Girocco::Config::basedir/jobd/update.sh", $p], $quiet);
74 my $p = $job->{'project'};
75 check_project_exists
($job) || return;
76 my $projpath = get_project_path
($p);
77 if (-e
"$projpath.nogc" || -e
"$projpath.bypass" ||
78 (-e
"$projpath.delaygc" && ! -e
"$projpath.allowgc" && ! -e
"$projpath.needsgc")) {
83 if (! -e
"$projpath.needsgc" &&
84 ($ts = is_operation_uptodate
($p, 'lastgc', rand_adjust
($Girocco::Config
::min_gc_interval
)))) {
85 job_skip
($job, "not needed right now, last run at $ts");
88 # allow garbage collection to run for longer than an update
89 $job->{'timeout_factor'} = 2;
90 exec_job_command
($job, ["$Girocco::Config::basedir/jobd/gc.sh", $p], $quiet);
96 project
=> $job->{'project'},
98 command
=> \
&gc_project
,
103 sub check_project_exists
{
105 my $p = $job->{'project'};
106 if (!-d get_project_path
($p)) {
107 job_skip
($job, "non-existent project");
113 sub get_project_path
{
114 "$Girocco::Config::reporoot/".shift().".git/";
117 sub is_operation_uptodate
{
118 my ($project, $which, $threshold) = @_;
119 my $path = get_project_path
($project);
120 my $timestamp = get_git
("--git-dir=$path", 'config', "gitweb.$which");
121 defined($timestamp) or $timestamp = '';
123 my $unix_ts = parse_rfc2822_date
($timestamp) || 0;
124 (time - $unix_ts) <= $threshold ?
$timestamp : undef;
129 my $path = get_project_path
($project);
130 my $baseurl = get_git
("--git-dir=$path", 'config', 'gitweb.baseurl');
131 defined($baseurl) or $baseurl = '';
133 my $svnurl = get_git
("--git-dir=$path", 'config', 'svn-remote.svn.url');
134 defined($svnurl) or $svnurl = '';
136 return $baseurl =~ /^svn[:+]/i && $svnurl;
144 command
=> \
&update_project
,
145 on_success
=> \
&setup_gc
,
146 on_error
=> \
&setup_gc
,
151 queue_one
($_) for (Girocco
::Project
->get_full_list());
154 ######### Daemon operation {{{1
164 # Kills and reaps the specified pid. Returns exit status ($?) on success
165 # otherwise undef if process could not be killed or reaped
166 # First sends SIGTERM and if process does not exit within 15 seconds then SIGKILL
167 # Usage: my $exitcode = kill_gently($pid, $kill_process_group = 0);
170 my $use_pg = shift || 0;
171 # Note that the docs for Perl's kill state that a negative signal
172 # number should be used to kill process groups and that while a
173 # a negative process id (and positive signal number) may also do that
174 # on some platforms, that's not portable.
175 my $pg = $use_pg ?
-1 : 1;
176 my $harsh = time() + 15; # SIGKILL after this delay
177 my $count = kill(15*$pg, $targ); # SIGTERM is 15
178 my $reaped = waitpid($targ, WNOHANG
);
179 return undef if $reaped < 0;
180 return $?
if $reaped == $targ;
181 while ($count && time() < $harsh) {
182 select(undef, undef, undef, 0.2);
183 $reaped = waitpid($targ, WNOHANG
);
184 return undef if $reaped < 0;
185 return $?
if $reaped == $targ;
188 $count = kill(9*$pg, $targ); # SIGKILL is 9
189 $reaped = waitpid($targ, WNOHANG
);
190 return undef if $reaped < 0;
191 return $?
if $reaped == $targ;
192 # We should not need to wait to reap a SIGKILL, however, just in case
193 # the system doesn't make a SIGKILL'd process immediately reapable
194 # (perhaps under extremely heavy load) we accomodate a brief delay
195 while ($count && time() < $harsh) {
196 select(undef, undef, undef, 0.2);
197 $reaped = waitpid($targ, WNOHANG
);
198 return undef if $reaped < 0;
199 return $?
if $reaped == $targ;
204 sub handle_softexit
{
205 error
("Waiting for outstanding jobs to finish... ".
206 "^C again to exit immediately");
209 $SIG{'INT'} = \
&handle_exit
;
213 error
("Killing outstanding jobs, please be patient...");
214 $SIG{'TERM'} = 'IGNORE';
216 kill_gently
($_->{'pid'}, 1);
218 unlink $lockfile if ($locked);
224 $opts{'queued_at'} = time;
225 $opts{'dont_run'} = 0;
226 $opts{'intensive'} = 0 unless exists $opts{'intensive'};
234 $job->{'command'}->($job);
235 if ($job->{'dont_run'}) {
244 "[".$job->{'type'}."::".$job->{'project'}."]";
247 # Only one of those per job!
248 sub exec_job_command
{
249 my ($job, $command, $err_only) = @_;
252 $job->{'finished'} = 0;
253 delete $job->{'pid'};
254 if (!defined($pid = fork)) {
255 error
(_job_name
($job) ." Can't fork job: $!");
256 $job->{'finished'} = 1;
261 select(undef, undef, undef, 0.1);
263 open STDIN
, '<', '/dev/null' || do {
264 error
(_job_name
($job) ."Can't read from /dev/null: $!");
268 open STDOUT
, '>', '/dev/null' || do {
269 error
(_job_name
($job) ." Can't write to /dev/null: $!");
273 # New process group so we can keep track of all of its children
274 if (!defined(POSIX
::setpgid
(0, 0))) {
275 error
(_job_name
($job) ." Can't create process group: $!");
280 # Stop perl from complaining
283 $job->{'pid'} = $pid;
284 $job->{'started_at'} = time;
288 my ($job, $msg) = @_;
289 $job->{'dont_run'} = 1;
290 error
(_job_name
($job) ." Skipping job: $msg") unless $quiet || !$msg;
293 sub reap_hanging_jobs
{
295 my $factor = $_->{'timeout_factor'} || 1;
296 if (defined($_->{'started_at'}) && (time - $_->{'started_at'}) > ($kill_after * $factor)) {
297 $_->{'finished'} = 1;
298 my $exitcode = kill_gently
($_->{'pid'}, 1);
301 error
(_job_name
($_) ." KILLED due to timeout" .
302 (($exitcode & 0x7f) == 9 ?
" with SIGKILL": ""));
303 push @jobs_killed, _job_name
($_);
310 if (!$job->{'finished'}) {
311 $job->{'on_success'}->($job) if defined($job->{'on_success'});
312 $job->{'finished'} = 1;
315 $job->{'on_error'}->($job) if defined($job->{'on_error'});
319 sub reap_finished_jobs
{
321 my $finished_any = 0;
322 foreach my $child (grep { !$_->{'pid'} && $_->{'killed'} } @running) {
323 delete $child->{'killed'};
324 reap_one_job
($child);
328 $pid = waitpid(-1, WNOHANG
);
332 my @child = grep { $_->{'pid'} && $_->{'pid'} == $pid } @running;
334 # any non-zero exit status should trigger on_error
335 $child[0]->{'finished'} = 1 if @child;
338 delete $child[0]->{'pid'};
339 reap_one_job
($child[0]);
342 @running = grep { $_->{'finished'} == 0 } @running;
346 sub have_intensive_jobs
{
347 grep { $_->{'intensive'} == 1 } @running;
351 "[". scalar(localtime) ."] ";
355 if ($^O
eq "linux") {
356 # Read /proc/loadavg on Linux
357 open(LOADAV
, '<', '/proc/loadavg') or return undef;
358 my $loadinfo = <LOADAV
>;
360 return (split(/\s/, $loadinfo, 4))[0..2];
362 # Read the output of uptime everywhere else (works on Linux too)
363 open(LOADAV
, '-|', 'uptime') or return undef;
364 my $loadinfo = <LOADAV
>;
366 $loadinfo =~ /load average[^0-9.]*([0-9.]+)[^0-9.]+([0-9.]+)[^0-9.]+([0-9.]+)/iso or return undef;
372 my $last_progress = time;
373 my $last_checkload = time - 5;
374 my $current_load = $load_trig;
381 my $s = @queue == 1 ?
'' : 's';
382 ferror
("--- Processing %d queued job$s", scalar(@queue));
384 $SIG{'INT'} = \
&handle_softexit
;
385 $SIG{'TERM'} = \
&handle_exit
;
386 while (@queue || @running) {
388 my $proceed_immediately = reap_finished_jobs
();
389 # Check current system load
390 if ($load_trig && (time - $last_checkload) >= 5 && defined((my @loadinfo = get_load_info
())[0])) {
391 my $current_load = $loadinfo[0];
392 if ($current_load > $load_trig && !$overloaded) {
394 error
("PAUSE: system load is at $current_load > $load_trig") if $progress;
395 } elsif ($current_load < $load_untrig && $overloaded) {
397 error
("RESUME: system load is at $current_load < $load_untrig") if $progress;
400 $load_info = ', paused (load '. $current_load .')';
402 $load_info = ', load '. $current_load;
404 $last_checkload = time;
407 if ($progress && (time - $last_progress) >= 60) {
408 ferror
("STATUS: %d queued, %d running, %d finished, %d skipped, %d killed$load_info", scalar(@queue), scalar(@running), $jobs_executed, $jobs_skipped, scalar(@jobs_killed));
412 push @run_status, _job_name
($_)." ". (time - $_->{'started_at'}) ."s";
414 error
("STATUS: currently running: ". join(', ', @run_status));
416 $last_progress = time;
418 # Back off if we're too busy
419 if (@running >= $max_par || have_intensive_jobs
() >= $max_par_intensive || !@queue || $overloaded) {
420 sleep 1 unless $proceed_immediately;
424 run_job
(shift(@queue)) if @queue;
427 my $s = $jobs_executed == 1 ?
'' : 's';
428 ferror
("--- Queue processed. %d job$s executed, %d skipped, %d killed.", $jobs_executed, $jobs_skipped, scalar(@jobs_killed));
432 sub run_perpetually
{
434 die "Lockfile '$lockfile' exists. Please make sure no other instance of jobd is running.";
436 open LOCK
, '>', $lockfile || die "Cannot create lockfile '$lockfile': $!";
442 # touch ctime of lockfile to prevent it from being removed by /tmp cleaning
443 chmod 0444, $lockfile;
444 chmod 0644, $lockfile;
447 sleep($restart_delay) if $perpetual; # Let the system breathe for a moment
452 ######### Helpers {{{1
455 print STDERR ts
().shift()."\n";
458 error
(sprintf($_[0], @_[1..$#_]));
467 close(DATA
) if fileno(DATA
);
469 Getopt
::Long
::Configure
('bundling');
470 my $parse_res = GetOptions
(
471 'help|?|h' => sub { pod2usage
(-verbose
=> 2, -exitval
=> 0); },
472 'quiet|q' => \
$quiet,
473 'progress|P' => \
$progress,
474 'kill-after|k=i' => \
$kill_after,
475 'max-parallel|p=i' => \
$max_par,
476 'max-intensive-parallel|i=i' => \
$max_par_intensive,
477 'load-triggers=s' => \
$load_triggers,
478 'restart-delay|d=i' => \
$restart_delay,
479 'lockfile|l=s' => \
$lockfile,
480 'all-once|a' => \
$all_once,
483 fatal
("Error: can only use one out of --all-once and --one")
484 if ($all_once && $one);
487 $ENV{'show_progress'} = '1';
491 $load_triggers = '0,0' unless defined((get_load_info
())[0]);
492 ($load_trig, $load_untrig) = split(/,/, $load_triggers);
508 ########## Documentation {{{1
514 jobd.pl - Perform Girocco maintenance jobs
521 -h | --help detailed instructions
522 -q | --quiet run quietly
523 -P | --progress show occasional status updates
524 -k SECONDS | --kill-after SECONDS how long to wait before killing jobs
525 -p NUM | --max-parallel NUM how many jobs to run at the same time
526 -i NUM | --max-intensive-parallel NUM how many resource-hungry jobs to run
528 --load-triggers TRIG,UNTRIG stop queueing jobs at load above
529 TRIG and resume at load below UNTRIG
530 -d NUM | --restart-delay SECONDS wait for this many seconds between
532 -l FILE | --lockfile FILE create a lockfile in the given
534 -a | --all-once process the list only once
535 -o PRJNAME | --one PRJNAME process only one project
543 Print the full description of jobd.pl's options.
547 Suppress non-error messages, e.g. for use when running this task as a cronjob.
551 Show information about the current status of the job queue occasionally. This
552 is automatically enabled if --quiet is not given.
554 =item B<--kill-after SECONDS>
556 Kill supervised jobs after a certain time to avoid hanging the daemon.
558 =item B<--max-parallel NUM>
560 Run no more than that many jobs at the same time. The default is the number
561 of cpus * 2. If the number of cpus cannot be determined, the default is 8.
563 =item B<--max-intensive-parallel NUM>
565 Run no more than that many resource-hungry jobs at the same time. Right now,
566 this refers to repacking jobs. The default is 1.
568 =item B<--load-triggers TRIG,UNTRIG>
570 If the first system load average (1 minute average) exceeds TRIG, don't queue
571 any more jobs until it goes below UNTRIG. This is currently only supported on
572 Linux and any other platforms that provide an uptime command with load average
575 If both values are zero, load checks are disabled. The default is the number
576 of cpus * 1.5 for TRIG and half that for UNTRIG. If the number of cpus cannot
577 be determined, the default is 6,3.
579 =item B<--restart-delay NUM>
581 After processing the queue, wait this many seconds until the queue is
582 restarted. The default is 300 seconds.
584 =item B<--lockfile FILE>
586 For perpetual operation, specify the full path to a lock file to create and
587 then remove after finishing/aborting. The default is /tmp/jobd-$suffix.lock
588 where $suffix is a 6-character string uniquely determined by the name and
589 nicknme of this Girocco instance. The pid of the running jobd instance will
590 be written to the lock file.
594 Instead of perpetually processing all projects over and over again, process
595 them just once and then exit.
597 =item B<--one PRJNAME>
599 Process only the given project (given as just the project name without C<.git>
600 suffix) and then exit.
606 jobd.pl is Girocco's repositories maintenance servant; it periodically checks
607 all the repositories and updates mirrored repositories and repacks push-mode
608 repositories when needed.