86f82098a67f0c897c36415f2e8dd3f596f9b315
3 # jobd - perform Girocco maintenance jobs
5 # Run with --help for details
12 use POSIX
":sys_wait_h";
23 my $max_par_intensive = 1;
24 my $lockfile = "/tmp/jobd.lock";
25 my $restart_delay = 10; # not currently configurable
33 my $p = $job->{'project'};
34 check_project_exists
($job) || return;
35 if (-e get_project_path
($p).".nofetch") {
37 return setup_gc
($job);
39 if (-e get_project_path
($p).".clone_in_progress") {
40 job_skip
($job, "initial mirroring not complete yet");
43 if (my $ts = is_operation_uptodate
($p, 'lastrefresh', $Girocco::Config
::min_mirror_interval
)) {
44 job_skip
($job, "not needed right now, last run at $ts");
48 exec_job_command
($job, ["$Girocco::Config::basedir/jobd/update.sh", $p], $quiet);
53 my $p = $job->{'project'};
54 check_project_exists
($job) || return;
55 if (my $ts = is_operation_uptodate
($p, 'lastgc', $Girocco::Config
::min_gc_interval
)) {
56 job_skip
($job, "not needed right now, last run at $ts");
59 exec_job_command
($job, ["$Girocco::Config::basedir/jobd/gc.sh", $p], $quiet);
65 project
=> $job->{'project'},
67 command
=> \
&gc_project
,
72 sub check_project_exists
{
74 my $p = $job->{'project'};
75 if (!-d get_project_path
($p)) {
76 job_skip
($job, "non-existent project");
82 sub get_project_path
{
83 "$Girocco::Config::reporoot/".shift().".git/";
86 sub is_operation_uptodate
{
87 my ($project, $which, $threshold) = @_;
88 my $path = get_project_path
($project);
89 my $timestamp = `GIT_DIR="$path" $Girocco::Config::git_bin config "gitweb.$which"`;
90 my $unix_ts = `date +%s -d "$timestamp"`;
91 (time - $unix_ts) <= $threshold ?
$timestamp : undef;
99 command
=> \
&update_project
,
100 on_success
=> \
&setup_gc
,
101 on_error
=> \
&setup_gc
,
106 queue_one
($_) for (Girocco
::Project
->get_full_list());
109 ######### Daemon operation {{{1
119 sub handle_softexit
{
120 error
("Waiting for outstanding jobs to finish... ".
121 "^C again to exit immediately");
124 $SIG{'INT'} = \
&handle_exit
;
128 error
("Killing outstanding jobs...");
129 $SIG{'TERM'} = 'IGNORE';
131 kill 'KILL', -($_->{'pid'});
133 unlink $lockfile if ($locked);
139 $opts{'queued_at'} = time;
140 $opts{'dont_run'} = 0;
141 $opts{'intensive'} = 0 unless exists $opts{'intensive'};
149 $job->{'command'}->($job);
150 if ($job->{'dont_run'}) {
159 "[".$job->{'type'}."::".$job->{'project'}."]";
162 # Only one of those per job!
163 sub exec_job_command
{
164 my ($job, $command, $err_only) = @_;
167 if (!defined($pid = fork)) {
168 error
(_job_name
($job) ." Can't fork job: $!");
169 $job->{'finished'} = 1;
173 open STDIN
, '/dev/null' || do {
174 error
(_job_name
($job) ."Can't read from /dev/null: $!");
175 $job->{'finished'} = 1;
179 open STDOUT
, '>/dev/null' || do {
180 error
(_job_name
($job) ." Can't write to /dev/null: $!");
181 $job->{'finished'} = 1;
185 # New process group so we can keep track of all of its children
186 if (!defined(POSIX
::setpgid
(0, 0))) {
187 error
(_job_name
($job) ." Can't create process group: $!");
188 $job->{'finished'} = 1;
192 select(undef, undef, undef, 0.1);
194 # Stop perl from complaining
197 $job->{'pid'} = $pid;
198 $job->{'finished'} = 0;
199 $job->{'started_at'} = time;
203 my ($job, $msg) = @_;
204 $job->{'dont_run'} = 1;
205 error
(_job_name
($job) ." Skipping job: $msg") unless $quiet || !$msg;
208 sub reap_hanging_jobs
{
210 if (defined($_->{'started_at'}) && (time - $_->{'started_at'}) > $kill_after) {
211 $_->{'finished'} = 1;
212 kill 'KILL', -($_->{'pid'});
213 print STDERR _job_name
($_) ." KILLED due to timeout\n";
214 push @jobs_killed, _job_name
($_);
219 sub reap_finished_jobs
{
221 my $finished_any = 0;
223 $pid = waitpid(-1, WNOHANG
);
227 my @child = grep { $_->{'pid'} && $_->{'pid'} == $pid } @running;
229 # XXX- we currently don't care
231 if (@child && !$child[0]->{'finished'}) {
232 $child[0]->{'on_success'}->($child[0]) if defined($child[0]->{'on_success'});
233 $child[0]->{'finished'} = 1;
236 $child[0]->{'on_error'}->($child[0]) if defined($child[0]->{'on_error'});
239 @running = grep { $_->{'finished'} == 0 } @running;
243 sub have_intensive_jobs
{
244 grep { $_->{'intensive'} == 1 } @running;
248 "[". scalar(localtime) ."] ";
252 my $last_progress = time;
257 printf STDERR ts
() ."--- Processing %d queued jobs\n", scalar(@queue);
259 $SIG{'INT'} = \
&handle_softexit
;
260 $SIG{'TERM'} = \
&handle_exit
;
261 while (@queue || @running) {
263 my $proceed_immediately = reap_finished_jobs
();
265 if ($progress && (time - $last_progress) >= 60) {
266 printf STDERR ts
() ."status: %d queued, %d running, %d finished, %d skipped, %d killed\n", scalar(@queue), scalar(@running), $jobs_executed, $jobs_skipped, scalar(@jobs_killed);
270 push @run_status, _job_name
($_)." ". (time - $_->{'started_at'}) ."s";
272 error
("STATUS: currently running: ". join(', ', @run_status));
274 $last_progress = time;
276 # Back off if we're too busy
277 if (@running >= $max_par || have_intensive_jobs
() >= $max_par_intensive || !@queue) {
278 sleep 1 unless $proceed_immediately;
282 run_job
(shift(@queue)) if @queue;
285 printf STDERR ts
() ."--- Queue processed. %d jobs executed, %d skipped, %d killed.\n", $jobs_executed, $jobs_skipped, scalar(@jobs_killed);
289 sub run_perpetually
{
291 die "Lockfile exists. Please make sure no other instance of jobd is running.";
293 open LOCK
, '>', $lockfile || die "Cannot create lockfile $lockfile: $!";
301 sleep($restart_delay) if $perpetual; # Let the system breathe for a moment
306 ######### Helpers {{{1
309 print STDERR
shift()."\n";
319 Getopt
::Long
::Configure
('bundling');
320 my $parse_res = GetOptions
(
321 'help|?' => sub { pod2usage
(-verbose
=> 1, -exitval
=> 0); },
322 'quiet|q' => \
$quiet,
323 'progress|P' => \
$progress,
324 'kill-after|k=i' => \
$kill_after,
325 'max-parallel|p=i' => \
$max_par,
326 'max-intensive-parallel|i=i' => \
$max_par_intensive,
327 'lockfile|l=s' => \
$lockfile,
328 'all-once|a' => \
$all_once,
331 fatal
("Error: can only use one out of --all-once and --one")
332 if ($all_once && $one);
335 $ENV{'show_progress'} = '1';
353 ########## Documentation {{{1
359 jobd - Perform Girocco maintenance jobs
366 -h | --help detailed instructions
367 -q | --quiet run quietly
368 -P | --progress show occasional status updates
369 -k SECONDS | --kill-after=SECONDS how long to wait before killing jobs
370 -p NUM | --max-parallel=NUM how many jobs to run at the same time
371 -i NUM | --max-intensive-parallel=NUM how many resource-hungry jobs to run
373 -l FILE | --lockfile=FILE create a lockfile in the given
375 -a | --all-once process the list only once
376 -o PRJNAME | --one=PRJNAME process only one project
384 Print the full description of jobd's options.
388 Suppress non-error messages, e.g. for use when running this task as a cronjob.
392 Show information about the current status of the job queue occasionally. This
393 is automatically enabled if --quiet is not given.
395 =item B<--kill-after=SECONDS>
397 Kill supervised jobs after a certain time to avoid hanging the daemon.
399 =item B<--max-parallel=NUM>
401 Run no more than that many jobs at the same time.
403 =item B<--max-intensive-parallel=NUM>
405 Run no more than that many resource-hungry jobs at the same time. Right now,
406 this refers to repacking jobs.
408 =item B<--lockfile=FILE>
410 For perpetual operation, create a lockfile in that place and clean it up after
415 Instead of perpetuously processing all projects over and over again, process
416 them just once and then exit.
418 =item B<--one=PRJNAME>
420 Process only the given project (given as just the project name without C<.git>
421 suffix) and then exit.
427 jobd is Girocco's repositories maintenance servant; it periodically checks all
428 the repositories and updates mirrored repositories and repacks push-mode
429 repositories when needed.