projtool.pl: make bundle dates readable

[girocco.git] / jobd / jobd.pl

#!/usr/bin/perl
#
# jobd - perform Girocco maintenance jobs
#
# Run with --help for details

use strict;
use warnings;

use Getopt::Long;
use Pod::Usage;
use POSIX ":sys_wait_h";
use Cwd qw(realpath);

use lib "__BASEDIR__";
use Girocco::Config;
use Girocco::Project;
use Girocco::User;
use Girocco::Util;
BEGIN {noFatalsToBrowser}
use Girocco::ExecUtil;

sub _defjobs {
        my $cpus = shift;
        return 4 unless defined($cpus) && $cpus ne "" && int($cpus) >= 1;
        return int($cpus * 2) if $cpus <= 2;
        return 5 if $cpus < 4;
        return int($cpus * 1.5) if $cpus <= 10;
        return 16;
}

# Options
my $quiet = 0;
my $progress = 0;
my $cpus = online_cpus;
my $kill_after = 900;
my $max_par = $cpus ? _defjobs($cpus) : 4;
my $max_par_intensive = 1;
my $load_triggers = $cpus ? sprintf("%g,%g", $cpus * 1.5, $cpus * 0.75) : "6,3";
my $lockfile = "/tmp/jobd-$Girocco::Config::tmpsuffix.lock";
my $restart_delay = 300;
my $all_once;
my $same_pid;
my @one = ();
my ($update_only, $gc_only, $needs_gc_only);

my ($load_trig, $load_untrig);

######### Jobs {{{1

sub update_project {
        my $job = shift;
        my $p = $job->{'project'};
        check_project_exists($job) || return;
        my $projpath = get_project_path($p);
        if ($gc_only || $needs_gc_only ||
            -e "$projpath/.nofetch" ||
            -e "$projpath/.bypass" ||
            -e "$projpath/.bypass_fetch" ||
            is_mirror_disabled($p)) {
                job_skip($job);
                setup_gc($job) unless ! -e "$projpath/.nofetch" &&
                        -e "$projpath/.clone_in_progress" && ! -e "$projpath/.clone_failed";
                return;
        }
        if (-e "$projpath/.clone_in_progress" && ! -e "$projpath/.clone_failed") {
                job_skip($job, "initial mirroring not complete yet");
                return;
        }
        if (-e "$projpath/.clone_failed") {
                job_skip($job, "initial mirroring failed");
                # Still need to gc clones even if they've failed
                setup_gc($job);
                return;
        }
        if (my $ts = is_operation_uptodate($p, 'lastrefresh', rand_adjust($Girocco::Config::min_mirror_interval))) {
                job_skip($job, "not needed right now, last run at $ts");
                setup_gc($job);
                return;
        };
        if (is_svn_clone($p)) {
                # git svn can be very, very slow at times
                $job->{'timeout_factor'} = 3;
        }
        exec_job_command($job, ["$Girocco::Config::basedir/jobd/update.sh", $p], $quiet);
}

sub gc_project {
        my $job = shift;
        my $p = $job->{'project'};
        check_project_exists($job) || return;
        my $projpath = get_project_path($p);
        if ($update_only || -e "$projpath/.nogc" || -e "$projpath/.bypass" ||
            (-e "$projpath/.delaygc" && ! -e "$projpath/.allowgc" && ! -e "$projpath/.needsgc")) {
                job_skip($job);
                return;
        }
        my $ts = "";
        if (! -e "$projpath/.needsgc" && ($needs_gc_only ||
            ($ts = is_operation_uptodate($p, 'lastgc', rand_adjust($Girocco::Config::min_gc_interval))))) {
                job_skip($job, ($needs_gc_only ? undef : "not needed right now, last run at $ts"));
                return;
        }
        # allow garbage collection to run for longer than an update
        $job->{'lastgc'} = get_git_config($projpath, "gitweb.lastgc");
        $job->{'timeout_factor'} = 2;
        exec_job_command($job, ["$Girocco::Config::basedir/jobd/gc.sh", $p], $quiet);
}

sub setup_gc {
        my $job = shift;
        queue_job(
                project => $job->{'project'},
                type => 'gc',
                command => \&gc_project,
                intensive => 1,
                on_success => \&maybe_setup_gc_again,
        );
}

sub maybe_setup_gc_again {
        my $job = shift;
        # If lastgc was set then gc.sh ran successfully and now it's not set
        # then queue up another run of gc.sh for the project.
        # However, just in case, no matter what happens with the extra
        # gc.sh run no more "bonus" runs are possible to avoid any loops.
        # This allows a "mini" gc that triggers a full gc to have the
        # full gc run as part of the same --all-once run through instead
        # of waiting.  A very good thing for users of the --all-once option.
        if ($job->{'lastgc'}) {
                my $projpath = get_project_path($job->{'project'});
                get_git_config($projpath, "gitweb.lastgc") or
                    queue_job(
                            project => $job->{'project'},
                            type => 'gc',
                            command => \&gc_project,
                            intensive => 1,
                    );
        }
}

sub check_project_exists {
        my $job = shift;
        my $p = $job->{'project'};
        if (! -d get_project_path($p)) {
                job_skip($job, "non-existent project");
                return 0;
        }
        1;
}

sub get_project_path {
        "$Girocco::Config::reporoot/".shift().".git";
}

my $_last_config_path;
my $_last_config_id;
my $_last_config;
BEGIN {
    $_last_config_path = "";
    $_last_config_id = "";
    $_last_config = {};
}

sub get_git_config {
        my ($projdir, $name) = @_;
        defined($projdir) && -d $projdir && -f "$projdir/config" or return undef;
        my $cf = "$projdir/config";
        my @stat = stat($cf);
        @stat && $stat[7] && $stat[9] or return undef;
        my $id = join(":", $stat[0], $stat[1], $stat[7], $stat[9]); # dev,ino,size,mtime
        if ($_last_config_path ne $cf || $_last_config_id ne $id || ref($_last_config) ne 'HASH') {
                my $data = read_config_file_hash($cf);
                defined($data) or $data = {};
                $_last_config_path = $_last_config_id = "";
                $_last_config = $data;
                $_last_config_id = $id;
                $_last_config_path = $cf;
        }
        return $_last_config->{$name};
}

sub is_operation_uptodate {
        my ($project, $which, $threshold) = @_;
        my $path = get_project_path($project);
        my $timestamp = get_git_config($path, "gitweb.$which");
        defined($timestamp) or $timestamp = '';
        my $unix_ts = parse_rfc2822_date($timestamp) || 0;
        (time - $unix_ts) <= $threshold ? $timestamp : undef;
}

sub is_mirror_disabled {
        my ($project) = @_;
        my $path = get_project_path($project);
        my $baseurl = get_git_config($path, 'gitweb.baseurl');
        defined($baseurl) or $baseurl = '';
        $baseurl =~ s/^\s+//;
        $baseurl =~ s/\s+$//;
        return $baseurl eq "" || $baseurl =~ /\s/ || $baseurl =~ /^disabled(?:\s|$)/i;
}

sub is_svn_clone {
        my ($project) = @_;
        my $path = get_project_path($project);
        my $baseurl = get_git_config($path, 'gitweb.baseurl');
        defined($baseurl) or $baseurl = '';
        my $svnurl = get_git_config($path, 'svn-remote.svn.url');
        defined($svnurl) or $svnurl = '';
        return $baseurl =~ /^svn[:+]/i && $svnurl;
}

sub queue_one {
        my $project = shift;
        queue_job(
                project => $project,
                type => 'update',
                command => \&update_project,
                on_success => \&setup_gc,
                on_error => \&setup_gc,
        );
}

sub queue_all {
        queue_one($_) for (Girocco::Project->get_full_list());
}

######### Daemon operation {{{1

my @queue;
my @running;
my $perpetual = 1;
my $locked = 0;
my $jobs_executed;
my $jobs_skipped;
my @jobs_killed;

# Kills and reaps the specified pid.  Returns exit status ($?) on success
# otherwise undef if process could not be killed or reaped
# First sends SIGINT and if process does not exit within 15 seconds then SIGKILL
# We used to send SIGTERM instead of SIGINT, but by using SIGINT we can take
# advantage of "tee -i" in our update scripts and really anything we're killing
# should respond the same to either SIGINT or SIGTERM and exit gracefully.
# Usage: my $exitcode = kill_gently($pid, $kill_process_group = 0);
sub kill_gently {
        my $targ = shift;
        my $use_pg = shift || 0;
        # Note that the docs for Perl's kill state that a negative signal
        # number should be used to kill process groups and that while a
        # a negative process id (and positive signal number) may also do that
        # on some platforms, that's not portable.
        my $pg = $use_pg ? -1 : 1;
        my $harsh = time() + 15; # SIGKILL after this delay
        my $count = kill(2*$pg, $targ); # SIGINT is 2
        my $reaped = waitpid($targ, WNOHANG);
        return undef if $reaped < 0;
        return $? if $reaped == $targ;
        while ($count && time() < $harsh) {
                select(undef, undef, undef, 0.2);
                $reaped = waitpid($targ, WNOHANG);
                return undef if $reaped < 0;
                return $? if $reaped == $targ;
        }
        $harsh = time() + 2;
        $count = kill(9*$pg, $targ); # SIGKILL is 9
        $reaped = waitpid($targ, WNOHANG);
        return undef if $reaped < 0;
        return $? if $reaped == $targ;
        # We should not need to wait to reap a SIGKILL, however, just in case
        # the system doesn't make a SIGKILL'd process immediately reapable
        # (perhaps under extremely heavy load) we accomodate a brief delay
        while ($count && time() < $harsh) {
                select(undef, undef, undef, 0.2);
                $reaped = waitpid($targ, WNOHANG);
                return undef if $reaped < 0;
                return $? if $reaped == $targ;
        }
        return undef;
}

sub handle_softexit {
        error("Waiting for outstanding jobs to finish... ".
                "^C again to exit immediately");
        @queue = ();
        $perpetual = 0;
        $SIG{'INT'} = \&handle_exit;
}

sub handle_exit {
        error("Killing outstanding jobs, please be patient...");
        $SIG{'TERM'} = 'IGNORE';
        for (@running) {
                kill_gently($_->{'pid'}, 1);
        }
        unlink $lockfile if ($locked);
        exit(0);
}

sub queue_job {
        my %opts = @_;
        $opts{'queued_at'} = time;
        $opts{'dont_run'} = 0;
        $opts{'intensive'} = 0 unless exists $opts{'intensive'};
        push @queue, \%opts;
}

sub run_job {
        my $job = shift;

        push @running, $job;
        $job->{'command'}->($job);
        if ($job->{'dont_run'}) {
                pop @running;
                $jobs_skipped++;
                return;
        }
}

sub _job_name {
        my $job = shift;
        "[".$job->{'type'}."::".$job->{'project'}."]";
}

# Only one of those per job!
sub exec_job_command {
        my ($job, $command, $err_only) = @_;

        my $pid;
        $job->{'finished'} = 0;
        delete $job->{'pid'};
        if (!defined($pid = fork)) {
                error(_job_name($job) ." Can't fork job: $!");
                $job->{'finished'} = 1;
                return;
        }
        if (!$pid) {
                # "Prevent" races
                select(undef, undef, undef, 0.1);

                open STDIN, '<', '/dev/null' || do {
                        error(_job_name($job) ."Can't read from /dev/null: $!");
                        exit 71; # EX_OSERR
                };
                if ($err_only) {
                        open STDOUT, '>', '/dev/null' || do {
                                error(_job_name($job) ." Can't write to /dev/null: $!");
                                exit 71; # EX_OSERR
                        };
                }
                # New process group so we can keep track of all of its children
                if (!defined(POSIX::setpgid(0, 0))) {
                        error(_job_name($job) ." Can't create process group: $!");
                        exit 71; # EX_OSERR
                }

                exec @$command;
                # Stop perl from complaining
                exit 71; # EX_OSERR
        }
        $job->{'pid'} = $pid;
        $job->{'started_at'} = time;
}

sub job_skip {
        my ($job, $msg) = @_;
        $job->{'dont_run'} = 1;
        error(_job_name($job) ." Skipping job: $msg") unless $quiet || !$msg;
}

sub reap_hanging_jobs {
        for (@running) {
                my $factor = $_->{'timeout_factor'} || 1;
                if (defined($_->{'started_at'}) && (time - $_->{'started_at'}) > ($kill_after * $factor)) {
                        $_->{'finished'} = 1;
                        my $exitcode = kill_gently($_->{'pid'}, 1);
                        delete $_->{'pid'};
                        $_->{'killed'} = 1;
                        error(_job_name($_) ." KILLED due to timeout" .
                                (($exitcode & 0x7f) == 9 ? " with SIGKILL": ""));
                        push @jobs_killed, _job_name($_);
                }
        }
}

sub reap_one_job {
        my $job = shift;
        if (!$job->{'finished'}) {
                $job->{'on_success'}->($job) if defined($job->{'on_success'});
                $job->{'finished'} = 1;
                $jobs_executed++;
        } else {
                $job->{'on_error'}->($job) if defined($job->{'on_error'});
        }
}

sub reap_finished_jobs {
        my $pid;
        my $finished_any = 0;
        foreach my $child (grep { !$_->{'pid'} && $_->{'killed'} } @running) {
                delete $child->{'killed'};
                reap_one_job($child);
                $finished_any = 1;
        }
        while (1) {
                $pid = waitpid(-1, WNOHANG);
                last if $pid <= 0;
                $finished_any = 1;

                my @child = grep { $_->{'pid'} && $_->{'pid'} == $pid } @running;
                if ($?) {
                        # any non-zero exit status should trigger on_error
                        $child[0]->{'finished'} = 1 if @child;
                }
                if (@child) {
                        delete $child[0]->{'pid'};
                        reap_one_job($child[0]);
                }
        }
        @running = grep { $_->{'finished'} == 0 } @running;
        $finished_any;
}

sub have_intensive_jobs {
        grep { $_->{'intensive'} == 1 } @running;
}

sub ts {
        "[". scalar(localtime) ."] ";
}

sub get_load_info {
        if ($^O eq "linux") {
                # Read /proc/loadavg on Linux
                open(LOADAV, '<', '/proc/loadavg') or return undef;
                my $loadinfo = <LOADAV>;
                close LOADAV;
                return (split(/\s/, $loadinfo, 4))[0..2];
        } else {
                # Read the output of uptime everywhere else (works on Linux too)
                open(LOADAV, '-|', 'uptime') or return undef;
                my $loadinfo = <LOADAV>;
                close LOADAV;
                $loadinfo =~ /load average[^0-9.]*([0-9.]+)[^0-9.]+([0-9.]+)[^0-9.]+([0-9.]+)/iso or return undef;
                return ($1, $2, $3);
        }
}

sub run_queue {
        my $start = time;
        my $last_progress = $start;
        my $last_checkload = $start - 5;
        my $current_load = $load_trig;
        my $overloaded = 0;
        my $load_info = '';
        $jobs_executed = 0;
        $jobs_skipped = 0;
        @jobs_killed = ();
        if ($progress) {
                my $s = @queue == 1 ? '' : 's';
                ferror("--- Processing %d queued job$s", scalar(@queue));
        }
        $SIG{'INT'} = \&handle_softexit;
        $SIG{'TERM'} = \&handle_exit;
        while (@queue || @running) {
                reap_hanging_jobs();
                my $proceed_immediately = reap_finished_jobs();
                # Check current system load
                if ($load_trig && (time - $last_checkload) >= 5 && defined((my @loadinfo = get_load_info())[0])) {
                        my $current_load = $loadinfo[0];
                        if ($current_load > $load_trig && !$overloaded) {
                                $overloaded = 1;
                                error("PAUSE: system load is at $current_load > $load_trig") if $progress;
                        } elsif ($current_load < $load_untrig && $overloaded) {
                                $overloaded = 0;
                                error("RESUME: system load is at $current_load < $load_untrig") if $progress;
                        }
                        if ($overloaded) {
                                $load_info = ', paused (load '. $current_load .')';
                        } else {
                                $load_info = ', load '. $current_load;
                        }
                        $last_checkload = time;
                }
                # Status output
                if ($progress && (time - $last_progress) >= 60) {
                        ferror("STATUS: %d queued, %d running, %d finished, %d skipped, %d killed$load_info", scalar(@queue), scalar(@running), $jobs_executed, $jobs_skipped, scalar(@jobs_killed));
                        if (@running) {
                                my @run_status;
                                for (@running) {
                                        push @run_status, _job_name($_)." ". (time - $_->{'started_at'}) ."s";
                                }
                                error("STATUS: currently running: ". join(', ', @run_status));
                        }
                        $last_progress = time;
                }
                # Back off if we're too busy
                if (@running >= $max_par || have_intensive_jobs() >= $max_par_intensive || !@queue || $overloaded) {
                        sleep 1 unless $proceed_immediately;
                        next;
                }
                # Run next
                run_job(shift(@queue)) if @queue;
        }
        if ($progress) {
                my $s = $jobs_executed == 1 ? '' : 's';
                ferror("--- Queue processed in %s. %d job$s executed, %d skipped, %d killed.",
                        human_duration(time - $start), $jobs_executed, $jobs_skipped, scalar(@jobs_killed));
        }
}

sub run_perpetually {
        if (-e $lockfile) {
                die "Lockfile '$lockfile' exists. Please make sure no other instance of jobd is running.\n";
        }
        open LOCK, '>', $lockfile || die "Cannot create lockfile '$lockfile': $!\n";
        print LOCK $$;
        close LOCK;
        $locked = 1;

        my $result = "";
        while ($perpetual) {
                # touch ctime of lockfile to prevent it from being removed by /tmp cleaning
                chmod 0640, $lockfile;
                chmod 0644, $lockfile;
                # check for restart request
                open LOCK, '<', $lockfile || die "Lock file '$lockfile' has disappeared!\n";
                my $request = <LOCK>;
                close LOCK;
                chomp $request if defined($request);
                if (defined($request) && $request eq "restart") {
                        $result = $request;
                        last;
                }
                queue_all();
                run_queue();
                sleep($restart_delay) if $perpetual; # Let the system breathe for a moment
        }
        unlink $lockfile;
        $locked = 0;
        return $result;
}

######### Helpers {{{1

sub error($) {
        print STDERR ts().shift()."\n";
}
sub ferror(@) {
        error(sprintf($_[0], @_[1..$#_]));
}
sub fatal($) {
        error(shift);
        exit 1;
}

######### Main {{{1

my $reexec = Girocco::ExecUtil->new;
my $realpath0 = realpath($0);
chdir "/";
close(DATA) if fileno(DATA);
# Parse options
Getopt::Long::Configure('bundling');
my %one_once;
my $parse_res = GetOptions(
        'help|?|h' => sub {
                pod2usage(-verbose => 2, -exitval => 0, -input => $realpath0)},
        'quiet|q' => sub {++$quiet},
        'progress|P' => sub {++$progress},
        'kill-after|k=i' => \$kill_after,
        'max-parallel|p=i' => \$max_par,
        'max-intensive-parallel|i=i' => \$max_par_intensive,
        'load-triggers=s' => \$load_triggers,
        'restart-delay|d=i' => \$restart_delay,
        'lockfile|l=s' => \$lockfile,
        'same-pid' => \$same_pid,
        'all-once|a' => \$all_once,
        'one|o=s' => sub {$one_once{$_[1]} = 1, push(@one, $_[1])
                                        unless exists $one_once{$_[1]}},
        'update-only' => \$update_only,
        'gc-only' => \$gc_only,
        'needs-gc-only' => \$needs_gc_only,
) || pod2usage(-exitval => 2, -input => $realpath0);
fatal("Error: can only use one out of --all-once and --one")
        if $all_once && @one;
my $onlycnt = ($update_only?1:0) + ($gc_only?1:0) + ($needs_gc_only?1:0);
fatal("Error: can only use one out of --update-only, --gc-only and --needs-gc-only")
        if $onlycnt > 1;
fatal("Error: --update-only, --gc-only or --needs-gc-only requires --all-once or --one")
        if $onlycnt && !($all_once || @one);

delete $ENV{'show_progress'};
if ($quiet) {
        $ENV{'show_progress'} = 0 if $quiet > 1;
} else {
        $progress = 1 unless $progress;
        $ENV{'show_progress'} = $progress;
}

$load_triggers = '0,0' unless defined((get_load_info())[0]);
($load_trig, $load_untrig) = split(/,/, $load_triggers);

if (@one) {
        queue_one($_) foreach @one;
        run_queue();
        exit;
}

if ($all_once) {
        queue_all();
        run_queue();
        exit;
}

{
        if (run_perpetually() eq "restart") {
                error("Restarting in response to restart request... ");
                $reexec->reexec($same_pid);
                error("Continuing after failed restart: $!");
                chdir "/";
                redo;
        }
}

########## Documentation {{{1

__END__

=head1 NAME

jobd.pl - Perform Girocco maintenance jobs

=head1 SYNOPSIS

jobd.pl [options]

 Options:
   -h | --help                           detailed instructions
   -q | --quiet                          run quietly
   -P | --progress                       show occasional status updates
   -k SECONDS | --kill-after SECONDS     how long to wait before killing jobs
   -p NUM | --max-parallel NUM           how many jobs to run at the same time
   -i NUM | --max-intensive-parallel NUM how many resource-hungry jobs to run
                                         at the same time
   --load-triggers TRIG,UNTRIG           stop queueing jobs at load above
                                         TRIG and resume at load below UNTRIG
   -d NUM | --restart-delay SECONDS      wait for this many seconds between
                                         queue runs
   -l FILE | --lockfile FILE             create a lockfile in the given
                                         location
   --same-pid                            keep same pid during graceful restart
   -a | --all-once                       process the list only once
   -o PRJNAME | --one PRJNAME            process only one project
   --update-only                         process mirror updates only
   --gc-only                             perform needed garbage collection only
   --needs-gc-only                       perform needed mini gc only

=head1 OPTIONS

=over 8

=item B<--help>

Print the full description of jobd.pl's options.

=item B<--quiet>

Suppress non-error messages, e.g. for use when running this task as a cronjob.
When given two or more times suppress update ref change lines in logs as well.

=item B<--progress>

Show information about the current status of the job queue occasionally. This
is automatically enabled if --quiet is not given.  When specified two or more
times full ref change details will be shown for updates.

=item B<--kill-after SECONDS>

Kill supervised jobs after a certain time to avoid hanging the daemon.

=item B<--max-parallel NUM>

Run no more than that many jobs at the same time.  The default is the number
of cpus * 2 for 1 or 2 cpus, 5 for 3 cpus and int(cpus * 1.5) for 4 cpus or
more with the default capped to 16 when more than 10 cpus are detected.
If the number of cpus cannot be determined, the default is 4.

=item B<--max-intensive-parallel NUM>

Run no more than that many resource-hungry jobs at the same time. Right now,
this refers to repacking jobs.  The default is 1.

=item B<--load-triggers TRIG,UNTRIG>

If the first system load average (1 minute average) exceeds TRIG, don't queue
any more jobs until it goes below UNTRIG. This is currently only supported on
Linux and any other platforms that provide an uptime command with load average
output.

If both values are zero, load checks are disabled.  The default is the number
of cpus * 1.5 for TRIG and half that for UNTRIG.  If the number of cpus cannot
be determined, the default is 6,3.

=item B<--restart-delay NUM>

After processing the queue, wait this many seconds until the queue is
restarted.  The default is 300 seconds.

=item B<--lockfile FILE>

For perpetual operation, specify the full path to a lock file to create and
then remove after finishing/aborting.  The default is /tmp/jobd-$suffix.lock
where $suffix is a 6-character string uniquely determined by the name and
nickname of this Girocco instance.  The pid of the running jobd instance will
be written to the lock file.

=item B<--same-pid>

When performing a graceful restart, keep the same pid rather than switching to
a new one.

=item B<--all-once>

Instead of perpetually processing all projects over and over again, process
them just once and then exit.
Conflicts with B<--one PRJNAME> option.

=item B<--one PRJNAME>

Process only the given project (given as just the project name without C<.git>
suffix) and then exit.  May be repeated to process more than one project.
Conflicts with B<--all-once> option.

=item B<--update-only>

Limit processing to only those projects that need a mirror update.
Behaves as though every project has a C<.nogc> file present in it.
Requires use of B<--all-once> or B<--one PRJNAME> option.
Conflicts with B<--gc-only> and B<--needs-gc-only> options.

=item B<--gc-only>

Limit processing to only those projects that need to have garbage collection
run on them.  Behaves as though every project has a C<.bypass_fetch> file
present in it.  Requires use of B<--all-once> or B<--one PRJNAME> option.
Conflicts with B<--update-only> and B<--needs-gc-only> options.

=item B<--needs-gc-only>

Limit processing to only those projects that need to have mini garbage
collection run on them.  Behaves as though every project with a C<.needsgc>
file present in it also has a C<.bypass_fetch> file present in it and as though
every project without a C<.needsgc> file present in it has a C<.bypass> file
present in it.  Requires use of B<--all-once> or B<--one PRJNAME> option.
Conflicts with B<--update-only> and B<--gc-only> options.

=back

=head1 DESCRIPTION

jobd.pl is Girocco's repositories maintenance servant; it periodically checks
all the repositories and updates mirrored repositories and repacks push-mode
repositories when needed.

=cut