From 73927bbbc86e851be6fefd17fb5a310764362a68 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Sun, 28 Apr 2013 00:22:59 -0700 Subject: [PATCH] Add more sample configuration and utility files --- Girocco/Config.pm | 6 +++++- INSTALL | 3 +++ cron/README | 13 ++++++++++++ cron/girocco | 49 +++++++++++++++++++++++++++++++++++++++++++ fixupd/README | 5 +++++ fstab/README | 11 ++++++++++ fstab/fstab.girocco | 17 +++++++++++++++ install.sh | 10 +++++++-- jobs/README | 3 +++ jobs/updateweb.sh | 24 +++++++++++++++++++++ rsyslog/README | 11 ++++++++++ rsyslog/girocco.conf | 4 ++++ screen/README | 35 +++++++++++++++++++++++++++++++ screen/girocco | 26 +++++++++++++++++++++++ screen/giroccoacl | 14 +++++++++++++ screen/root | 19 +++++++++++++++++ screen/screenrc | 46 ++++++++++++++++++++++++++++++++++++++++ toolbox/kill-stale-daemons.pl | 23 ++++++++++++++++++++ toolbox/run-fixupd.sh | 15 +++++++++++++ toolbox/run-jobd.sh | 12 +++++++++++ toolbox/run-taskd.sh | 10 +++++++++ toolbox/updateroot.sh | 48 ++++++++++++++++++++++++++++++++++++++++++ xinetd/README | 14 +++++++++++++ xinetd/git | 15 +++++++++++++ 24 files changed, 430 insertions(+), 3 deletions(-) create mode 100644 cron/README create mode 100644 cron/girocco create mode 100644 fstab/README create mode 100644 fstab/fstab.girocco create mode 100755 jobs/updateweb.sh create mode 100644 rsyslog/README create mode 100644 rsyslog/girocco.conf create mode 100644 screen/README create mode 100644 screen/girocco create mode 100644 screen/giroccoacl create mode 100644 screen/root create mode 100644 screen/screenrc create mode 100755 toolbox/kill-stale-daemons.pl create mode 100644 toolbox/run-fixupd.sh create mode 100755 toolbox/run-jobd.sh create mode 100755 toolbox/run-taskd.sh create mode 100755 toolbox/updateroot.sh create mode 100644 xinetd/README create mode 100644 xinetd/git diff --git a/Girocco/Config.pm b/Girocco/Config.pm index a74975d..affb9aa 100644 --- a/Girocco/Config.pm +++ b/Girocco/Config.pm @@ -232,11 +232,15 @@ our $permission_control = 'Group'; # Will be read by fixupd.sh our $fixup_queue = '/var/tmp/fixup.fifo'; +# Path to alternate screen multiuser acl file (see screen/README, undef for none) +our $screen_acl_file = undef; + # Couple of sanity checks and default settings (do not change these) $nickname = (split(/[.]/, $Girocco::Config::name))[0] unless $nickname; ($mirror_user) or die "Girocco::Config: \$mirror_user must be set even if to current user"; +$screen_acl_file = "$basedir/screen/giroccoacl" unless $screen_acl_file; (not $mob or $mob eq 'mob') or die "Girocco::Config \$mob must be undef (or '') or 'mob'"; -($mirror or $push) or die "Girocco::Config: neither \$mirror or \$push is set?!"; +($mirror or $push) or die "Girocco::Config: neither \$mirror nor \$push is set?!"; (not $push or ($pushurl or $gitpullurl or $httppullurl)) or die "Girocco::Config: no pull URL is set"; (not $push or $pushurl) or die "Girocco::Config: \$push set but \$pushurl is undef"; (not $mirror or $mirror_user) or die "Girocco::Config: \$mirror set but \$mirror_user is undef"; diff --git a/INSTALL b/INSTALL index 4ded7be..f987101 100644 --- a/INSTALL +++ b/INSTALL @@ -29,6 +29,9 @@ can either just keep jobd.sh running (good if you have large amount of repositories), or add a special invocation (advised by make install) to your crontab. +The cron, fstab, rsyslog, screen and xinetd directories contain README files +and sample configuration files to help configure the system for Girocco. + Girocco is reasonably self-sufficient, but does depend on a few external programs and libraries: diff --git a/cron/README b/cron/README new file mode 100644 index 0000000..68ab70b --- /dev/null +++ b/cron/README @@ -0,0 +1,13 @@ +The girocco file in this directory is a SAMPLE cron file that can +be placed in /etc/cron.d/ to run everything needed for a full Girocco +installation (except the web server itself). + +Please be sure to examine all the paths in the girocco file and +adjust them as necessary for your installation before using! + +Note that the girocco file will take effect as soon as it's copied +into /etc/cron.d/, no reboot is necessary. + +IMPORTANT: Please also see the ../screen/README file to make sure +that screen is properly configured to support this girocco cron +file. diff --git a/cron/girocco b/cron/girocco new file mode 100644 index 0000000..f56965b --- /dev/null +++ b/cron/girocco @@ -0,0 +1,49 @@ +# This file collects all the startup jobs needed +# for girocco into one place. It can be placed +# into the /etc/cron.d/ directory and then the +# system restarted. + +# Alternatively the individual items could be +# placed into the repo and root per-user crontabs +# or made into a startup script that uses su, but +# it's much simpler just to put them in the global +# crontab so they're all in one place + +# IMPORTANT: Be sure to update the users/paths +# below before using to match Config.pm! + +# +## girocco tasks that run as root +# + +# The ssh daemon for git ssh access in the chroot jail +@reboot root cd ~repo/j && /usr/sbin/chroot ~repo/j /sbin/sshd + +# The fixup daemon is run in a screen +# For this line to work, the @basedir@/toolbox/updateroot.sh script needs +# to be run as root AFTER "make install" in order to install the fixupd +# files into ~root. Additionally the ../screen/screenrc file needs to +# be installed to ~root/.screenrc to support the SCREENCOMMAND functionality. +@reboot root env SCREENCOMMAND='source "$HOME/repomgr/screen/root"' SHELL=/bin/bash /usr/bin/screen -d -m + +# +## girocco tasks that run as repo +# + +# Update the gitweb project list cache from Girocco gitweb.list file every 8 minutes +# Currently only project creation updates the gitweb.list file +*/8 * * * * repo /usr/bin/ionice -c3 "$HOME/repomgr/jobs/gitwebcache.sh" + +# Do a git fetch origin and if any new commits are present on the branch then +# merge them in and if successful run make install (but not as root) every 15 minutes. +# The branch name to use is hard-coded in updateweb.sh and should be updated before +# enabling this item +#*/15 * * * * repo /usr/bin/ionice -c3 "$HOME/repomgr/jobs/updateweb.sh" + +# Kill stale connections every 6 hours +0 */6 * * * repo "$HOME/repomgr/toolbox/kill-stale-daemons.pl" + +# The job daemon and task daemon are run in a screen +# The ../screen/screenrc file needs to be installed to +# ~repo/.screenrc to support the SCREENCOMMAND functionality. +@reboot repo env SCREENCOMMAND='source "$HOME/repomgr/screen/girocco"' SHELL=/bin/bash /usr/bin/screen -d -m diff --git a/fixupd/README b/fixupd/README index b1cb880..4549f92 100644 --- a/fixupd/README +++ b/fixupd/README @@ -6,3 +6,8 @@ Otherwise, you need to copy fixup* over to ~root and run them as root. You will want to check the configuration at the beginning of fixupd.sh. (The scripts are completely self-contained so that you can easily review them.) + +The ../toolbox/updateroot.sh script can be used to easily copy the +necessary fixupd files to ~root/repomgr. Simply run it as root +AFTER "make install". See also ../cron/README and ../screen/README for +more information on using the sample files to automate running fixupd. diff --git a/fstab/README b/fstab/README new file mode 100644 index 0000000..ba9794c --- /dev/null +++ b/fstab/README @@ -0,0 +1,11 @@ +The fstab.girocco file in this directory is a SAMPLE of additional +lines that can be added to /etc/fstab to properly mount the filesystems +needed for a full Girocco installation that uses a fully chrooted +configuration (see the manage_users setting in Config.pm). + +Please be sure to examine all the paths in the fstab.girocco file +and adjust them as necessary for your installation before using! + +To pick up the fstab.girocco file changes before the next reboot, +after fstab.girocco has been added to /etc/fstab run the +"sudo mount -a" command. diff --git a/fstab/fstab.girocco b/fstab/fstab.girocco new file mode 100644 index 0000000..2a1774b --- /dev/null +++ b/fstab/fstab.girocco @@ -0,0 +1,17 @@ +# These entries should be ADDED to the end of /etc/fstab to +# properly set up the girocco chroot jail + +# /etc/fstab: static file system information. +# +# + +# mount the git repositories in the jail +/srv/git /home/repo/j/srv/git none bind 0 0 + +# mount the proc filesystem in the jail +# Note that there are two possiblities here, the first is a bind mount and +# the second is another proc mount. +# The bind mount will always mirror everything in /proc whereas a second +# proc mount has the potential of having a separate proc namespace. +#/proc /home/repo/j/proc none bind 0 0 +proc /home/repo/j/proc proc nodev,noexec,nosuid 0 0 diff --git a/install.sh b/install.sh index ed48382..3266304 100755 --- a/install.sh +++ b/install.sh @@ -20,14 +20,20 @@ owngroup="" echo "*** Setting up basedir..." rm -fr "$cfg_basedir" mkdir -p "$cfg_basedir" -cp -pR Girocco jobd taskd fixupd gitweb html jobs toolbox hooks apache.conf shlib.sh "$cfg_basedir" +cp -pR Girocco jobd taskd fixupd gitweb html jobs toolbox hooks apache.conf shlib.sh screen "$cfg_basedir" # Put the correct Config in place [ "$GIROCCO_CONF" = "Girocco::Config" ] || cp "$(echo "$GIROCCO_CONF" | sed 's#::#/#g; s/$/.pm/')" "$cfg_basedir/Girocco/Config.pm" echo "*** Preprocessing scripts..." -perl -I. -M$GIROCCO_CONF -i -pe 's/\@basedir\@/"$Girocco::Config::basedir"/g' "$cfg_basedir"/jobs/*.sh "$cfg_basedir"/jobd/*.sh "$cfg_basedir"/taskd/*.sh "$cfg_basedir"/gitweb/*.sh "$cfg_basedir"/shlib.sh "$cfg_basedir"/hooks/* "$cfg_basedir"/toolbox/* +perl -I. -M$GIROCCO_CONF -i -p \ + -e 's/(?&2 + exit 1 +fi +echo $$ >${LOCK_FILE} + +pushd ${REPO_DIR} > /dev/null +git fetch origin +if [ "`git rev-list $BRANCH..$REMOTE/$BRANCH`" ]; then + git merge --ff-only $REMOTE/$BRANCH && make --quiet && make install +fi +popd > /dev/null + +rm ${LOCK_FILE} diff --git a/rsyslog/README b/rsyslog/README new file mode 100644 index 0000000..3c75a8c --- /dev/null +++ b/rsyslog/README @@ -0,0 +1,11 @@ +The girocco.conf file in this directory is a SAMPLE rsyslog file +that can be placed in /etc/rsyslog.d/ to properly set up the chroot +jail log socket for a full Girocco installation that uses a fully +chrooted configuration (see the manage_users setting in Config.pm). + +Please be sure to examine all the paths in the girocco.conf file +and adjust them as necessary for your installation before using! + +To pick up the girocco.conf file changes before the next reboot, +after girocco.conf has been installed in /etc/rsyslog.d/ run the +"sudo /etc/init.d/rsyslog restart" command. diff --git a/rsyslog/girocco.conf b/rsyslog/girocco.conf new file mode 100644 index 0000000..b5c8e37 --- /dev/null +++ b/rsyslog/girocco.conf @@ -0,0 +1,4 @@ +# Create an additional socket in girocco's chroot in order to not break +# logging from within the chroot jail. If the directory is missing, +# rsyslog will silently skip creating the socket. +$AddUnixListenSocket /home/repo/j/dev/log diff --git a/screen/README b/screen/README new file mode 100644 index 0000000..efbe89f --- /dev/null +++ b/screen/README @@ -0,0 +1,35 @@ +The files in this directory are for use with screen. + +If using the ../cron/girocco cron setup to run the jobd and taskd, +then the screenrc file in this directory should be installed as +$HOME/.screenrc in order for the ../cron/girocco cron file to work +properly. (The $HOME directory here is that of the Config.pm +mirror_user.) + +If using the ../cron/girocco cron setup to run the fixupd, +then the screenrc file in this directory should be installed as +~root/.screenrc in order for the ../cron/girocco cron file to work +and AFTER make install, @basedir@/toolbox/updateroot.sh should be +run as root to install the fixupd files into ~root/repomgr. + +The screenrc file in this directory should not need to be modified. + +The root file in this directory should not need to be modified and +is copied into ~root/repomgr/screen/root for use by the entry in +../cron/girocco that starts fixupd as root. + +The girocco file in this directory is used by the ../cron/girocco +cron file as-is and should not need to be modified. + +The giroccoacl file in this directory adds multiuser screen access +and should be modified to add the desired users to the multiuser +screen that runs jobd and taskd. To use an alternate giroccoacl +file instead, simply copy the giroccoacl file somewhere else, modify +it and then set the Config.pm $screen_acl_file setting to the new +location. + +Note that in order to make screen multiuser capable, these commands +need to be run: + + sudo chmod u+s /usr/bin/screen + sudo chmod g-w /var/run/screen diff --git a/screen/girocco b/screen/girocco new file mode 100644 index 0000000..425f0ce --- /dev/null +++ b/screen/girocco @@ -0,0 +1,26 @@ +# See README +# If using the screenrc file with SCREENCOMMAND +# support this file can be run as part of the +# initial screen command like so: +# SCREENCOMMAND='source girocco' screen + +# or source this from a .screenrc file with: +# source girocco + +# or use the screen -X command like so: +# screen -X 'source girocco' + +# Avoid HUP problems +autodetach on + +multiuser on +sessionname admins +screen -t jobd 0 "@basedir@/toolbox/run-jobd.sh" --shell +screen -t taskd 1 "@basedir@/toolbox/run-taskd.sh" --shell +screen +select 0 + +# Set up the multiuser access permissions +# This defaults to the giroccoacl file in this directory +# See README for further information about changing it +source @screen_acl_file@ diff --git a/screen/giroccoacl b/screen/giroccoacl new file mode 100644 index 0000000..7cc87d3 --- /dev/null +++ b/screen/giroccoacl @@ -0,0 +1,14 @@ +# This file is sourced from the girocco file to +# add multiuser access permissions + +# Note that if the Config.pm $screen_acl_file setting is +# set an alternate copy of this file will be sourced. +# See README for further information. + +# Add other users here +# If the Config.pm mirror_user setting is not "repo" +# then that value should be used below instead of "repo" + +#aclgrp userlogin repo +#aclgrp alice repo +#aclgrp bob repo diff --git a/screen/root b/screen/root new file mode 100644 index 0000000..0055b9d --- /dev/null +++ b/screen/root @@ -0,0 +1,19 @@ +# See README +# If using the screenrc file with SCREENCOMMAND +# support this file can be run as part of the +# initial screen command like so: +# SCREENCOMMAND='source root' screen + +# or source this from a .screenrc file with: +# source root + +# or use the screen -X command like so: +# screen -X 'source root' + +# Avoid HUP problems +autodetach on + +sessionname admin +screen -t fixupd 0 "$HOME/repomgr/toolbox/run-fixupd.sh" --shell +screen +select 0 diff --git a/screen/screenrc b/screen/screenrc new file mode 100644 index 0000000..4301c3c --- /dev/null +++ b/screen/screenrc @@ -0,0 +1,46 @@ +# Turn off startup message +startup_message off +# 100 times the default scrollback size +defscrollback 10000 +# Don't block sending output +defnonblock on +# Select a less interfering escape sequence +# (Disabled to avoid a nasty surprise) +#escape ^^\^ +# Remember the original TERM type +setenv SCREENTERM "$TERM" +# UTF-8 by default +defutf8 on +# Disable the alternate screen to enable terminal scrollback +termcapinfo xterm*|rxvt*|kterm*|Eterm* ti@:te@ +# Fix the delete key for broken xterm-color terminfo on linux +termcapinfo xterm-color kD=\E[3~ +# Use this to fix the delete key in a running screen +#bindkey -d -k kD +# Fix the down-one-line/delete-line problems on vt100 +termcapinfo vt100* do=\E7^J\E8\E[1B:dl=5\E[M +# Send hardstatus to the window title +termcapinfo xterm*|rxvt*|kterm*|Eterm* 'hs:ts=\E]2;:fs=\007:ds=\E]2;screen\007' +# Attempt to update running command name +shelltitle "$ |bash" +# Do not use any hardstatus terminal capability +hardstatus off +# Status always on last line +hardstatus alwayslastline "%{.kr}%H %{-}%{.kg}%0c:%s %{-}%{.ky}%M%d %{-}%0L=%{.kw}%-Lw%{-}%{.YB}%50>%n*%f %t%?(%u)%?%{-}%{.kw}%+Lw%{-}%{.kr}%-17= [%l]%{-}" +# Caption any splits +caption splitonly "%?%F%{.r.}%:%{=r}%?%n%?%F*%?%f %t%?(%u)%?%=%? [%h]%?%{-}" +# set visual bell message +vbell_msg " *** WUFF *** *** WUFF *** " +# turn visual bell on +vbell on + +# Every new screen a login shell +shell -$SHELL +# Re-run PATH setter since screen can screw up the PATH on OS X +unsetenv BASHRC + +# Finally execute any SCREENCOMMAND command +setenv _ "$SCREENCOMMAND" +unsetenv SCREENCOMMAND +eval "$_" +unsetenv _ diff --git a/toolbox/kill-stale-daemons.pl b/toolbox/kill-stale-daemons.pl new file mode 100755 index 0000000..030b106 --- /dev/null +++ b/toolbox/kill-stale-daemons.pl @@ -0,0 +1,23 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use lib @basedir@; + +use Girocco::Config; +use Proc::ProcessTable; + +use Data::Dumper; + +my $week_ago = time-60*60*24*7; +my $t = new Proc::ProcessTable; + +my $reporoot = $Girocco::Config::reporoot; +foreach my $p ( @{$t->table} ) { + if ($p->start() < $week_ago && + ($p->cmndline() =~ /git-daemon --inetd --verbose --enable=upload-archive --base-path=\Q$reporoot\E --export-all/o || + $p->cmndline() =~ /git upload-pack --strict --timeout=0 \./)) { + print Dumper $p if $ENV{'PERL_DEBUG'}; + kill 9, $p->pid; + } +} diff --git a/toolbox/run-fixupd.sh b/toolbox/run-fixupd.sh new file mode 100644 index 0000000..81d134c --- /dev/null +++ b/toolbox/run-fixupd.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Note that since the fixupd-related files are always installed into a fixed +# location in ~root by updateroot.sh (specifically ~root/repomgr/...), and this +# script must always be run as root, it would be incorrect to use the basedir +# setting in this script. + +cd "$HOME" + +if [ "$1" = "--shell" ]; then + shift + $HOME/repomgr/fixupd/fixupd.sh "$@" + exec "$SHELL" -l -i +fi +exec $HOME/repomgr/fixupd/fixupd.sh "$@" diff --git a/toolbox/run-jobd.sh b/toolbox/run-jobd.sh new file mode 100755 index 0000000..98644d9 --- /dev/null +++ b/toolbox/run-jobd.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +cd "$HOME" + +DEFAULTOPTS="-q -P --load-triggers 12,6 -d 300" + +if [ "$1" = "--shell" ]; then + shift + /usr/bin/nice -n 18 perl -I@basedir@/jobd @basedir@/jobd/jobd.pl $DEFAULTOPTS "$@" + exec "$SHELL" -l -i +fi +exec /usr/bin/nice -n 18 perl -I@basedir@/jobd @basedir@/jobd/jobd.pl $DEFAULTOPTS "$@" diff --git a/toolbox/run-taskd.sh b/toolbox/run-taskd.sh new file mode 100755 index 0000000..165a44c --- /dev/null +++ b/toolbox/run-taskd.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +cd "$HOME" + +if [ "$1" = "--shell" ]; then + shift + perl -I@basedir@/taskd @basedir@/taskd/taskd.pl "$@" + exec "$SHELL" -l -i +fi +exec perl -I@basedir@/taskd @basedir@/taskd/taskd.pl "$@" diff --git a/toolbox/updateroot.sh b/toolbox/updateroot.sh new file mode 100755 index 0000000..f96bdcf --- /dev/null +++ b/toolbox/updateroot.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# See ../cron/README and ../screen/README + +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: Must be run as root" + exit 1 +fi + +set -e + +basedir=@basedir@ + +FILES='' +FILES="$FILES fixupd/README fixupd/fixup.sh fixupd/fixupd.sh" +FILES="$FILES toolbox/run-fixupd.sh screen/root" + +umask 022 +CPFLAGS=--preserve=timestamps + +echo "*** Checking for source files ($basedir)..." +bad= +for sf in $FILES; do + if [ ! -r "$basedir/$sf" ]; then + bad=1 + echo "ERROR: Missing source file $basedir/$sf" + fi +done +if [ -n "$bad" ]; then + echo "ERROR: Some source files missing, cannot copy" + exit 1 +fi + +echo "*** Setting up root scripts (~root/repomgr)..." +mkdir -p ~root/repomgr/fixupd ~root/repomgr/toolbox ~root/repomgr/screen +chown -R 0:0 ~root/repomgr +chmod -R u+w,go-w ~root/repomgr +for sf in $FILES; do + cp $CPFLAGS "$basedir/$sf" ~root/repomgr/$sf.new +done +chown -R 0:0 ~root/repomgr +chmod -R u+w,go-w ~root/repomgr +for sf in $FILES; do + mv -f ~root/repomgr/$sf.new ~root/repomgr/$sf +done +echo "--- ~root/repomgr created/updated" +echo "--- Restart the fixupd.sh script if needed" +echo "--- Add $basedir/crontab/girocco to /etc/cron.d/" diff --git a/xinetd/README b/xinetd/README new file mode 100644 index 0000000..2046515 --- /dev/null +++ b/xinetd/README @@ -0,0 +1,14 @@ +The git file in this directory is a SAMPLE xinetd file +that can be placed in /etc/xinetd.d/ to properly set up the git +daemon to enable cloning over the git protocol on port 9418. + +Please be sure to examine all the paths in the git file +and adjust them as necessary for your installation before using! + +Especially note that if IPv6 is disabled on the system or +net.ipv6.bindv6only is enabled (set to other than 0), then the flags +setting will certainly need to be adjusted! + +To pick up the git xinetd changes before the next reboot, +after the git file has been installed in /etc/xinetd.d/ run the +"sudo killall -HUP xinetd" command. diff --git a/xinetd/git b/xinetd/git new file mode 100644 index 0000000..2edc2cc --- /dev/null +++ b/xinetd/git @@ -0,0 +1,15 @@ +service git +{ + type = UNLISTED + flags = IPv6 + socket_type = stream + protocol = tcp + port = 9418 + wait = no + user = repo + server = /home/repo/bin/git + server_args = daemon --inetd --verbose --enable=upload-archive --base-path=/srv/git --export-all + per_source = 2 + cps = 10 5 + max_load = 12 +} -- 2.11.4.GIT