From f45c3bb1ea9c0e80337e2fff5abf6bd4eee6ddae Mon Sep 17 00:00:00 2001 From: Jakub Narebski Date: Sun, 5 Dec 2010 21:48:53 +0100 Subject: [PATCH] gitweb: Add optional output caching This commit actually adds output caching to gitweb, as we have now minimal features required for it in GitwebCache::SimpleFileCache (a 'dumb' but fast file-based cache engine). To enable cache you need (at least) set $caching_enabled to true in gitweb config, and copy required modules alongside generated gitweb.cgi - this is described in more detail in the new "Gitweb caching" section in gitweb/README. "make install-gitweb" would install all modules alongside gitweb itself. Caching in theory can be done using any Perl module that implements Cache::Cache compatibile get/set (method) interface. The default is to use GitwebCache::SimpleFileCache. Capturing and caching output is done via cache_output subroutine from GitwebCache::CacheOutput. The cache_output subroutine in GitwebCache::CacheOutput currently uses GitwebCache::Capture::Simple compatibile capturing engine passed as one of parameters to cache_output subroutine. The default is to use GitwebCache::Capture::Simple package. Capturing and caching is designed in such way that there is no behaviour change if $caching_enabled is false. If caching is not enabled, then capturing is also turned off. Enabling caching causes the following additional changes to gitweb output: * Disables content-type negotiation (choosing between 'text/html' mimetype and 'application/xhtml+xml') when caching, as there is no content-type negotiation done when retrieving page from cache. Use lowest common denominator of 'text/html' mimetype which can be used by all browsers. This may change in the future. * Disable optional timing info (how much time it took to generate the original page, and how many git commands it took), and in its place show unconditionally when page was originally generated (in GMT / UTC timezone). * Disable 'blame_incremental' view, as it doesn't make sense without printing data as soon as it is generated (which would require tee-ing when capturing output for caching)... and it doesn't work currently anyway. Alternate solution would be to run 'blame_incremental' view with caching disabled. Add basic tests of caching support to t9500-gitweb-standalone-no-errors test: set $caching_enabled to true and check for errors for first time run (generating cache) and second time run (retrieving from cache) for a single view - summary view for a project. Check in the t9501-gitweb-standalone-http-status test that gitweb at least correctly handles "404 Not Found" error pages also in the case when gitweb caching is enabled. Check in the t9502-gitweb-standalone-parse-output test that gitweb produces the same output with and without caching, for first and second run, with binary or text output. All those tests make use of new gitweb_enable_caching subroutine added to gitweb-lib.sh Inspired-by-code-by: John 'Warthog9' Hawley Signed-off-by: Jakub Narebski --- gitweb/Makefile | 5 ++ gitweb/README | 58 ++++++++++++ gitweb/gitweb.perl | 144 ++++++++++++++++++++++++++---- t/gitweb-lib.sh | 11 +++ t/t9500-gitweb-standalone-no-errors.sh | 20 +++++ t/t9501-gitweb-standalone-http-status.sh | 13 +++ t/t9502-gitweb-standalone-parse-output.sh | 33 +++++++ 7 files changed, 265 insertions(+), 19 deletions(-) mode change 100644 => 100755 t/gitweb-lib.sh diff --git a/gitweb/Makefile b/gitweb/Makefile index e6029e13e6..d67c1388e3 100644 --- a/gitweb/Makefile +++ b/gitweb/Makefile @@ -113,6 +113,11 @@ endif GITWEB_FILES += static/git-logo.png static/git-favicon.png +# gitweb output caching +GITWEB_MODULES += GitwebCache/CacheOutput.pm +GITWEB_MODULES += GitwebCache/SimpleFileCache.pm +GITWEB_MODULES += GitwebCache/Capture/Simple.pm + GITWEB_REPLACE = \ -e 's|++GIT_VERSION++|$(GIT_VERSION)|g' \ -e 's|++GIT_BINDIR++|$(bindir)|g' \ diff --git a/gitweb/README b/gitweb/README index bf3664f2b7..3dc01bd114 100644 --- a/gitweb/README +++ b/gitweb/README @@ -246,6 +246,12 @@ not include variables usually directly set during build): http://www.andre-simon.de due to assumptions about parameters and output). Useful if highlight is not installed on your webserver's PATH. [Default: highlight] + * $caching_enabled + If true, gitweb would use caching to speed up generating response. + Currently supported is only output (response) caching. See "Gitweb caching" + section below for details on how to configure and customize caching. + The default is false (caching is disabled). + Projects list file format ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -317,6 +323,58 @@ You can use the following files in repository: descriptions. +Gitweb caching +~~~~~~~~~~~~~~ + +Currently gitweb supports only output (HTTP response) caching, similar +to the one used on http://git.kernel.org. To turn it on, set +$caching_enabled variable to true value in gitweb config file, i.e.: + + our $caching_enabled = 1; + +You can choose which caching engine should gitweb use by setting $cache +variable to _initialized_ instance of cache interface, e.g.: + + use CHI; + our $cache = CHI->new( driver => 'Memcached', + servers => [ "10.0.0.15:11211", "10.0.0.15:11212" ], + l1_cache => { driver => 'FastMmap', root_dir => '/var/cache/gitweb' } + ); + +Alternatively you can set $cache variable to the name of cache class, +e.g.: + + our $cache = 'Cache::FileCache'; + +In this case caching engine should support Cache::Cache or CHI names for +cache config (see below), and ignore unrecognized options. Such caching +engine should also implement (at least) ->get($key) and ->set($key, $data) +methods (Cache::Cache and CHI compatible interface). + +If $cache is left unset (if it is left undefined), then gitweb would use +GitwebCache::SimpleFileCache as caching engine. This engine is 'dumb' (but +fast) file based caching layer, currently without any support for cache size +limiting, or even removing expired / grossly expired entries. It has +therefore the downside of requiring a huge amount of disk space if there are +a number of repositories involved. It is not uncommon for git.kernel.org to +have on the order of 80G - 120G accumulate over the course of a few months. +It is therefore recommended that the cache directory be periodically +completely deleted; this operation is safe to perform. Suggested mechanism +(substitute $cachedir for actual path to gitweb cache): + + # mv $cachedir $cachedir.flush && mkdir $cachedir && rm -rf $cachedir.flush + +Site-wide cache options are defined in %cache_options hash. Those options +apply only when $cache is unset (GitwebCache::SimpleFileCache is used), or +if $cache is name of cache class (e.g. $cache = 'Cache::FileCache'). You +can override cache options in gitweb config, e.g.: + + $cache_options{'expires_in'} = 60; # 60 seconds = 1 minute + +Please read comments for %cache_options entries in gitweb/gitweb.perl for +description of available cache options. + + Webserver configuration ----------------------- diff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl index cfa511c3d3..32867098b2 100755 --- a/gitweb/gitweb.perl +++ b/gitweb/gitweb.perl @@ -270,6 +270,48 @@ our %highlight_ext = ( map { $_ => 'xml' } qw(xhtml html htm), ); + +# This enables/disables the caching layer in gitweb. Currently supported +# is only output (response) caching, similar to the one used on git.kernel.org. +our $caching_enabled = 0; +# Set to _initialized_ instance of cache interface implementing (at least) +# get($key) and set($key, $data) methods (Cache::Cache and CHI interfaces), +# or to name of class of cache interface implementing said methods. +# If unset, GitwebCache::SimpleFileCache would be used, which is 'dumb' +# (but fast) file based caching layer, currently without any support for +# cache size limiting. It is therefore recommended that the cache directory +# be periodically completely deleted; this operation is safe to perform. +# Suggested mechanism: +# mv $cachedir $cachedir.flush && mkdir $cachedir && rm -rf $cachedir.flush +our $cache; +# You define site-wide cache options defaults here; override them with +# $GITWEB_CONFIG as necessary. +our %cache_options = ( + # The location in the filesystem that will hold the root of the cache. + # This directory will be created as needed (if possible) on the first + # cache set. Note that either this directory must exists and web server + # has to have write permissions to it, or web server must be able to + # create this directory. + # Possible values: + # * 'cache' (relative to gitweb), + # * File::Spec->catdir(File::Spec->tmpdir(), 'gitweb-cache'), + # * '/var/cache/gitweb' (FHS compliant, requires being set up), + 'cache_root' => 'cache', + + # The number of subdirectories deep to cache object item. This should be + # large enough that no cache directory has more than a few hundred + # objects. Each non-leaf directory contains up to 256 subdirectories + # (00-ff). Must be larger than 0. + 'cache_depth' => 1, + + # The (global) expiration time for objects placed in the cache, in seconds. + 'expires_in' => 20, +); +# Set to _initialized_ instance of GitwebCache::Capture compatibile capturing +# engine, i.e. one implementing ->new() constructor, and ->capture($code) +# method. If unset (default), the GitwebCache::Capture::Simple would be used. +our $capture; + # You define site-wide feature defaults here; override them with # $GITWEB_CONFIG as necessary. our %feature = ( @@ -1069,7 +1111,15 @@ sub dispatch { !$project) { die_error(400, "Project needed"); } - $actions{$action}->(); + + if ($caching_enabled) { + # human readable key identifying gitweb output + my $output_key = href(-replay => 1, -full => 1, -path_info => 0); + + cache_output($cache, $capture, $output_key, $actions{$action}); + } else { + $actions{$action}->(); + } } sub reset_timer { @@ -1085,6 +1135,8 @@ sub run_request { evaluate_gitweb_config(); evaluate_git_version(); check_loadavg(); + configure_caching() + if ($caching_enabled); # $projectroot and $projects_list might be set in gitweb config file $projects_list ||= $projectroot; @@ -1157,6 +1209,42 @@ sub run { 1; } +sub configure_caching { + if (!eval { require GitwebCache::CacheOutput; 1; }) { + # cache is configured _before_ handling request, so $cgi is not defined, + # so we can't just "die" with sending error message to web browser + #die_error(500, "Caching enabled and GitwebCache::CacheOutput not found"); + + # turn off caching and warn instead + $caching_enabled = 0; + warn "Caching enabled and GitwebCache::CacheOutput not found"; + } + GitwebCache::CacheOutput->import(); + + # $cache might be initialized (instantiated) cache, i.e. cache object, + # or it might be name of class, or it might be undefined + unless (defined $cache && ref($cache)) { + $cache ||= 'GitwebCache::SimpleFileCache'; + eval "require $cache"; + die $@ if $@; + $cache = $cache->new({ + %cache_options, + #'cache_root' => '/tmp/cache', + #'cache_depth' => 2, + #'expires_in' => 20, # in seconds (CHI compatibile) + # (Cache::Cache compatibile initialization) + 'default_expires_in' => $cache_options{'expires_in'}, + # (CHI compatibile initialization) + 'root_dir' => $cache_options{'cache_root'}, + 'depth' => $cache_options{'cache_depth'}, + }); + } + unless (defined $capture && ref($capture)) { + require GitwebCache::Capture::Simple; + $capture = GitwebCache::Capture::Simple->new(); + } +} + run(); if (defined caller) { @@ -3420,7 +3508,9 @@ sub git_header_html { # 'application/xhtml+xml', otherwise send it as plain old 'text/html'. # we have to do this because MSIE sometimes globs '*/*', pretending to # support xhtml+xml but choking when it gets what it asked for. - if (defined $cgi->http('HTTP_ACCEPT') && + # Disable content-type negotiation when caching (use mimetype good for all). + if (!$caching_enabled && + defined $cgi->http('HTTP_ACCEPT') && $cgi->http('HTTP_ACCEPT') =~ m/(,|;|\s|^)application\/xhtml\+xml(,|;|\s|$)/ && $cgi->Accept('application/xhtml+xml') != 0) { $content_type = 'application/xhtml+xml'; @@ -3445,7 +3535,9 @@ sub git_header_html { EOF # the stylesheet, favicon etc urls won't work correctly with path_info # unless we set the appropriate base URL - if ($ENV{'PATH_INFO'}) { + # if caching is enabled we can get it from cache for path_info when it + # is generated without path_info + if ($ENV{'PATH_INFO'} || $caching_enabled) { print "\n"; } # print out each stylesheet that exist, providing backwards capability @@ -3594,17 +3686,25 @@ sub git_footer_html { } print "\n"; # class="page_footer" - if (defined $t0 && gitweb_check_feature('timed')) { + # timing info doesn't make much sense with output (response) caching, + # so when caching is enabled gitweb prints the time of page generation + if ((defined $t0 || $caching_enabled) && + gitweb_check_feature('timed')) { print "
\n"; - print 'This page took '. - ''. - Time::HiRes::tv_interval($t0, [Time::HiRes::gettimeofday()]). - ' seconds '. - ' and '. - ''. - $number_of_git_cmds. - ' git commands '. - " to generate.\n"; + if ($caching_enabled) { + print 'This page was generated at '. + gmtime( time() )." GMT\n"; + } else { + print 'This page took '. + ''. + Time::HiRes::tv_interval($t0, [Time::HiRes::gettimeofday()]). + ' seconds '. + ' and '. + ''. + $number_of_git_cmds. + ' git commands '. + " to generate.\n"; + } print "
\n"; # class="page_footer" } @@ -3613,8 +3713,8 @@ sub git_footer_html { } print qq!\n!; - if (defined $action && - $action eq 'blame_incremental') { + if (!$caching_enabled && + defined $action && $action eq 'blame_incremental') { print qq!