[mod_cgi] skip local-redir handling if to self (fixes #2779, #2108)
[lighttpd.git] / src / stat_cache.c
blob8fbcca55aeb6e10d9a4fdee0d467f5522b75e4df
1 #include "first.h"
3 #include "log.h"
4 #include "stat_cache.h"
5 #include "fdevent.h"
6 #include "etag.h"
8 #include <sys/types.h>
9 #include <sys/stat.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include <errno.h>
14 #include <unistd.h>
15 #include <stdio.h>
16 #include <fcntl.h>
17 #include <assert.h>
19 #ifdef HAVE_ATTR_ATTRIBUTES_H
20 # include <attr/attributes.h>
21 #endif
23 #ifdef HAVE_SYS_EXTATTR_H
24 # include <sys/extattr.h>
25 #endif
27 #ifdef HAVE_FAM_H
28 # include <fam.h>
29 #endif
31 #ifndef HAVE_LSTAT
32 # define lstat stat
33 #endif
35 #if 0
36 /* enables debug code for testing if all nodes in the stat-cache as accessable */
37 #define DEBUG_STAT_CACHE
38 #endif
41 * stat-cache
43 * we cache the stat() calls in our own storage
44 * the directories are cached in FAM
46 * if we get a change-event from FAM, we increment the version in the FAM->dir mapping
48 * if the stat()-cache is queried we check if the version id for the directory is the
49 * same and return immediatly.
52 * What we need:
54 * - for each stat-cache entry we need a fast indirect lookup on the directory name
55 * - for each FAMRequest we have to find the version in the directory cache (index as userdata)
57 * stat <<-> directory <-> FAMRequest
59 * if file is deleted, directory is dirty, file is rechecked ...
60 * if directory is deleted, directory mapping is removed
62 * */
64 #ifdef HAVE_FAM_H
65 typedef struct {
66 FAMRequest *req;
68 buffer *name;
70 int version;
71 } fam_dir_entry;
72 #endif
74 /* the directory name is too long to always compare on it
75 * - we need a hash
76 * - the hash-key is used as sorting criteria for a tree
77 * - a splay-tree is used as we can use the caching effect of it
80 /* we want to cleanup the stat-cache every few seconds, let's say 10
82 * - remove entries which are outdated since 30s
83 * - remove entries which are fresh but havn't been used since 60s
84 * - if we don't have a stat-cache entry for a directory, release it from the monitor
87 #ifdef DEBUG_STAT_CACHE
88 typedef struct {
89 int *ptr;
91 size_t used;
92 size_t size;
93 } fake_keys;
95 static fake_keys ctrl;
96 #endif
98 stat_cache *stat_cache_init(void) {
99 stat_cache *sc = NULL;
101 sc = calloc(1, sizeof(*sc));
102 force_assert(NULL != sc);
104 sc->dir_name = buffer_init();
105 sc->hash_key = buffer_init();
107 #ifdef HAVE_FAM_H
108 sc->fam_fcce_ndx = -1;
109 #endif
111 #ifdef DEBUG_STAT_CACHE
112 ctrl.size = 0;
113 #endif
115 return sc;
118 static stat_cache_entry * stat_cache_entry_init(void) {
119 stat_cache_entry *sce = NULL;
121 sce = calloc(1, sizeof(*sce));
122 force_assert(NULL != sce);
124 sce->name = buffer_init();
125 sce->etag = buffer_init();
126 sce->content_type = buffer_init();
128 return sce;
131 static void stat_cache_entry_free(void *data) {
132 stat_cache_entry *sce = data;
133 if (!sce) return;
135 buffer_free(sce->etag);
136 buffer_free(sce->name);
137 buffer_free(sce->content_type);
139 free(sce);
142 #ifdef HAVE_FAM_H
143 static fam_dir_entry * fam_dir_entry_init(void) {
144 fam_dir_entry *fam_dir = NULL;
146 fam_dir = calloc(1, sizeof(*fam_dir));
147 force_assert(NULL != fam_dir);
149 fam_dir->name = buffer_init();
151 return fam_dir;
154 static void fam_dir_entry_free(FAMConnection *fc, void *data) {
155 fam_dir_entry *fam_dir = data;
157 if (!fam_dir) return;
159 FAMCancelMonitor(fc, fam_dir->req);
161 buffer_free(fam_dir->name);
162 free(fam_dir->req);
164 free(fam_dir);
166 #endif
168 void stat_cache_free(stat_cache *sc) {
169 while (sc->files) {
170 int osize;
171 splay_tree *node = sc->files;
173 osize = sc->files->size;
175 stat_cache_entry_free(node->data);
176 sc->files = splaytree_delete(sc->files, node->key);
178 force_assert(osize - 1 == splaytree_size(sc->files));
181 buffer_free(sc->dir_name);
182 buffer_free(sc->hash_key);
184 #ifdef HAVE_FAM_H
185 while (sc->dirs) {
186 int osize;
187 splay_tree *node = sc->dirs;
189 osize = sc->dirs->size;
191 fam_dir_entry_free(&sc->fam, node->data);
192 sc->dirs = splaytree_delete(sc->dirs, node->key);
194 if (osize == 1) {
195 force_assert(NULL == sc->dirs);
196 } else {
197 force_assert(osize == (sc->dirs->size + 1));
201 if (-1 != sc->fam_fcce_ndx) {
202 /* fd events already gone */
203 sc->fam_fcce_ndx = -1;
205 FAMClose(&sc->fam);
207 #endif
208 free(sc);
211 #if defined(HAVE_XATTR)
212 static int stat_cache_attr_get(buffer *buf, char *name, char *xattrname) {
213 int attrlen;
214 int ret;
216 buffer_string_prepare_copy(buf, 1023);
217 attrlen = buf->size - 1;
218 if(0 == (ret = attr_get(name, xattrname, buf->ptr, &attrlen, 0))) {
219 buffer_commit(buf, attrlen);
221 return ret;
223 #elif defined(HAVE_EXTATTR)
224 static int stat_cache_attr_get(buffer *buf, char *name, char *xattrname) {
225 ssize_t attrlen;
227 buffer_string_prepare_copy(buf, 1023);
229 if (-1 != (attrlen = extattr_get_file(name, EXTATTR_NAMESPACE_USER, xattrname, buf->ptr, buf->size - 1))) {
230 buf->used = attrlen + 1;
231 buf->ptr[attrlen] = '\0';
232 return 0;
234 return -1;
236 #endif
238 /* the famous DJB hash function for strings */
239 static uint32_t hashme(buffer *str) {
240 uint32_t hash = 5381;
241 const char *s;
242 for (s = str->ptr; *s; s++) {
243 hash = ((hash << 5) + hash) + *s;
246 hash &= ~(((uint32_t)1) << 31); /* strip the highest bit */
248 return hash;
251 #ifdef HAVE_FAM_H
252 handler_t stat_cache_handle_fdevent(server *srv, void *_fce, int revent) {
253 size_t i;
254 stat_cache *sc = srv->stat_cache;
255 size_t events;
257 UNUSED(_fce);
258 /* */
260 if (revent & FDEVENT_IN) {
261 events = FAMPending(&sc->fam);
263 for (i = 0; i < events; i++) {
264 FAMEvent fe;
265 fam_dir_entry *fam_dir;
266 splay_tree *node;
267 int ndx, j;
269 FAMNextEvent(&sc->fam, &fe);
271 /* handle event */
273 switch(fe.code) {
274 case FAMChanged:
275 case FAMDeleted:
276 case FAMMoved:
277 /* if the filename is a directory remove the entry */
279 fam_dir = fe.userdata;
280 fam_dir->version++;
282 /* file/dir is still here */
283 if (fe.code == FAMChanged) break;
285 /* we have 2 versions, follow and no-follow-symlink */
287 for (j = 0; j < 2; j++) {
288 buffer_copy_string(sc->hash_key, fe.filename);
289 buffer_append_int(sc->hash_key, j);
291 ndx = hashme(sc->hash_key);
293 sc->dirs = splaytree_splay(sc->dirs, ndx);
294 node = sc->dirs;
296 if (node && (node->key == ndx)) {
297 int osize = splaytree_size(sc->dirs);
299 fam_dir_entry_free(&sc->fam, node->data);
300 sc->dirs = splaytree_delete(sc->dirs, ndx);
302 force_assert(osize - 1 == splaytree_size(sc->dirs));
305 break;
306 default:
307 break;
312 if (revent & FDEVENT_HUP) {
313 /* fam closed the connection */
314 fdevent_event_del(srv->ev, &(sc->fam_fcce_ndx), FAMCONNECTION_GETFD(&sc->fam));
315 fdevent_unregister(srv->ev, FAMCONNECTION_GETFD(&sc->fam));
317 FAMClose(&sc->fam);
320 return HANDLER_GO_ON;
323 static int buffer_copy_dirname(buffer *dst, buffer *file) {
324 size_t i;
326 if (buffer_string_is_empty(file)) return -1;
328 for (i = buffer_string_length(file); i > 0; i--) {
329 if (file->ptr[i] == '/') {
330 buffer_copy_string_len(dst, file->ptr, i);
331 return 0;
335 return -1;
337 #endif
339 #ifdef HAVE_LSTAT
340 static int stat_cache_lstat(server *srv, buffer *dname, struct stat *lst) {
341 if (lstat(dname->ptr, lst) == 0) {
342 return S_ISLNK(lst->st_mode) ? 0 : 1;
344 else {
345 log_error_write(srv, __FILE__, __LINE__, "sbs",
346 "lstat failed for:",
347 dname, strerror(errno));
349 return -1;
351 #endif
353 /***
357 * returns:
358 * - HANDLER_FINISHED on cache-miss (don't forget to reopen the file)
359 * - HANDLER_ERROR on stat() failed -> see errno for problem
362 handler_t stat_cache_get_entry(server *srv, connection *con, buffer *name, stat_cache_entry **ret_sce) {
363 #ifdef HAVE_FAM_H
364 fam_dir_entry *fam_dir = NULL;
365 int dir_ndx = -1;
366 #endif
367 stat_cache_entry *sce = NULL;
368 stat_cache *sc;
369 struct stat st;
370 size_t k;
371 int fd;
372 struct stat lst;
373 #ifdef DEBUG_STAT_CACHE
374 size_t i;
375 #endif
377 int file_ndx;
379 *ret_sce = NULL;
382 * check if the directory for this file has changed
385 sc = srv->stat_cache;
387 buffer_copy_buffer(sc->hash_key, name);
388 buffer_append_int(sc->hash_key, con->conf.follow_symlink);
390 file_ndx = hashme(sc->hash_key);
391 sc->files = splaytree_splay(sc->files, file_ndx);
393 #ifdef DEBUG_STAT_CACHE
394 for (i = 0; i < ctrl.used; i++) {
395 if (ctrl.ptr[i] == file_ndx) break;
397 #endif
399 if (sc->files && (sc->files->key == file_ndx)) {
400 #ifdef DEBUG_STAT_CACHE
401 /* it was in the cache */
402 force_assert(i < ctrl.used);
403 #endif
405 /* we have seen this file already and
406 * don't stat() it again in the same second */
408 sce = sc->files->data;
410 /* check if the name is the same, we might have a collision */
412 if (buffer_is_equal(name, sce->name)) {
413 if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_SIMPLE) {
414 if (sce->stat_ts == srv->cur_ts && con->conf.follow_symlink) {
415 *ret_sce = sce;
416 return HANDLER_GO_ON;
419 } else {
420 /* collision, forget about the entry */
421 sce = NULL;
423 } else {
424 #ifdef DEBUG_STAT_CACHE
425 if (i != ctrl.used) {
426 log_error_write(srv, __FILE__, __LINE__, "xSB",
427 file_ndx, "was already inserted but not found in cache, ", name);
429 force_assert(i == ctrl.used);
430 #endif
433 #ifdef HAVE_FAM_H
434 /* dir-check */
435 if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM) {
436 if (0 != buffer_copy_dirname(sc->dir_name, name)) {
437 log_error_write(srv, __FILE__, __LINE__, "sb",
438 "no '/' found in filename:", name);
439 return HANDLER_ERROR;
442 buffer_copy_buffer(sc->hash_key, sc->dir_name);
443 buffer_append_int(sc->hash_key, con->conf.follow_symlink);
445 dir_ndx = hashme(sc->hash_key);
447 sc->dirs = splaytree_splay(sc->dirs, dir_ndx);
449 if ((NULL != sc->dirs) && (sc->dirs->key == dir_ndx)) {
450 fam_dir = sc->dirs->data;
452 /* check whether we got a collision */
453 if (buffer_is_equal(sc->dir_name, fam_dir->name)) {
454 /* test whether a found file cache entry is still ok */
455 if ((NULL != sce) && (fam_dir->version == sce->dir_version)) {
456 /* the stat()-cache entry is still ok */
458 *ret_sce = sce;
459 return HANDLER_GO_ON;
461 } else {
462 /* hash collision, forget about the entry */
463 fam_dir = NULL;
467 #endif
470 * *lol*
471 * - open() + fstat() on a named-pipe results in a (intended) hang.
472 * - stat() if regular file + open() to see if we can read from it is better
474 * */
475 if (-1 == stat(name->ptr, &st)) {
476 return HANDLER_ERROR;
480 if (S_ISREG(st.st_mode)) {
481 /* fix broken stat/open for symlinks to reg files with appended slash on freebsd,osx */
482 if (name->ptr[buffer_string_length(name) - 1] == '/') {
483 errno = ENOTDIR;
484 return HANDLER_ERROR;
487 /* try to open the file to check if we can read it */
488 if (-1 == (fd = open(name->ptr, O_RDONLY))) {
489 return HANDLER_ERROR;
491 close(fd);
494 if (NULL == sce) {
496 sce = stat_cache_entry_init();
497 buffer_copy_buffer(sce->name, name);
499 /* already splayed file_ndx */
500 if ((NULL != sc->files) && (sc->files->key == file_ndx)) {
501 /* hash collision: replace old entry */
502 stat_cache_entry_free(sc->files->data);
503 sc->files->data = sce;
504 } else {
505 int osize = splaytree_size(sc->files);
507 sc->files = splaytree_insert(sc->files, file_ndx, sce);
508 force_assert(osize + 1 == splaytree_size(sc->files));
510 #ifdef DEBUG_STAT_CACHE
511 if (ctrl.size == 0) {
512 ctrl.size = 16;
513 ctrl.used = 0;
514 ctrl.ptr = malloc(ctrl.size * sizeof(*ctrl.ptr));
515 force_assert(NULL != ctrl.ptr);
516 } else if (ctrl.size == ctrl.used) {
517 ctrl.size += 16;
518 ctrl.ptr = realloc(ctrl.ptr, ctrl.size * sizeof(*ctrl.ptr));
519 force_assert(NULL != ctrl.ptr);
522 ctrl.ptr[ctrl.used++] = file_ndx;
523 #endif
525 force_assert(sc->files);
526 force_assert(sc->files->data == sce);
529 sce->st = st;
530 sce->stat_ts = srv->cur_ts;
532 /* catch the obvious symlinks
534 * this is not a secure check as we still have a race-condition between
535 * the stat() and the open. We can only solve this by
536 * 1. open() the file
537 * 2. fstat() the fd
539 * and keeping the file open for the rest of the time. But this can
540 * only be done at network level.
542 * per default it is not a symlink
543 * */
544 #ifdef HAVE_LSTAT
545 sce->is_symlink = 0;
547 /* we want to only check for symlinks if we should block symlinks.
549 if (!con->conf.follow_symlink) {
550 if (stat_cache_lstat(srv, name, &lst) == 0) {
551 #ifdef DEBUG_STAT_CACHE
552 log_error_write(srv, __FILE__, __LINE__, "sb",
553 "found symlink", name);
554 #endif
555 sce->is_symlink = 1;
559 * we assume "/" can not be symlink, so
560 * skip the symlink stuff if our path is /
562 else if (buffer_string_length(name) > 1) {
563 buffer *dname;
564 char *s_cur;
566 dname = buffer_init();
567 buffer_copy_buffer(dname, name);
569 while ((s_cur = strrchr(dname->ptr, '/'))) {
570 buffer_string_set_length(dname, s_cur - dname->ptr);
571 if (dname->ptr == s_cur) {
572 #ifdef DEBUG_STAT_CACHE
573 log_error_write(srv, __FILE__, __LINE__, "s", "reached /");
574 #endif
575 break;
577 #ifdef DEBUG_STAT_CACHE
578 log_error_write(srv, __FILE__, __LINE__, "sbs",
579 "checking if", dname, "is a symlink");
580 #endif
581 if (stat_cache_lstat(srv, dname, &lst) == 0) {
582 sce->is_symlink = 1;
583 #ifdef DEBUG_STAT_CACHE
584 log_error_write(srv, __FILE__, __LINE__, "sb",
585 "found symlink", dname);
586 #endif
587 break;
590 buffer_free(dname);
593 #endif
595 if (S_ISREG(st.st_mode)) {
596 /* determine mimetype */
597 buffer_reset(sce->content_type);
598 #if defined(HAVE_XATTR) || defined(HAVE_EXTATTR)
599 if (con->conf.use_xattr) {
600 stat_cache_attr_get(sce->content_type, name->ptr, srv->srvconf.xattr_name->ptr);
602 #endif
603 /* xattr did not set a content-type. ask the config */
604 if (buffer_string_is_empty(sce->content_type)) {
605 size_t namelen = buffer_string_length(name);
607 for (k = 0; k < con->conf.mimetypes->used; k++) {
608 data_string *ds = (data_string *)con->conf.mimetypes->data[k];
609 buffer *type = ds->key;
610 size_t typelen = buffer_string_length(type);
612 if (buffer_is_empty(type)) continue;
614 /* check if the right side is the same */
615 if (typelen > namelen) continue;
617 if (0 == strncasecmp(name->ptr + namelen - typelen, type->ptr, typelen)) {
618 buffer_copy_buffer(sce->content_type, ds->value);
619 break;
623 etag_create(sce->etag, &(sce->st), con->etag_flags);
624 } else if (S_ISDIR(st.st_mode)) {
625 etag_create(sce->etag, &(sce->st), con->etag_flags);
628 #ifdef HAVE_FAM_H
629 if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM) {
630 /* is this directory already registered ? */
631 if (NULL == fam_dir) {
632 fam_dir = fam_dir_entry_init();
634 buffer_copy_buffer(fam_dir->name, sc->dir_name);
636 fam_dir->version = 1;
638 fam_dir->req = calloc(1, sizeof(FAMRequest));
639 force_assert(NULL != fam_dir);
641 if (0 != FAMMonitorDirectory(&sc->fam, fam_dir->name->ptr,
642 fam_dir->req, fam_dir)) {
644 log_error_write(srv, __FILE__, __LINE__, "sbsbs",
645 "monitoring dir failed:",
646 fam_dir->name,
647 "file:", name,
648 FamErrlist[FAMErrno]);
650 fam_dir_entry_free(&sc->fam, fam_dir);
651 fam_dir = NULL;
652 } else {
653 int osize = splaytree_size(sc->dirs);
655 /* already splayed dir_ndx */
656 if ((NULL != sc->dirs) && (sc->dirs->key == dir_ndx)) {
657 /* hash collision: replace old entry */
658 fam_dir_entry_free(&sc->fam, sc->dirs->data);
659 sc->dirs->data = fam_dir;
660 } else {
661 sc->dirs = splaytree_insert(sc->dirs, dir_ndx, fam_dir);
662 force_assert(osize == (splaytree_size(sc->dirs) - 1));
665 force_assert(sc->dirs);
666 force_assert(sc->dirs->data == fam_dir);
670 /* bind the fam_fc to the stat() cache entry */
672 if (fam_dir) {
673 sce->dir_version = fam_dir->version;
676 #endif
678 *ret_sce = sce;
680 return HANDLER_GO_ON;
683 int stat_cache_open_rdonly_fstat (server *srv, connection *con, buffer *name, struct stat *st) {
684 /*(Note: O_NOFOLLOW affects only the final path segment, the target file,
685 * not any intermediate symlinks along the path)*/
686 #ifndef O_BINARY
687 #define O_BINARY 0
688 #endif
689 #ifndef O_LARGEFILE
690 #define O_LARGEFILE 0
691 #endif
692 #ifndef O_NOCTTY
693 #define O_NOCTTY 0
694 #endif
695 #ifndef O_NONBLOCK
696 #define O_NONBLOCK 0
697 #endif
698 #ifndef O_NOFOLLOW
699 #define O_NOFOLLOW 0
700 #endif
701 const int oflags = O_BINARY | O_LARGEFILE | O_NOCTTY | O_NONBLOCK
702 | (con->conf.follow_symlink ? 0 : O_NOFOLLOW);
703 const int fd = open(name->ptr, O_RDONLY | oflags);
704 if (fd >= 0) {
705 if (0 == fstat(fd, st)) {
706 return fd;
707 } else {
708 close(fd);
711 UNUSED(srv); /*(might log_error_write(srv, ...) in the future)*/
712 return -1;
716 * remove stat() from cache which havn't been stat()ed for
717 * more than 10 seconds
720 * walk though the stat-cache, collect the ids which are too old
721 * and remove them in a second loop
724 static int stat_cache_tag_old_entries(server *srv, splay_tree *t, int *keys, size_t *ndx) {
725 stat_cache_entry *sce;
727 if (!t) return 0;
729 stat_cache_tag_old_entries(srv, t->left, keys, ndx);
730 stat_cache_tag_old_entries(srv, t->right, keys, ndx);
732 sce = t->data;
734 if (srv->cur_ts - sce->stat_ts > 2) {
735 keys[(*ndx)++] = t->key;
738 return 0;
741 int stat_cache_trigger_cleanup(server *srv) {
742 stat_cache *sc;
743 size_t max_ndx = 0, i;
744 int *keys;
746 sc = srv->stat_cache;
748 if (!sc->files) return 0;
750 keys = calloc(1, sizeof(int) * sc->files->size);
751 force_assert(NULL != keys);
753 stat_cache_tag_old_entries(srv, sc->files, keys, &max_ndx);
755 for (i = 0; i < max_ndx; i++) {
756 int ndx = keys[i];
757 splay_tree *node;
759 sc->files = splaytree_splay(sc->files, ndx);
761 node = sc->files;
763 if (node && (node->key == ndx)) {
764 #ifdef DEBUG_STAT_CACHE
765 size_t j;
766 int osize = splaytree_size(sc->files);
767 stat_cache_entry *sce = node->data;
768 #endif
769 stat_cache_entry_free(node->data);
770 sc->files = splaytree_delete(sc->files, ndx);
772 #ifdef DEBUG_STAT_CACHE
773 for (j = 0; j < ctrl.used; j++) {
774 if (ctrl.ptr[j] == ndx) {
775 ctrl.ptr[j] = ctrl.ptr[--ctrl.used];
776 break;
780 force_assert(osize - 1 == splaytree_size(sc->files));
781 #endif
785 free(keys);
787 return 0;