2 Unix SMB/CIFS implementation.
4 Generic, persistent and shared between processes cache mechanism for use
5 by various parts of the Samba code
7 Copyright (C) Rafal Szczesniak 2002
8 Copyright (C) Volker Lendecke 2009
10 This program is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3 of the License, or
13 (at your option) any later version.
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
20 You should have received a copy of the GNU General Public License
21 along with this program. If not, see <http://www.gnu.org/licenses/>.
25 #include "system/filesys.h"
26 #include "system/glob.h"
28 #include "tdb_wrap/tdb_wrap.h"
29 #include "../lib/util/memcache.h"
32 #define DBGC_CLASS DBGC_TDB
34 #define CACHE_DATA_FMT "%12u/"
36 static struct tdb_wrap
*cache
;
37 static struct tdb_wrap
*cache_notrans
;
38 static int cache_notrans_seqnum
;
42 * @brief Generic, persistent and shared between processes cache mechanism
43 * for use by various parts of the Samba code
49 * Cache initialisation function. Opens cache tdb file or creates
50 * it if does not exist.
52 * @return true on successful initialisation of the cache or
56 static bool gencache_init(void)
58 char* cache_fname
= NULL
;
59 int open_flags
= O_RDWR
|O_CREAT
;
61 /* skip file open if it's already opened */
66 cache_fname
= cache_path("gencache.tdb");
67 if (cache_fname
== NULL
) {
71 DEBUG(5, ("Opening cache file at %s\n", cache_fname
));
73 cache
= tdb_wrap_open(NULL
, cache_fname
, 0,
74 TDB_DEFAULT
|TDB_INCOMPATIBLE_HASH
,
78 ret
= tdb_check(cache
->tdb
, NULL
, NULL
);
83 * Retry with CLEAR_IF_FIRST.
85 * Warning: Converting this to dbwrap won't work
86 * directly. gencache.c does transactions on this tdb,
87 * and dbwrap forbids this for CLEAR_IF_FIRST
88 * databases. tdb does allow transactions on
89 * CLEAR_IF_FIRST databases, so lets use it here to
90 * clean up a broken database.
92 cache
= tdb_wrap_open(NULL
, cache_fname
, 0,
94 TDB_INCOMPATIBLE_HASH
|
100 if (!cache
&& (errno
== EACCES
)) {
101 open_flags
= O_RDONLY
;
102 cache
= tdb_wrap_open(NULL
, cache_fname
, 0,
103 TDB_DEFAULT
|TDB_INCOMPATIBLE_HASH
,
106 DEBUG(5, ("gencache_init: Opening cache file %s read-only.\n", cache_fname
));
109 TALLOC_FREE(cache_fname
);
112 DEBUG(5, ("Attempt to open gencache.tdb has failed.\n"));
116 cache_fname
= lock_path("gencache_notrans.tdb");
117 if (cache_fname
== NULL
) {
122 DEBUG(5, ("Opening cache file at %s\n", cache_fname
));
124 cache_notrans
= tdb_wrap_open(NULL
, cache_fname
, 0,
126 TDB_INCOMPATIBLE_HASH
|
131 if (cache_notrans
== NULL
) {
132 DEBUG(5, ("Opening %s failed: %s\n", cache_fname
,
134 TALLOC_FREE(cache_fname
);
138 TALLOC_FREE(cache_fname
);
143 static TDB_DATA
last_stabilize_key(void)
146 result
.dptr
= discard_const_p(uint8_t, "@LAST_STABILIZED");
151 struct gencache_have_val_state
{
153 const DATA_BLOB
*data
;
157 static void gencache_have_val_parser(time_t old_timeout
, DATA_BLOB data
,
160 struct gencache_have_val_state
*state
=
161 (struct gencache_have_val_state
*)private_data
;
162 time_t now
= time(NULL
);
163 int cache_time_left
, new_time_left
, additional_time
;
166 * Excuse the many variables, but these time calculations are
167 * confusing to me. We do not want to write to gencache with a
168 * possibly expensive transaction if we are about to write the same
169 * value, just extending the remaining timeout by less than 10%.
172 cache_time_left
= old_timeout
- now
;
173 if (cache_time_left
<= 0) {
175 * timed out, write new value
180 new_time_left
= state
->new_timeout
- now
;
181 if (new_time_left
<= 0) {
183 * Huh -- no new timeout?? Write it.
188 if (new_time_left
< cache_time_left
) {
190 * Someone wants to shorten the timeout. Let it happen.
196 * By how much does the new timeout extend the remaining cache time?
198 additional_time
= new_time_left
- cache_time_left
;
200 if (additional_time
* 10 < 0) {
202 * Integer overflow. We extend by so much that we have to write it.
208 * The comparison below is essentially equivalent to
210 * new_time_left > cache_time_left * 1.10
212 * but without floating point calculations.
215 if (additional_time
* 10 > cache_time_left
) {
217 * We extend the cache timeout by more than 10%. Do it.
223 * Now the more expensive data compare.
225 if (data_blob_cmp(state
->data
, &data
) != 0) {
227 * Write a new value. Certainly do it.
233 * Extending the timeout by less than 10% for the same cache value is
234 * not worth the trouble writing a value into gencache under a
235 * possibly expensive transaction.
240 static bool gencache_have_val(const char *keystr
, const DATA_BLOB
*data
,
243 struct gencache_have_val_state state
;
245 state
.new_timeout
= timeout
;
249 if (!gencache_parse(keystr
, gencache_have_val_parser
, &state
)) {
255 static int last_stabilize_parser(TDB_DATA key
, TDB_DATA data
,
258 time_t *last_stabilize
= private_data
;
260 if ((data
.dsize
!= 0) && (data
.dptr
[data
.dsize
-1] == '\0')) {
261 *last_stabilize
= atoi((char *)data
.dptr
);
267 * Set an entry in the cache file. If there's no such
270 * @param keystr string that represents a key of this entry
271 * @param blob DATA_BLOB value being cached
272 * @param timeout time when the value is expired
274 * @retval true when entry is successfully stored
275 * @retval false on failure
278 bool gencache_set_data_blob(const char *keystr
, const DATA_BLOB
*blob
,
285 time_t last_stabilize
;
286 static int writecount
;
288 if (tdb_data_cmp(string_term_tdb_data(keystr
),
289 last_stabilize_key()) == 0) {
290 DEBUG(10, ("Can't store %s as a key\n", keystr
));
294 if ((keystr
== NULL
) || (blob
== NULL
)) {
298 if (!gencache_init()) {
302 if (gencache_have_val(keystr
, blob
, timeout
)) {
303 DEBUG(10, ("Did not store value for %s, we already got it\n",
308 hdr_len
= fstr_sprintf(hdr
, CACHE_DATA_FMT
, (int)timeout
);
313 if ((blob
->length
+ (size_t)hdr_len
) < blob
->length
) {
317 val
= talloc_array(talloc_tos(), char, hdr_len
+ blob
->length
);
322 memcpy(val
, hdr
, hdr_len
);
323 memcpy(val
+hdr_len
, blob
->data
, blob
->length
);
325 DEBUG(10, ("Adding cache entry with key=[%s] and timeout="
326 "[%s] (%d seconds %s)\n", keystr
,
327 timestring(talloc_tos(), timeout
),
328 (int)(timeout
- time(NULL
)),
329 timeout
> time(NULL
) ? "ahead" : "in the past"));
331 ret
= tdb_store_bystring(
332 cache_notrans
->tdb
, keystr
,
333 make_tdb_data((uint8_t *)val
, talloc_array_length(val
)),
342 * Every 100 writes within a single process, stabilize the cache with
343 * a transaction. This is done to prevent a single transaction to
344 * become huge and chew lots of memory.
347 if (writecount
> lp_parm_int(-1, "gencache", "stabilize_count", 100)) {
348 gencache_stabilize();
354 * Every 5 minutes, call gencache_stabilize() to not let grow
355 * gencache_notrans.tdb too large.
360 tdb_parse_record(cache_notrans
->tdb
, last_stabilize_key(),
361 last_stabilize_parser
, &last_stabilize
);
364 + lp_parm_int(-1, "gencache", "stabilize_interval", 300))
366 gencache_stabilize();
373 static void gencache_del_parser(time_t timeout
, DATA_BLOB blob
,
377 bool *exists
= private_data
;
383 * Delete one entry from the cache file.
385 * @param keystr string that represents a key of this entry
387 * @retval true upon successful deletion
388 * @retval false in case of failure
391 bool gencache_del(const char *keystr
)
393 TDB_DATA key
= string_term_tdb_data(keystr
);
398 if (keystr
== NULL
) {
402 if (!gencache_init()) {
406 DEBUG(10, ("Deleting cache entry (key=[%s])\n", keystr
));
408 ret
= tdb_chainlock(cache_notrans
->tdb
, key
);
413 gencache_parse(keystr
, gencache_del_parser
, &exists
);
417 * We delete an element by setting its timeout to
418 * 0. This way we don't have to do a transaction on
419 * gencache.tdb every time we delete an element.
421 result
= gencache_set(keystr
, "", 0);
424 tdb_chainunlock(cache_notrans
->tdb
, key
);
429 static bool gencache_pull_timeout(char *val
, time_t *pres
, char **pendptr
)
438 res
= strtol(val
, &endptr
, 10);
440 if ((endptr
== NULL
) || (*endptr
!= '/')) {
441 DEBUG(2, ("Invalid gencache data format: %s\n", val
));
447 if (pendptr
!= NULL
) {
453 struct gencache_parse_state
{
454 void (*parser
)(time_t timeout
, DATA_BLOB blob
, void *private_data
);
459 static int gencache_parse_fn(TDB_DATA key
, TDB_DATA data
, void *private_data
)
461 struct gencache_parse_state
*state
;
467 if (data
.dptr
== NULL
) {
470 ret
= gencache_pull_timeout((char *)data
.dptr
, &t
, &endptr
);
474 state
= (struct gencache_parse_state
*)private_data
;
475 blob
= data_blob_const(
476 endptr
+1, data
.dsize
- PTR_DIFF(endptr
+1, data
.dptr
));
477 state
->parser(t
, blob
, state
->private_data
);
479 if (!state
->is_memcache
) {
480 memcache_add(NULL
, GENCACHE_RAM
,
481 data_blob_const(key
.dptr
, key
.dsize
),
482 data_blob_const(data
.dptr
, data
.dsize
));
488 bool gencache_parse(const char *keystr
,
489 void (*parser
)(time_t timeout
, DATA_BLOB blob
,
493 struct gencache_parse_state state
;
494 TDB_DATA key
= string_term_tdb_data(keystr
);
495 DATA_BLOB memcache_val
;
498 if (keystr
== NULL
) {
501 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
504 if (!gencache_init()) {
508 state
.parser
= parser
;
509 state
.private_data
= private_data
;
511 if (memcache_lookup(NULL
, GENCACHE_RAM
,
512 data_blob_const(key
.dptr
, key
.dsize
),
515 * Make sure that nobody has changed the gencache behind our
518 int current_seqnum
= tdb_get_seqnum(cache_notrans
->tdb
);
519 if (current_seqnum
== cache_notrans_seqnum
) {
521 * Ok, our memcache is still current, use it without
522 * going to the tdb files.
524 state
.is_memcache
= true;
525 gencache_parse_fn(key
, make_tdb_data(memcache_val
.data
,
526 memcache_val
.length
),
530 memcache_flush(NULL
, GENCACHE_RAM
);
531 cache_notrans_seqnum
= current_seqnum
;
534 state
.is_memcache
= false;
536 ret
= tdb_parse_record(cache_notrans
->tdb
, key
,
537 gencache_parse_fn
, &state
);
541 ret
= tdb_parse_record(cache
->tdb
, key
, gencache_parse_fn
, &state
);
545 struct gencache_get_data_blob_state
{
552 static void gencache_get_data_blob_parser(time_t timeout
, DATA_BLOB blob
,
555 struct gencache_get_data_blob_state
*state
=
556 (struct gencache_get_data_blob_state
*)private_data
;
559 state
->result
= false;
562 state
->timeout
= timeout
;
564 if (state
->blob
== NULL
) {
565 state
->result
= true;
569 *state
->blob
= data_blob_talloc(state
->mem_ctx
, blob
.data
,
571 if (state
->blob
->data
== NULL
) {
572 state
->result
= false;
575 state
->result
= true;
579 * Get existing entry from the cache file.
581 * @param keystr string that represents a key of this entry
582 * @param blob DATA_BLOB that is filled with entry's blob
583 * @param timeout pointer to a time_t that is filled with entry's
586 * @retval true when entry is successfuly fetched
587 * @retval false for failure
590 bool gencache_get_data_blob(const char *keystr
, TALLOC_CTX
*mem_ctx
,
592 time_t *timeout
, bool *was_expired
)
594 struct gencache_get_data_blob_state state
;
595 bool expired
= false;
597 state
.result
= false;
598 state
.mem_ctx
= mem_ctx
;
601 if (!gencache_parse(keystr
, gencache_get_data_blob_parser
, &state
)) {
607 if (state
.timeout
<= time(NULL
)) {
609 * We're expired, delete the entry. We can't use gencache_del
610 * here, because that uses gencache_get_data_blob for checking
611 * the existence of a record. We know the thing exists and
612 * directly store an empty value with 0 timeout.
614 gencache_set(keystr
, "", 0);
619 *timeout
= state
.timeout
;
625 if (was_expired
!= NULL
) {
626 *was_expired
= expired
;
628 if (state
.result
&& state
.blob
) {
629 data_blob_free(state
.blob
);
634 struct stabilize_state
{
637 static int stabilize_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
640 static int wipe_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
646 * Migrate the clear-if-first gencache data to the stable,
647 * transaction-based gencache.tdb
650 bool gencache_stabilize(void)
652 struct stabilize_state state
;
656 if (!gencache_init()) {
660 res
= tdb_transaction_start_nonblock(cache
->tdb
);
662 if (tdb_error(cache
->tdb
) == TDB_ERR_NOLOCK
)
665 * Someone else already does the stabilize,
666 * this does not have to be done twice
671 DEBUG(10, ("Could not start transaction on gencache.tdb: "
672 "%s\n", tdb_errorstr(cache
->tdb
)));
676 res
= tdb_lockall(cache_notrans
->tdb
);
678 tdb_transaction_cancel(cache
->tdb
);
679 DEBUG(10, ("Could not get allrecord lock on "
680 "gencache_notrans.tdb: %s\n",
681 tdb_errorstr(cache_notrans
->tdb
)));
685 state
.written
= false;
687 res
= tdb_traverse(cache_notrans
->tdb
, stabilize_fn
, &state
);
689 tdb_unlockall(cache_notrans
->tdb
);
690 tdb_transaction_cancel(cache
->tdb
);
694 if (!state
.written
) {
695 tdb_unlockall(cache_notrans
->tdb
);
696 tdb_transaction_cancel(cache
->tdb
);
700 res
= tdb_transaction_commit(cache
->tdb
);
702 DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
703 "%s\n", tdb_errorstr(cache
->tdb
)));
704 tdb_unlockall(cache_notrans
->tdb
);
708 res
= tdb_traverse(cache_notrans
->tdb
, wipe_fn
, NULL
);
710 DEBUG(10, ("tdb_traverse with wipe_fn on gencache_notrans.tdb "
712 tdb_errorstr(cache_notrans
->tdb
)));
713 tdb_unlockall(cache_notrans
->tdb
);
717 res
= tdb_unlockall(cache_notrans
->tdb
);
719 DEBUG(10, ("tdb_unlockall on gencache.tdb failed: "
720 "%s\n", tdb_errorstr(cache
->tdb
)));
724 now
= talloc_asprintf(talloc_tos(), "%d", (int)time(NULL
));
726 tdb_store(cache_notrans
->tdb
, last_stabilize_key(),
727 string_term_tdb_data(now
), 0);
734 static int stabilize_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
737 struct stabilize_state
*state
= (struct stabilize_state
*)priv
;
741 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
745 if (!gencache_pull_timeout((char *)val
.dptr
, &timeout
, NULL
)) {
746 DEBUG(10, ("Ignoring invalid entry\n"));
749 if ((timeout
< time(NULL
)) || (val
.dsize
== 0)) {
750 res
= tdb_delete(cache
->tdb
, key
);
752 state
->written
= true;
753 } else if (tdb_error(cache
->tdb
) == TDB_ERR_NOEXIST
) {
757 res
= tdb_store(cache
->tdb
, key
, val
, 0);
759 state
->written
= true;
764 DEBUG(10, ("Transfer to gencache.tdb failed: %s\n",
765 tdb_errorstr(cache
->tdb
)));
772 static int wipe_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
779 res
= tdb_data_cmp(key
, last_stabilize_key());
784 ok
= gencache_pull_timeout((char *)val
.dptr
, &timeout
, NULL
);
786 DEBUG(10, ("Ignoring invalid entry\n"));
790 res
= tdb_delete(tdb
, key
);
792 DEBUG(10, ("tdb_delete from gencache_notrans.tdb failed: "
793 "%s\n", tdb_errorstr(cache_notrans
->tdb
)));
802 * Get existing entry from the cache file.
804 * @param keystr string that represents a key of this entry
805 * @param valstr buffer that is allocated and filled with the entry value
806 * buffer's disposing must be done outside
807 * @param timeout pointer to a time_t that is filled with entry's
810 * @retval true when entry is successfuly fetched
811 * @retval false for failure
814 bool gencache_get(const char *keystr
, TALLOC_CTX
*mem_ctx
, char **value
,
820 ret
= gencache_get_data_blob(keystr
, mem_ctx
, &blob
, ptimeout
, NULL
);
824 if ((blob
.data
== NULL
) || (blob
.length
== 0)) {
825 data_blob_free(&blob
);
828 if (blob
.data
[blob
.length
-1] != '\0') {
829 /* Not NULL terminated, can't be a string */
830 data_blob_free(&blob
);
835 * talloc_move generates a type-punned warning here. As we
836 * leave the function immediately, do a simple talloc_steal.
838 *value
= (char *)talloc_steal(mem_ctx
, blob
.data
);
841 data_blob_free(&blob
);
846 * Set an entry in the cache file. If there's no such
849 * @param keystr string that represents a key of this entry
850 * @param value text representation value being cached
851 * @param timeout time when the value is expired
853 * @retval true when entry is successfuly stored
854 * @retval false on failure
857 bool gencache_set(const char *keystr
, const char *value
, time_t timeout
)
859 DATA_BLOB blob
= data_blob_const(value
, strlen(value
)+1);
860 return gencache_set_data_blob(keystr
, &blob
, timeout
);
863 struct gencache_iterate_blobs_state
{
864 void (*fn
)(const char *key
, DATA_BLOB value
,
865 time_t timeout
, void *private_data
);
871 static int gencache_iterate_blobs_fn(struct tdb_context
*tdb
, TDB_DATA key
,
872 TDB_DATA data
, void *priv
)
874 struct gencache_iterate_blobs_state
*state
=
875 (struct gencache_iterate_blobs_state
*)priv
;
877 char *free_key
= NULL
;
881 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
884 if (state
->in_persistent
&& tdb_exists(cache_notrans
->tdb
, key
)) {
888 if (key
.dptr
[key
.dsize
-1] == '\0') {
889 keystr
= (char *)key
.dptr
;
891 /* ensure 0-termination */
892 keystr
= talloc_strndup(talloc_tos(), (char *)key
.dptr
, key
.dsize
);
894 if (keystr
== NULL
) {
899 if (!gencache_pull_timeout((char *)data
.dptr
, &timeout
, &endptr
)) {
909 if (fnmatch(state
->pattern
, keystr
, 0) != 0) {
913 DEBUG(10, ("Calling function with arguments "
914 "(key=[%s], timeout=[%s])\n",
915 keystr
, timestring(talloc_tos(), timeout
)));
918 data_blob_const(endptr
,
919 data
.dsize
- PTR_DIFF(endptr
, data
.dptr
)),
920 timeout
, state
->private_data
);
923 TALLOC_FREE(free_key
);
927 void gencache_iterate_blobs(void (*fn
)(const char *key
, DATA_BLOB value
,
928 time_t timeout
, void *private_data
),
929 void *private_data
, const char *pattern
)
931 struct gencache_iterate_blobs_state state
;
933 if ((fn
== NULL
) || (pattern
== NULL
) || !gencache_init()) {
937 DEBUG(5, ("Searching cache keys with pattern %s\n", pattern
));
940 state
.pattern
= pattern
;
941 state
.private_data
= private_data
;
943 state
.in_persistent
= false;
944 tdb_traverse(cache_notrans
->tdb
, gencache_iterate_blobs_fn
, &state
);
946 state
.in_persistent
= true;
947 tdb_traverse(cache
->tdb
, gencache_iterate_blobs_fn
, &state
);
951 * Iterate through all entries which key matches to specified pattern
953 * @param fn pointer to the function that will be supplied with each single
954 * matching cache entry (key, value and timeout) as an arguments
955 * @param data void pointer to an arbitrary data that is passed directly to the fn
956 * function on each call
957 * @param keystr_pattern pattern the existing entries' keys are matched to
961 struct gencache_iterate_state
{
962 void (*fn
)(const char *key
, const char *value
, time_t timeout
,
967 static void gencache_iterate_fn(const char *key
, DATA_BLOB value
,
968 time_t timeout
, void *private_data
)
970 struct gencache_iterate_state
*state
=
971 (struct gencache_iterate_state
*)private_data
;
973 char *free_val
= NULL
;
975 if (value
.data
[value
.length
-1] == '\0') {
976 valstr
= (char *)value
.data
;
978 /* ensure 0-termination */
979 valstr
= talloc_strndup(talloc_tos(), (char *)value
.data
, value
.length
);
981 if (valstr
== NULL
) {
986 DEBUG(10, ("Calling function with arguments "
987 "(key=[%s], value=[%s], timeout=[%s])\n",
988 key
, valstr
, timestring(talloc_tos(), timeout
)));
990 state
->fn(key
, valstr
, timeout
, state
->private_data
);
994 TALLOC_FREE(free_val
);
997 void gencache_iterate(void (*fn
)(const char *key
, const char *value
,
998 time_t timeout
, void *dptr
),
999 void *private_data
, const char *pattern
)
1001 struct gencache_iterate_state state
;
1007 state
.private_data
= private_data
;
1008 gencache_iterate_blobs(gencache_iterate_fn
, &state
, pattern
);