2 Unix SMB/CIFS implementation.
4 Generic, persistent and shared between processes cache mechanism for use
5 by various parts of the Samba code
7 Copyright (C) Rafal Szczesniak 2002
8 Copyright (C) Volker Lendecke 2009
10 This program is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3 of the License, or
13 (at your option) any later version.
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
20 You should have received a copy of the GNU General Public License
21 along with this program. If not, see <http://www.gnu.org/licenses/>.
25 #include "system/filesys.h"
26 #include "system/glob.h"
28 #include "tdb_wrap/tdb_wrap.h"
29 #include "../lib/util/memcache.h"
32 #define DBGC_CLASS DBGC_TDB
34 #define TIMEOUT_LEN 12
35 #define CACHE_DATA_FMT "%12u/"
36 #define READ_CACHE_DATA_FMT_TEMPLATE "%%12u/%%%us"
37 #define BLOB_TYPE "DATA_BLOB"
38 #define BLOB_TYPE_LEN 9
40 static struct tdb_wrap
*cache
;
41 static struct tdb_wrap
*cache_notrans
;
42 static int cache_notrans_seqnum
;
46 * @brief Generic, persistent and shared between processes cache mechanism
47 * for use by various parts of the Samba code
53 * Cache initialisation function. Opens cache tdb file or creates
54 * it if does not exist.
56 * @return true on successful initialisation of the cache or
60 static bool gencache_init(void)
62 char* cache_fname
= NULL
;
63 int open_flags
= O_RDWR
|O_CREAT
;
65 /* skip file open if it's already opened */
70 cache_fname
= cache_path("gencache.tdb");
71 if (cache_fname
== NULL
) {
75 DEBUG(5, ("Opening cache file at %s\n", cache_fname
));
77 cache
= tdb_wrap_open(NULL
, cache_fname
, 0,
78 TDB_DEFAULT
|TDB_INCOMPATIBLE_HASH
,
82 ret
= tdb_check(cache
->tdb
, NULL
, NULL
);
87 * Retry with CLEAR_IF_FIRST.
89 * Warning: Converting this to dbwrap won't work
90 * directly. gencache.c does transactions on this tdb,
91 * and dbwrap forbids this for CLEAR_IF_FIRST
92 * databases. tdb does allow transactions on
93 * CLEAR_IF_FIRST databases, so lets use it here to
94 * clean up a broken database.
96 cache
= tdb_wrap_open(NULL
, cache_fname
, 0,
98 TDB_INCOMPATIBLE_HASH
|
104 if (!cache
&& (errno
== EACCES
)) {
105 open_flags
= O_RDONLY
;
106 cache
= tdb_wrap_open(NULL
, cache_fname
, 0,
107 TDB_DEFAULT
|TDB_INCOMPATIBLE_HASH
,
110 DEBUG(5, ("gencache_init: Opening cache file %s read-only.\n", cache_fname
));
113 TALLOC_FREE(cache_fname
);
116 DEBUG(5, ("Attempt to open gencache.tdb has failed.\n"));
120 cache_fname
= lock_path("gencache_notrans.tdb");
121 if (cache_fname
== NULL
) {
126 DEBUG(5, ("Opening cache file at %s\n", cache_fname
));
128 cache_notrans
= tdb_wrap_open(NULL
, cache_fname
, 0,
130 TDB_INCOMPATIBLE_HASH
|
135 if (cache_notrans
== NULL
) {
136 DEBUG(5, ("Opening %s failed: %s\n", cache_fname
,
138 TALLOC_FREE(cache_fname
);
142 TALLOC_FREE(cache_fname
);
147 static TDB_DATA
last_stabilize_key(void)
150 result
.dptr
= discard_const_p(uint8_t, "@LAST_STABILIZED");
155 struct gencache_have_val_state
{
157 const DATA_BLOB
*data
;
161 static void gencache_have_val_parser(time_t old_timeout
, DATA_BLOB data
,
164 struct gencache_have_val_state
*state
=
165 (struct gencache_have_val_state
*)private_data
;
166 time_t now
= time(NULL
);
167 int cache_time_left
, new_time_left
, additional_time
;
170 * Excuse the many variables, but these time calculations are
171 * confusing to me. We do not want to write to gencache with a
172 * possibly expensive transaction if we are about to write the same
173 * value, just extending the remaining timeout by less than 10%.
176 cache_time_left
= old_timeout
- now
;
177 if (cache_time_left
<= 0) {
179 * timed out, write new value
184 new_time_left
= state
->new_timeout
- now
;
185 if (new_time_left
<= 0) {
187 * Huh -- no new timeout?? Write it.
192 if (new_time_left
< cache_time_left
) {
194 * Someone wants to shorten the timeout. Let it happen.
200 * By how much does the new timeout extend the remaining cache time?
202 additional_time
= new_time_left
- cache_time_left
;
204 if (additional_time
* 10 < 0) {
206 * Integer overflow. We extend by so much that we have to write it.
212 * The comparison below is essentially equivalent to
214 * new_time_left > cache_time_left * 1.10
216 * but without floating point calculations.
219 if (additional_time
* 10 > cache_time_left
) {
221 * We extend the cache timeout by more than 10%. Do it.
227 * Now the more expensive data compare.
229 if (data_blob_cmp(state
->data
, &data
) != 0) {
231 * Write a new value. Certainly do it.
237 * Extending the timeout by less than 10% for the same cache value is
238 * not worth the trouble writing a value into gencache under a
239 * possibly expensive transaction.
244 static bool gencache_have_val(const char *keystr
, const DATA_BLOB
*data
,
247 struct gencache_have_val_state state
;
249 state
.new_timeout
= timeout
;
253 if (!gencache_parse(keystr
, gencache_have_val_parser
, &state
)) {
259 static int last_stabilize_parser(TDB_DATA key
, TDB_DATA data
,
262 time_t *last_stabilize
= private_data
;
264 if ((data
.dsize
!= 0) && (data
.dptr
[data
.dsize
-1] == '\0')) {
265 *last_stabilize
= atoi((char *)data
.dptr
);
271 * Set an entry in the cache file. If there's no such
274 * @param keystr string that represents a key of this entry
275 * @param blob DATA_BLOB value being cached
276 * @param timeout time when the value is expired
278 * @retval true when entry is successfully stored
279 * @retval false on failure
282 bool gencache_set_data_blob(const char *keystr
, const DATA_BLOB
*blob
,
289 time_t last_stabilize
;
290 static int writecount
;
292 if (tdb_data_cmp(string_term_tdb_data(keystr
),
293 last_stabilize_key()) == 0) {
294 DEBUG(10, ("Can't store %s as a key\n", keystr
));
298 if ((keystr
== NULL
) || (blob
== NULL
)) {
302 if (!gencache_init()) {
306 if (gencache_have_val(keystr
, blob
, timeout
)) {
307 DEBUG(10, ("Did not store value for %s, we already got it\n",
312 hdr_len
= fstr_sprintf(hdr
, CACHE_DATA_FMT
, (int)timeout
);
317 if ((blob
->length
+ (size_t)hdr_len
) < blob
->length
) {
321 val
= talloc_array(talloc_tos(), char, hdr_len
+ blob
->length
);
326 memcpy(val
, hdr
, hdr_len
);
327 memcpy(val
+hdr_len
, blob
->data
, blob
->length
);
329 DEBUG(10, ("Adding cache entry with key=[%s] and timeout="
330 "[%s] (%d seconds %s)\n", keystr
,
331 timestring(talloc_tos(), timeout
),
332 (int)(timeout
- time(NULL
)),
333 timeout
> time(NULL
) ? "ahead" : "in the past"));
335 ret
= tdb_store_bystring(
336 cache_notrans
->tdb
, keystr
,
337 make_tdb_data((uint8_t *)val
, talloc_array_length(val
)),
346 * Every 100 writes within a single process, stabilize the cache with
347 * a transaction. This is done to prevent a single transaction to
348 * become huge and chew lots of memory.
351 if (writecount
> lp_parm_int(-1, "gencache", "stabilize_count", 100)) {
352 gencache_stabilize();
358 * Every 5 minutes, call gencache_stabilize() to not let grow
359 * gencache_notrans.tdb too large.
364 tdb_parse_record(cache_notrans
->tdb
, last_stabilize_key(),
365 last_stabilize_parser
, &last_stabilize
);
368 + lp_parm_int(-1, "gencache", "stabilize_interval", 300))
370 gencache_stabilize();
378 * Delete one entry from the cache file.
380 * @param keystr string that represents a key of this entry
382 * @retval true upon successful deletion
383 * @retval false in case of failure
386 bool gencache_del(const char *keystr
)
388 bool exists
, was_expired
;
392 if (keystr
== NULL
) {
396 if (!gencache_init()) {
400 DEBUG(10, ("Deleting cache entry (key=[%s])\n", keystr
));
403 * We delete an element by setting its timeout to 0. This way we don't
404 * have to do a transaction on gencache.tdb every time we delete an
408 exists
= gencache_get_data_blob(keystr
, NULL
, &value
, NULL
,
411 if (!exists
&& was_expired
) {
413 * gencache_get_data_blob has implicitly deleted this
414 * entry, so we have to return success here.
420 data_blob_free(&value
);
421 ret
= gencache_set(keystr
, "", 0);
426 static bool gencache_pull_timeout(char *val
, time_t *pres
, char **pendptr
)
435 res
= strtol(val
, &endptr
, 10);
437 if ((endptr
== NULL
) || (*endptr
!= '/')) {
438 DEBUG(2, ("Invalid gencache data format: %s\n", val
));
444 if (pendptr
!= NULL
) {
450 struct gencache_parse_state
{
451 void (*parser
)(time_t timeout
, DATA_BLOB blob
, void *private_data
);
456 static int gencache_parse_fn(TDB_DATA key
, TDB_DATA data
, void *private_data
)
458 struct gencache_parse_state
*state
;
464 if (data
.dptr
== NULL
) {
467 ret
= gencache_pull_timeout((char *)data
.dptr
, &t
, &endptr
);
471 state
= (struct gencache_parse_state
*)private_data
;
472 blob
= data_blob_const(
473 endptr
+1, data
.dsize
- PTR_DIFF(endptr
+1, data
.dptr
));
474 state
->parser(t
, blob
, state
->private_data
);
476 if (!state
->is_memcache
) {
477 memcache_add(NULL
, GENCACHE_RAM
,
478 data_blob_const(key
.dptr
, key
.dsize
),
479 data_blob_const(data
.dptr
, data
.dsize
));
485 bool gencache_parse(const char *keystr
,
486 void (*parser
)(time_t timeout
, DATA_BLOB blob
,
490 struct gencache_parse_state state
;
491 TDB_DATA key
= string_term_tdb_data(keystr
);
492 DATA_BLOB memcache_val
;
495 if (keystr
== NULL
) {
498 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
501 if (!gencache_init()) {
505 state
.parser
= parser
;
506 state
.private_data
= private_data
;
508 if (memcache_lookup(NULL
, GENCACHE_RAM
,
509 data_blob_const(key
.dptr
, key
.dsize
),
512 * Make sure that nobody has changed the gencache behind our
515 int current_seqnum
= tdb_get_seqnum(cache_notrans
->tdb
);
516 if (current_seqnum
== cache_notrans_seqnum
) {
518 * Ok, our memcache is still current, use it without
519 * going to the tdb files.
521 state
.is_memcache
= true;
522 gencache_parse_fn(key
, make_tdb_data(memcache_val
.data
,
523 memcache_val
.length
),
527 memcache_flush(NULL
, GENCACHE_RAM
);
528 cache_notrans_seqnum
= current_seqnum
;
531 state
.is_memcache
= false;
533 ret
= tdb_parse_record(cache_notrans
->tdb
, key
,
534 gencache_parse_fn
, &state
);
538 ret
= tdb_parse_record(cache
->tdb
, key
, gencache_parse_fn
, &state
);
542 struct gencache_get_data_blob_state
{
549 static void gencache_get_data_blob_parser(time_t timeout
, DATA_BLOB blob
,
552 struct gencache_get_data_blob_state
*state
=
553 (struct gencache_get_data_blob_state
*)private_data
;
556 state
->result
= false;
559 state
->timeout
= timeout
;
561 if (state
->blob
== NULL
) {
562 state
->result
= true;
566 *state
->blob
= data_blob_talloc(state
->mem_ctx
, blob
.data
,
568 if (state
->blob
->data
== NULL
) {
569 state
->result
= false;
572 state
->result
= true;
576 * Get existing entry from the cache file.
578 * @param keystr string that represents a key of this entry
579 * @param blob DATA_BLOB that is filled with entry's blob
580 * @param timeout pointer to a time_t that is filled with entry's
583 * @retval true when entry is successfuly fetched
584 * @retval false for failure
587 bool gencache_get_data_blob(const char *keystr
, TALLOC_CTX
*mem_ctx
,
589 time_t *timeout
, bool *was_expired
)
591 struct gencache_get_data_blob_state state
;
592 bool expired
= false;
594 state
.result
= false;
595 state
.mem_ctx
= mem_ctx
;
598 if (!gencache_parse(keystr
, gencache_get_data_blob_parser
, &state
)) {
604 if (state
.timeout
<= time(NULL
)) {
606 * We're expired, delete the entry. We can't use gencache_del
607 * here, because that uses gencache_get_data_blob for checking
608 * the existence of a record. We know the thing exists and
609 * directly store an empty value with 0 timeout.
611 gencache_set(keystr
, "", 0);
616 *timeout
= state
.timeout
;
622 if (was_expired
!= NULL
) {
623 *was_expired
= expired
;
625 if (state
.result
&& state
.blob
) {
626 data_blob_free(state
.blob
);
631 struct stabilize_state
{
634 static int stabilize_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
637 static int wipe_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
643 * Migrate the clear-if-first gencache data to the stable,
644 * transaction-based gencache.tdb
647 bool gencache_stabilize(void)
649 struct stabilize_state state
;
653 if (!gencache_init()) {
657 res
= tdb_transaction_start_nonblock(cache
->tdb
);
659 if (tdb_error(cache
->tdb
) == TDB_ERR_NOLOCK
)
662 * Someone else already does the stabilize,
663 * this does not have to be done twice
668 DEBUG(10, ("Could not start transaction on gencache.tdb: "
669 "%s\n", tdb_errorstr(cache
->tdb
)));
673 res
= tdb_lockall(cache_notrans
->tdb
);
675 tdb_transaction_cancel(cache
->tdb
);
676 DEBUG(10, ("Could not get allrecord lock on "
677 "gencache_notrans.tdb: %s\n",
678 tdb_errorstr(cache_notrans
->tdb
)));
682 state
.written
= false;
684 res
= tdb_traverse(cache_notrans
->tdb
, stabilize_fn
, &state
);
686 tdb_unlockall(cache_notrans
->tdb
);
687 tdb_transaction_cancel(cache
->tdb
);
691 if (!state
.written
) {
692 tdb_unlockall(cache_notrans
->tdb
);
693 tdb_transaction_cancel(cache
->tdb
);
697 res
= tdb_transaction_commit(cache
->tdb
);
699 DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
700 "%s\n", tdb_errorstr(cache
->tdb
)));
701 tdb_unlockall(cache_notrans
->tdb
);
705 res
= tdb_traverse(cache_notrans
->tdb
, wipe_fn
, NULL
);
707 DEBUG(10, ("tdb_traverse with wipe_fn on gencache_notrans.tdb "
709 tdb_errorstr(cache_notrans
->tdb
)));
710 tdb_unlockall(cache_notrans
->tdb
);
714 res
= tdb_unlockall(cache_notrans
->tdb
);
716 DEBUG(10, ("tdb_unlockall on gencache.tdb failed: "
717 "%s\n", tdb_errorstr(cache
->tdb
)));
721 now
= talloc_asprintf(talloc_tos(), "%d", (int)time(NULL
));
723 tdb_store(cache_notrans
->tdb
, last_stabilize_key(),
724 string_term_tdb_data(now
), 0);
731 static int stabilize_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
734 struct stabilize_state
*state
= (struct stabilize_state
*)priv
;
738 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
742 if (!gencache_pull_timeout((char *)val
.dptr
, &timeout
, NULL
)) {
743 DEBUG(10, ("Ignoring invalid entry\n"));
746 if ((timeout
< time(NULL
)) || (val
.dsize
== 0)) {
747 res
= tdb_delete(cache
->tdb
, key
);
749 state
->written
= true;
750 } else if (tdb_error(cache
->tdb
) == TDB_ERR_NOEXIST
) {
754 res
= tdb_store(cache
->tdb
, key
, val
, 0);
756 state
->written
= true;
761 DEBUG(10, ("Transfer to gencache.tdb failed: %s\n",
762 tdb_errorstr(cache
->tdb
)));
769 static int wipe_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
776 res
= tdb_data_cmp(key
, last_stabilize_key());
781 ok
= gencache_pull_timeout((char *)val
.dptr
, &timeout
, NULL
);
783 DEBUG(10, ("Ignoring invalid entry\n"));
787 res
= tdb_delete(tdb
, key
);
789 DEBUG(10, ("tdb_delete from gencache_notrans.tdb failed: "
790 "%s\n", tdb_errorstr(cache_notrans
->tdb
)));
799 * Get existing entry from the cache file.
801 * @param keystr string that represents a key of this entry
802 * @param valstr buffer that is allocated and filled with the entry value
803 * buffer's disposing must be done outside
804 * @param timeout pointer to a time_t that is filled with entry's
807 * @retval true when entry is successfuly fetched
808 * @retval false for failure
811 bool gencache_get(const char *keystr
, TALLOC_CTX
*mem_ctx
, char **value
,
817 ret
= gencache_get_data_blob(keystr
, mem_ctx
, &blob
, ptimeout
, NULL
);
821 if ((blob
.data
== NULL
) || (blob
.length
== 0)) {
822 data_blob_free(&blob
);
825 if (blob
.data
[blob
.length
-1] != '\0') {
826 /* Not NULL terminated, can't be a string */
827 data_blob_free(&blob
);
832 * talloc_move generates a type-punned warning here. As we
833 * leave the function immediately, do a simple talloc_steal.
835 *value
= (char *)talloc_steal(mem_ctx
, blob
.data
);
838 data_blob_free(&blob
);
843 * Set an entry in the cache file. If there's no such
846 * @param keystr string that represents a key of this entry
847 * @param value text representation value being cached
848 * @param timeout time when the value is expired
850 * @retval true when entry is successfuly stored
851 * @retval false on failure
854 bool gencache_set(const char *keystr
, const char *value
, time_t timeout
)
856 DATA_BLOB blob
= data_blob_const(value
, strlen(value
)+1);
857 return gencache_set_data_blob(keystr
, &blob
, timeout
);
860 struct gencache_iterate_blobs_state
{
861 void (*fn
)(const char *key
, DATA_BLOB value
,
862 time_t timeout
, void *private_data
);
868 static int gencache_iterate_blobs_fn(struct tdb_context
*tdb
, TDB_DATA key
,
869 TDB_DATA data
, void *priv
)
871 struct gencache_iterate_blobs_state
*state
=
872 (struct gencache_iterate_blobs_state
*)priv
;
874 char *free_key
= NULL
;
878 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
881 if (state
->in_persistent
&& tdb_exists(cache_notrans
->tdb
, key
)) {
885 if (key
.dptr
[key
.dsize
-1] == '\0') {
886 keystr
= (char *)key
.dptr
;
888 /* ensure 0-termination */
889 keystr
= talloc_strndup(talloc_tos(), (char *)key
.dptr
, key
.dsize
);
891 if (keystr
== NULL
) {
896 if (!gencache_pull_timeout((char *)data
.dptr
, &timeout
, &endptr
)) {
901 if (fnmatch(state
->pattern
, keystr
, 0) != 0) {
905 DEBUG(10, ("Calling function with arguments "
906 "(key=[%s], timeout=[%s])\n",
907 keystr
, timestring(talloc_tos(), timeout
)));
910 data_blob_const(endptr
,
911 data
.dsize
- PTR_DIFF(endptr
, data
.dptr
)),
912 timeout
, state
->private_data
);
915 TALLOC_FREE(free_key
);
919 void gencache_iterate_blobs(void (*fn
)(const char *key
, DATA_BLOB value
,
920 time_t timeout
, void *private_data
),
921 void *private_data
, const char *pattern
)
923 struct gencache_iterate_blobs_state state
;
925 if ((fn
== NULL
) || (pattern
== NULL
) || !gencache_init()) {
929 DEBUG(5, ("Searching cache keys with pattern %s\n", pattern
));
932 state
.pattern
= pattern
;
933 state
.private_data
= private_data
;
935 state
.in_persistent
= false;
936 tdb_traverse(cache_notrans
->tdb
, gencache_iterate_blobs_fn
, &state
);
938 state
.in_persistent
= true;
939 tdb_traverse(cache
->tdb
, gencache_iterate_blobs_fn
, &state
);
943 * Iterate through all entries which key matches to specified pattern
945 * @param fn pointer to the function that will be supplied with each single
946 * matching cache entry (key, value and timeout) as an arguments
947 * @param data void pointer to an arbitrary data that is passed directly to the fn
948 * function on each call
949 * @param keystr_pattern pattern the existing entries' keys are matched to
953 struct gencache_iterate_state
{
954 void (*fn
)(const char *key
, const char *value
, time_t timeout
,
959 static void gencache_iterate_fn(const char *key
, DATA_BLOB value
,
960 time_t timeout
, void *private_data
)
962 struct gencache_iterate_state
*state
=
963 (struct gencache_iterate_state
*)private_data
;
965 char *free_val
= NULL
;
967 if (value
.data
[value
.length
-1] == '\0') {
968 valstr
= (char *)value
.data
;
970 /* ensure 0-termination */
971 valstr
= talloc_strndup(talloc_tos(), (char *)value
.data
, value
.length
);
973 if (valstr
== NULL
) {
978 DEBUG(10, ("Calling function with arguments "
979 "(key=[%s], value=[%s], timeout=[%s])\n",
980 key
, valstr
, timestring(talloc_tos(), timeout
)));
982 state
->fn(key
, valstr
, timeout
, state
->private_data
);
986 TALLOC_FREE(free_val
);
989 void gencache_iterate(void (*fn
)(const char *key
, const char *value
,
990 time_t timeout
, void *dptr
),
991 void *private_data
, const char *pattern
)
993 struct gencache_iterate_state state
;
999 state
.private_data
= private_data
;
1000 gencache_iterate_blobs(gencache_iterate_fn
, &state
, pattern
);