2 Unix SMB/CIFS implementation.
4 Generic, persistent and shared between processes cache mechanism for use
5 by various parts of the Samba code
7 Copyright (C) Rafal Szczesniak 2002
8 Copyright (C) Volker Lendecke 2009
10 This program is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3 of the License, or
13 (at your option) any later version.
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
20 You should have received a copy of the GNU General Public License
21 along with this program. If not, see <http://www.gnu.org/licenses/>.
25 #include "system/filesys.h"
26 #include "system/glob.h"
31 #define DBGC_CLASS DBGC_TDB
33 #define TIMEOUT_LEN 12
34 #define CACHE_DATA_FMT "%12u/"
35 #define READ_CACHE_DATA_FMT_TEMPLATE "%%12u/%%%us"
36 #define BLOB_TYPE "DATA_BLOB"
37 #define BLOB_TYPE_LEN 9
39 static struct tdb_context
*cache
;
40 static struct tdb_context
*cache_notrans
;
41 static int cache_notrans_seqnum
;
45 * @brief Generic, persistent and shared between processes cache mechanism
46 * for use by various parts of the Samba code
52 * Cache initialisation function. Opens cache tdb file or creates
53 * it if does not exist.
55 * @return true on successful initialisation of the cache or
59 static bool gencache_init(void)
61 char* cache_fname
= NULL
;
62 int open_flags
= O_RDWR
|O_CREAT
;
64 /* skip file open if it's already opened */
65 if (cache
) return True
;
67 cache_fname
= cache_path("gencache.tdb");
69 DEBUG(5, ("Opening cache file at %s\n", cache_fname
));
71 cache
= tdb_open_log(cache_fname
, 0, TDB_DEFAULT
|TDB_INCOMPATIBLE_HASH
, open_flags
, 0644);
74 ret
= tdb_check(cache
, NULL
, NULL
);
79 * Retry with CLEAR_IF_FIRST.
81 * Warning: Converting this to dbwrap won't work
82 * directly. gencache.c does transactions on this tdb,
83 * and dbwrap forbids this for CLEAR_IF_FIRST
84 * databases. tdb does allow transactions on
85 * CLEAR_IF_FIRST databases, so lets use it here to
86 * clean up a broken database.
88 cache
= tdb_open_log(cache_fname
, 0,
90 TDB_INCOMPATIBLE_HASH
|
96 if (!cache
&& (errno
== EACCES
)) {
97 open_flags
= O_RDONLY
;
98 cache
= tdb_open_log(cache_fname
, 0, TDB_DEFAULT
|TDB_INCOMPATIBLE_HASH
, open_flags
,
101 DEBUG(5, ("gencache_init: Opening cache file %s read-only.\n", cache_fname
));
106 DEBUG(5, ("Attempt to open gencache.tdb has failed.\n"));
110 cache_fname
= lock_path("gencache_notrans.tdb");
112 DEBUG(5, ("Opening cache file at %s\n", cache_fname
));
114 cache_notrans
= tdb_open_log(cache_fname
, 0,
116 TDB_INCOMPATIBLE_HASH
|
120 if (cache_notrans
== NULL
) {
121 DEBUG(5, ("Opening %s failed: %s\n", cache_fname
,
131 static TDB_DATA
last_stabilize_key(void)
134 result
.dptr
= discard_const_p(uint8_t, "@LAST_STABILIZED");
139 struct gencache_have_val_state
{
141 const DATA_BLOB
*data
;
145 static void gencache_have_val_parser(time_t old_timeout
, DATA_BLOB data
,
148 struct gencache_have_val_state
*state
=
149 (struct gencache_have_val_state
*)private_data
;
150 time_t now
= time(NULL
);
151 int cache_time_left
, new_time_left
, additional_time
;
154 * Excuse the many variables, but these time calculations are
155 * confusing to me. We do not want to write to gencache with a
156 * possibly expensive transaction if we are about to write the same
157 * value, just extending the remaining timeout by less than 10%.
160 cache_time_left
= old_timeout
- now
;
161 if (cache_time_left
<= 0) {
163 * timed out, write new value
168 new_time_left
= state
->new_timeout
- now
;
169 if (new_time_left
<= 0) {
171 * Huh -- no new timeout?? Write it.
176 if (new_time_left
< cache_time_left
) {
178 * Someone wants to shorten the timeout. Let it happen.
184 * By how much does the new timeout extend the remaining cache time?
186 additional_time
= new_time_left
- cache_time_left
;
188 if (additional_time
* 10 < 0) {
190 * Integer overflow. We extend by so much that we have to write it.
196 * The comparison below is essentially equivalent to
198 * new_time_left > cache_time_left * 1.10
200 * but without floating point calculations.
203 if (additional_time
* 10 > cache_time_left
) {
205 * We extend the cache timeout by more than 10%. Do it.
211 * Now the more expensive data compare.
213 if (data_blob_cmp(state
->data
, &data
) != 0) {
215 * Write a new value. Certainly do it.
221 * Extending the timeout by less than 10% for the same cache value is
222 * not worth the trouble writing a value into gencache under a
223 * possibly expensive transaction.
228 static bool gencache_have_val(const char *keystr
, const DATA_BLOB
*data
,
231 struct gencache_have_val_state state
;
233 state
.new_timeout
= timeout
;
237 if (!gencache_parse(keystr
, gencache_have_val_parser
, &state
)) {
244 * Set an entry in the cache file. If there's no such
247 * @param keystr string that represents a key of this entry
248 * @param blob DATA_BLOB value being cached
249 * @param timeout time when the value is expired
251 * @retval true when entry is successfuly stored
252 * @retval false on failure
255 bool gencache_set_data_blob(const char *keystr
, const DATA_BLOB
*blob
,
261 time_t last_stabilize
;
262 static int writecount
;
264 if (tdb_data_cmp(string_term_tdb_data(keystr
),
265 last_stabilize_key()) == 0) {
266 DEBUG(10, ("Can't store %s as a key\n", keystr
));
270 if ((keystr
== NULL
) || (blob
== NULL
)) {
274 if (!gencache_init()) return False
;
276 if (gencache_have_val(keystr
, blob
, timeout
)) {
277 DEBUG(10, ("Did not store value for %s, we already got it\n",
282 val
= talloc_asprintf(talloc_tos(), CACHE_DATA_FMT
, (int)timeout
);
286 val
= talloc_realloc(NULL
, val
, char, talloc_array_length(val
)-1);
290 val
= (char *)talloc_append_blob(NULL
, val
, *blob
);
295 DEBUG(10, ("Adding cache entry with key=[%s] and timeout="
296 "[%s] (%d seconds %s)\n", keystr
,
297 timestring(talloc_tos(), timeout
),
298 (int)(timeout
- time(NULL
)),
299 timeout
> time(NULL
) ? "ahead" : "in the past"));
301 ret
= tdb_store_bystring(
302 cache_notrans
, keystr
,
303 make_tdb_data((uint8_t *)val
, talloc_array_length(val
)),
312 * Every 100 writes within a single process, stabilize the cache with
313 * a transaction. This is done to prevent a single transaction to
314 * become huge and chew lots of memory.
317 if (writecount
> lp_parm_int(-1, "gencache", "stabilize_count", 100)) {
318 gencache_stabilize();
324 * Every 5 minutes, call gencache_stabilize() to not let grow
325 * gencache_notrans.tdb too large.
329 databuf
= tdb_fetch_compat(cache_notrans
, last_stabilize_key());
330 if ((databuf
.dptr
!= NULL
)
331 && (databuf
.dptr
[databuf
.dsize
-1] == '\0')) {
332 last_stabilize
= atoi((char *)databuf
.dptr
);
333 SAFE_FREE(databuf
.dptr
);
336 + lp_parm_int(-1, "gencache", "stabilize_interval", 300))
338 gencache_stabilize();
346 * Delete one entry from the cache file.
348 * @param keystr string that represents a key of this entry
350 * @retval true upon successful deletion
351 * @retval false in case of failure
354 bool gencache_del(const char *keystr
)
356 bool exists
, was_expired
;
360 if (keystr
== NULL
) {
364 if (!gencache_init()) return False
;
366 DEBUG(10, ("Deleting cache entry (key=[%s])\n", keystr
));
369 * We delete an element by setting its timeout to 0. This way we don't
370 * have to do a transaction on gencache.tdb every time we delete an
374 exists
= gencache_get_data_blob(keystr
, NULL
, &value
, NULL
,
377 if (!exists
&& was_expired
) {
379 * gencache_get_data_blob has implicitly deleted this
380 * entry, so we have to return success here.
386 data_blob_free(&value
);
387 ret
= gencache_set(keystr
, "", 0);
392 static bool gencache_pull_timeout(char *val
, time_t *pres
, char **pendptr
)
401 res
= strtol(val
, &endptr
, 10);
403 if ((endptr
== NULL
) || (*endptr
!= '/')) {
404 DEBUG(2, ("Invalid gencache data format: %s\n", val
));
410 if (pendptr
!= NULL
) {
416 struct gencache_parse_state
{
417 void (*parser
)(time_t timeout
, DATA_BLOB blob
, void *private_data
);
422 static int gencache_parse_fn(TDB_DATA key
, TDB_DATA data
, void *private_data
)
424 struct gencache_parse_state
*state
;
430 if (data
.dptr
== NULL
) {
433 ret
= gencache_pull_timeout((char *)data
.dptr
, &t
, &endptr
);
437 state
= (struct gencache_parse_state
*)private_data
;
438 blob
= data_blob_const(
439 endptr
+1, data
.dsize
- PTR_DIFF(endptr
+1, data
.dptr
));
440 state
->parser(t
, blob
, state
->private_data
);
442 if (!state
->is_memcache
) {
443 memcache_add(NULL
, GENCACHE_RAM
,
444 data_blob_const(key
.dptr
, key
.dsize
),
445 data_blob_const(data
.dptr
, data
.dsize
));
451 bool gencache_parse(const char *keystr
,
452 void (*parser
)(time_t timeout
, DATA_BLOB blob
,
456 struct gencache_parse_state state
;
457 TDB_DATA key
= string_term_tdb_data(keystr
);
458 DATA_BLOB memcache_val
;
461 if (keystr
== NULL
) {
464 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
467 if (!gencache_init()) {
471 state
.parser
= parser
;
472 state
.private_data
= private_data
;
474 if (memcache_lookup(NULL
, GENCACHE_RAM
,
475 data_blob_const(key
.dptr
, key
.dsize
),
478 * Make sure that nobody has changed the gencache behind our
481 int current_seqnum
= tdb_get_seqnum(cache_notrans
);
482 if (current_seqnum
== cache_notrans_seqnum
) {
484 * Ok, our memcache is still current, use it without
485 * going to the tdb files.
487 state
.is_memcache
= true;
488 gencache_parse_fn(key
, make_tdb_data(memcache_val
.data
,
489 memcache_val
.length
),
493 memcache_flush(NULL
, GENCACHE_RAM
);
494 cache_notrans_seqnum
= current_seqnum
;
497 state
.is_memcache
= false;
499 ret
= tdb_parse_record(cache_notrans
, key
, gencache_parse_fn
, &state
);
503 ret
= tdb_parse_record(cache
, key
, gencache_parse_fn
, &state
);
507 struct gencache_get_data_blob_state
{
514 static void gencache_get_data_blob_parser(time_t timeout
, DATA_BLOB blob
,
517 struct gencache_get_data_blob_state
*state
=
518 (struct gencache_get_data_blob_state
*)private_data
;
521 state
->result
= false;
524 state
->timeout
= timeout
;
526 if (state
->blob
== NULL
) {
527 state
->result
= true;
531 *state
->blob
= data_blob_talloc(state
->mem_ctx
, blob
.data
,
533 if (state
->blob
->data
== NULL
) {
534 state
->result
= false;
537 state
->result
= true;
541 * Get existing entry from the cache file.
543 * @param keystr string that represents a key of this entry
544 * @param blob DATA_BLOB that is filled with entry's blob
545 * @param timeout pointer to a time_t that is filled with entry's
548 * @retval true when entry is successfuly fetched
549 * @retval False for failure
552 bool gencache_get_data_blob(const char *keystr
, TALLOC_CTX
*mem_ctx
,
554 time_t *timeout
, bool *was_expired
)
556 struct gencache_get_data_blob_state state
;
557 bool expired
= false;
559 state
.result
= false;
560 state
.mem_ctx
= mem_ctx
;
563 if (!gencache_parse(keystr
, gencache_get_data_blob_parser
, &state
)) {
569 if (state
.timeout
<= time(NULL
)) {
571 * We're expired, delete the entry. We can't use gencache_del
572 * here, because that uses gencache_get_data_blob for checking
573 * the existence of a record. We know the thing exists and
574 * directly store an empty value with 0 timeout.
576 gencache_set(keystr
, "", 0);
581 *timeout
= state
.timeout
;
587 if (was_expired
!= NULL
) {
588 *was_expired
= expired
;
590 if (state
.result
&& state
.blob
) {
591 data_blob_free(state
.blob
);
596 struct stabilize_state
{
600 static int stabilize_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
606 * Migrate the clear-if-first gencache data to the stable,
607 * transaction-based gencache.tdb
610 bool gencache_stabilize(void)
612 struct stabilize_state state
;
616 if (!gencache_init()) {
620 res
= tdb_transaction_start_nonblock(cache
);
622 if (tdb_error(cache
) == TDB_ERR_NOLOCK
)
625 * Someone else already does the stabilize,
626 * this does not have to be done twice
631 DEBUG(10, ("Could not start transaction on gencache.tdb: "
632 "%s\n", tdb_errorstr_compat(cache
)));
635 res
= tdb_transaction_start(cache_notrans
);
637 tdb_transaction_cancel(cache
);
638 DEBUG(10, ("Could not start transaction on "
639 "gencache_notrans.tdb: %s\n",
640 tdb_errorstr_compat(cache_notrans
)));
645 state
.written
= false;
647 res
= tdb_traverse(cache_notrans
, stabilize_fn
, &state
);
648 if ((res
< 0) || state
.error
) {
649 tdb_transaction_cancel(cache_notrans
);
650 tdb_transaction_cancel(cache
);
654 if (!state
.written
) {
655 tdb_transaction_cancel(cache_notrans
);
656 tdb_transaction_cancel(cache
);
660 res
= tdb_transaction_commit(cache
);
662 DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
663 "%s\n", tdb_errorstr_compat(cache
)));
664 tdb_transaction_cancel(cache_notrans
);
668 res
= tdb_transaction_commit(cache_notrans
);
670 DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
671 "%s\n", tdb_errorstr_compat(cache
)));
675 now
= talloc_asprintf(talloc_tos(), "%d", (int)time(NULL
));
677 tdb_store(cache_notrans
, last_stabilize_key(),
678 string_term_tdb_data(now
), 0);
685 static int stabilize_fn(struct tdb_context
*tdb
, TDB_DATA key
, TDB_DATA val
,
688 struct stabilize_state
*state
= (struct stabilize_state
*)priv
;
692 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
696 if (!gencache_pull_timeout((char *)val
.dptr
, &timeout
, NULL
)) {
697 DEBUG(10, ("Ignoring invalid entry\n"));
700 if ((timeout
< time(NULL
)) || (val
.dsize
== 0)) {
701 res
= tdb_delete(cache
, key
);
702 if ((res
!= 0) && (tdb_error(cache
) == TDB_ERR_NOEXIST
)) {
705 state
->written
= true;
708 res
= tdb_store(cache
, key
, val
, 0);
710 state
->written
= true;
715 DEBUG(10, ("Transfer to gencache.tdb failed: %s\n",
716 tdb_errorstr_compat(cache
)));
721 if (tdb_delete(cache_notrans
, key
) != 0) {
722 DEBUG(10, ("tdb_delete from gencache_notrans.tdb failed: "
723 "%s\n", tdb_errorstr_compat(cache_notrans
)));
731 * Get existing entry from the cache file.
733 * @param keystr string that represents a key of this entry
734 * @param valstr buffer that is allocated and filled with the entry value
735 * buffer's disposing must be done outside
736 * @param timeout pointer to a time_t that is filled with entry's
739 * @retval true when entry is successfuly fetched
740 * @retval False for failure
743 bool gencache_get(const char *keystr
, TALLOC_CTX
*mem_ctx
, char **value
,
749 ret
= gencache_get_data_blob(keystr
, mem_ctx
, &blob
, ptimeout
, NULL
);
753 if ((blob
.data
== NULL
) || (blob
.length
== 0)) {
754 data_blob_free(&blob
);
757 if (blob
.data
[blob
.length
-1] != '\0') {
758 /* Not NULL terminated, can't be a string */
759 data_blob_free(&blob
);
764 * talloc_move generates a type-punned warning here. As we
765 * leave the function immediately, do a simple talloc_steal.
767 *value
= (char *)talloc_steal(mem_ctx
, blob
.data
);
770 data_blob_free(&blob
);
775 * Set an entry in the cache file. If there's no such
778 * @param keystr string that represents a key of this entry
779 * @param value text representation value being cached
780 * @param timeout time when the value is expired
782 * @retval true when entry is successfuly stored
783 * @retval false on failure
786 bool gencache_set(const char *keystr
, const char *value
, time_t timeout
)
788 DATA_BLOB blob
= data_blob_const(value
, strlen(value
)+1);
789 return gencache_set_data_blob(keystr
, &blob
, timeout
);
792 struct gencache_iterate_blobs_state
{
793 void (*fn
)(const char *key
, DATA_BLOB value
,
794 time_t timeout
, void *private_data
);
800 static int gencache_iterate_blobs_fn(struct tdb_context
*tdb
, TDB_DATA key
,
801 TDB_DATA data
, void *priv
)
803 struct gencache_iterate_blobs_state
*state
=
804 (struct gencache_iterate_blobs_state
*)priv
;
806 char *free_key
= NULL
;
810 if (tdb_data_cmp(key
, last_stabilize_key()) == 0) {
813 if (state
->in_persistent
&& tdb_exists(cache_notrans
, key
)) {
817 if (key
.dptr
[key
.dsize
-1] == '\0') {
818 keystr
= (char *)key
.dptr
;
820 /* ensure 0-termination */
821 keystr
= talloc_strndup(talloc_tos(), (char *)key
.dptr
, key
.dsize
);
823 if (keystr
== NULL
) {
828 if (!gencache_pull_timeout((char *)data
.dptr
, &timeout
, &endptr
)) {
833 if (fnmatch(state
->pattern
, keystr
, 0) != 0) {
837 DEBUG(10, ("Calling function with arguments "
838 "(key=[%s], timeout=[%s])\n",
839 keystr
, timestring(talloc_tos(), timeout
)));
842 data_blob_const(endptr
,
843 data
.dsize
- PTR_DIFF(endptr
, data
.dptr
)),
844 timeout
, state
->private_data
);
847 TALLOC_FREE(free_key
);
851 void gencache_iterate_blobs(void (*fn
)(const char *key
, DATA_BLOB value
,
852 time_t timeout
, void *private_data
),
853 void *private_data
, const char *pattern
)
855 struct gencache_iterate_blobs_state state
;
857 if ((fn
== NULL
) || (pattern
== NULL
) || !gencache_init()) {
861 DEBUG(5, ("Searching cache keys with pattern %s\n", pattern
));
864 state
.pattern
= pattern
;
865 state
.private_data
= private_data
;
867 state
.in_persistent
= false;
868 tdb_traverse(cache_notrans
, gencache_iterate_blobs_fn
, &state
);
870 state
.in_persistent
= true;
871 tdb_traverse(cache
, gencache_iterate_blobs_fn
, &state
);
875 * Iterate through all entries which key matches to specified pattern
877 * @param fn pointer to the function that will be supplied with each single
878 * matching cache entry (key, value and timeout) as an arguments
879 * @param data void pointer to an arbitrary data that is passed directly to the fn
880 * function on each call
881 * @param keystr_pattern pattern the existing entries' keys are matched to
885 struct gencache_iterate_state
{
886 void (*fn
)(const char *key
, const char *value
, time_t timeout
,
891 static void gencache_iterate_fn(const char *key
, DATA_BLOB value
,
892 time_t timeout
, void *private_data
)
894 struct gencache_iterate_state
*state
=
895 (struct gencache_iterate_state
*)private_data
;
897 char *free_val
= NULL
;
899 if (value
.data
[value
.length
-1] == '\0') {
900 valstr
= (char *)value
.data
;
902 /* ensure 0-termination */
903 valstr
= talloc_strndup(talloc_tos(), (char *)value
.data
, value
.length
);
905 if (valstr
== NULL
) {
910 DEBUG(10, ("Calling function with arguments "
911 "(key=[%s], value=[%s], timeout=[%s])\n",
912 key
, valstr
, timestring(talloc_tos(), timeout
)));
914 state
->fn(key
, valstr
, timeout
, state
->private_data
);
918 TALLOC_FREE(free_val
);
921 void gencache_iterate(void (*fn
)(const char *key
, const char *value
,
922 time_t timeout
, void *dptr
),
923 void *private_data
, const char *pattern
)
925 struct gencache_iterate_state state
;
931 state
.private_data
= private_data
;
932 gencache_iterate_blobs(gencache_iterate_fn
, &state
, pattern
);