2 Unix SMB/CIFS implementation.
4 trivial database library
6 Copyright (C) Andrew Tridgell 1999-2005
7 Copyright (C) Paul `Rusty' Russell 2000
8 Copyright (C) Jeremy Allison 2000-2003
10 ** NOTE! The following LGPL license applies to the tdb
11 ** library. This does NOT imply that all of Samba is released
14 This library is free software; you can redistribute it and/or
15 modify it under the terms of the GNU Lesser General Public
16 License as published by the Free Software Foundation; either
17 version 3 of the License, or (at your option) any later version.
19 This library is distributed in the hope that it will be useful,
20 but WITHOUT ANY WARRANTY; without even the implied warranty of
21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 Lesser General Public License for more details.
24 You should have received a copy of the GNU Lesser General Public
25 License along with this library; if not, see <http://www.gnu.org/licenses/>.
29 #include "tdb_private.h"
31 /* check for an out of bounds access - if it is out of bounds then
32 see if the database has been expanded by someone else and expand
35 static int tdb_oob(struct tdb_context
*tdb
, tdb_off_t off
, tdb_len_t len
,
39 if (len
+ off
< len
) {
41 /* Ensure ecode is set for log fn. */
42 tdb
->ecode
= TDB_ERR_IO
;
43 TDB_LOG((tdb
, TDB_DEBUG_FATAL
,"tdb_oob off %u len %u wrap\n",
49 if (off
+ len
<= tdb
->map_size
)
51 if (tdb
->flags
& TDB_INTERNAL
) {
53 /* Ensure ecode is set for log fn. */
54 tdb
->ecode
= TDB_ERR_IO
;
55 TDB_LOG((tdb
, TDB_DEBUG_FATAL
,"tdb_oob len %u beyond internal malloc size %u\n",
56 (int)(off
+ len
), (int)tdb
->map_size
));
61 if (fstat(tdb
->fd
, &st
) == -1) {
62 tdb
->ecode
= TDB_ERR_IO
;
66 /* Beware >4G files! */
67 if ((tdb_off_t
)st
.st_size
!= st
.st_size
) {
68 /* Ensure ecode is set for log fn. */
69 tdb
->ecode
= TDB_ERR_IO
;
70 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "tdb_oob len %llu too large!\n",
71 (long long)st
.st_size
));
75 /* Unmap, update size, remap. We do this unconditionally, to handle
76 * the unusual case where the db is truncated.
78 * This can happen to a child using tdb_reopen_all(true) on a
79 * TDB_CLEAR_IF_FIRST tdb whose parent crashes: the next
80 * opener will truncate the database. */
81 if (tdb_munmap(tdb
) == -1) {
82 tdb
->ecode
= TDB_ERR_IO
;
85 tdb
->map_size
= st
.st_size
;
86 if (tdb_mmap(tdb
) != 0) {
90 if (st
.st_size
< (size_t)off
+ len
) {
92 /* Ensure ecode is set for log fn. */
93 tdb
->ecode
= TDB_ERR_IO
;
94 TDB_LOG((tdb
, TDB_DEBUG_FATAL
,"tdb_oob len %u beyond eof at %u\n",
95 (int)(off
+ len
), (int)st
.st_size
));
102 /* write a lump of data at a specified offset */
103 static int tdb_write(struct tdb_context
*tdb
, tdb_off_t off
,
104 const void *buf
, tdb_len_t len
)
110 if (tdb
->read_only
|| tdb
->traverse_read
) {
111 tdb
->ecode
= TDB_ERR_RDONLY
;
115 if (tdb
->methods
->tdb_oob(tdb
, off
, len
, 0) != 0)
119 memcpy(off
+ (char *)tdb
->map_ptr
, buf
, len
);
121 #ifdef HAVE_INCOHERENT_MMAP
122 tdb
->ecode
= TDB_ERR_IO
;
125 ssize_t written
= pwrite(tdb
->fd
, buf
, len
, off
);
126 if ((written
!= (ssize_t
)len
) && (written
!= -1)) {
128 tdb
->ecode
= TDB_ERR_IO
;
129 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "tdb_write: wrote only "
130 "%zi of %u bytes at %u, trying once more\n",
132 written
= pwrite(tdb
->fd
, (const char *)buf
+written
,
137 /* Ensure ecode is set for log fn. */
138 tdb
->ecode
= TDB_ERR_IO
;
139 TDB_LOG((tdb
, TDB_DEBUG_FATAL
,"tdb_write failed at %u "
140 "len=%u (%s)\n", off
, len
, strerror(errno
)));
142 } else if (written
!= (ssize_t
)len
) {
143 tdb
->ecode
= TDB_ERR_IO
;
144 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "tdb_write: failed to "
145 "write %u bytes at %u in two attempts\n",
154 /* Endian conversion: we only ever deal with 4 byte quantities */
155 void *tdb_convert(void *buf
, uint32_t size
)
157 uint32_t i
, *p
= (uint32_t *)buf
;
158 for (i
= 0; i
< size
/ 4; i
++)
159 p
[i
] = TDB_BYTEREV(p
[i
]);
164 /* read a lump of data at a specified offset, maybe convert */
165 static int tdb_read(struct tdb_context
*tdb
, tdb_off_t off
, void *buf
,
166 tdb_len_t len
, int cv
)
168 if (tdb
->methods
->tdb_oob(tdb
, off
, len
, 0) != 0) {
173 memcpy(buf
, off
+ (char *)tdb
->map_ptr
, len
);
175 #ifdef HAVE_INCOHERENT_MMAP
176 tdb
->ecode
= TDB_ERR_IO
;
179 ssize_t ret
= pread(tdb
->fd
, buf
, len
, off
);
180 if (ret
!= (ssize_t
)len
) {
181 /* Ensure ecode is set for log fn. */
182 tdb
->ecode
= TDB_ERR_IO
;
183 TDB_LOG((tdb
, TDB_DEBUG_FATAL
,"tdb_read failed at %u "
184 "len=%u ret=%zi (%s) map_size=%u\n",
185 off
, len
, ret
, strerror(errno
),
192 tdb_convert(buf
, len
);
200 do an unlocked scan of the hash table heads to find the next non-zero head. The value
201 will then be confirmed with the lock held
203 static void tdb_next_hash_chain(struct tdb_context
*tdb
, uint32_t *chain
)
207 for (;h
< tdb
->hash_size
;h
++) {
208 if (0 != *(uint32_t *)(TDB_HASH_TOP(h
) + (unsigned char *)tdb
->map_ptr
)) {
214 for (;h
< tdb
->hash_size
;h
++) {
215 if (tdb_ofs_read(tdb
, TDB_HASH_TOP(h
), &off
) != 0 || off
!= 0) {
224 int tdb_munmap(struct tdb_context
*tdb
)
226 if (tdb
->flags
& TDB_INTERNAL
)
233 ret
= munmap(tdb
->map_ptr
, tdb
->map_size
);
242 /* If mmap isn't coherent, *everyone* must always mmap. */
243 static bool should_mmap(const struct tdb_context
*tdb
)
245 #ifdef HAVE_INCOHERENT_MMAP
248 return !(tdb
->flags
& TDB_NOMMAP
);
252 int tdb_mmap(struct tdb_context
*tdb
)
254 if (tdb
->flags
& TDB_INTERNAL
)
258 if (should_mmap(tdb
)) {
259 tdb
->map_ptr
= mmap(NULL
, tdb
->map_size
,
260 PROT_READ
|(tdb
->read_only
? 0:PROT_WRITE
),
261 MAP_SHARED
|MAP_FILE
, tdb
->fd
, 0);
264 * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
267 if (tdb
->map_ptr
== MAP_FAILED
) {
269 TDB_LOG((tdb
, TDB_DEBUG_WARNING
, "tdb_mmap failed for size %u (%s)\n",
270 tdb
->map_size
, strerror(errno
)));
271 #ifdef HAVE_INCOHERENT_MMAP
272 tdb
->ecode
= TDB_ERR_IO
;
285 /* expand a file. we prefer to use ftruncate, as that is what posix
286 says to use for mmap expansion */
287 static int tdb_expand_file(struct tdb_context
*tdb
, tdb_off_t size
, tdb_off_t addition
)
292 if (tdb
->read_only
|| tdb
->traverse_read
) {
293 tdb
->ecode
= TDB_ERR_RDONLY
;
297 if (!tdb_add_off_t(size
, addition
, &new_size
)) {
298 tdb
->ecode
= TDB_ERR_OOM
;
299 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "expand_file write "
300 "overflow detected current size[%u] addition[%u]!\n",
301 (unsigned)size
, (unsigned)addition
));
306 if (ftruncate(tdb
->fd
, new_size
) == -1) {
308 ssize_t written
= pwrite(tdb
->fd
, &b
, 1, new_size
- 1);
310 /* try once more, potentially revealing errno */
311 written
= pwrite(tdb
->fd
, &b
, 1, new_size
- 1);
314 /* again - give up, guessing errno */
318 tdb
->ecode
= TDB_ERR_OOM
;
319 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "expand_file to %u failed (%s)\n",
320 (unsigned)new_size
, strerror(errno
)));
325 /* now fill the file with something. This ensures that the
326 file isn't sparse, which would be very bad if we ran out of
327 disk. This must be done with write, not via mmap */
328 memset(buf
, TDB_PAD_BYTE
, sizeof(buf
));
330 size_t n
= addition
>sizeof(buf
)?sizeof(buf
):addition
;
331 ssize_t written
= pwrite(tdb
->fd
, buf
, n
, size
);
333 /* prevent infinite loops: try _once_ more */
334 written
= pwrite(tdb
->fd
, buf
, n
, size
);
337 /* give up, trying to provide a useful errno */
338 tdb
->ecode
= TDB_ERR_OOM
;
339 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "expand_file write "
340 "returned 0 twice: giving up!\n"));
345 tdb
->ecode
= TDB_ERR_OOM
;
346 TDB_LOG((tdb
, TDB_DEBUG_FATAL
, "expand_file write of "
347 "%u bytes failed (%s)\n", (int)n
,
352 TDB_LOG((tdb
, TDB_DEBUG_WARNING
, "expand_file: wrote "
353 "only %zu of %zi bytes - retrying\n", written
,
363 /* You need 'size', this tells you how much you should expand by. */
364 tdb_off_t
tdb_expand_adjust(tdb_off_t map_size
, tdb_off_t size
, int page_size
)
366 tdb_off_t new_size
, top_size
;
368 /* limit size in order to avoid using up huge amounts of memory for
369 * in memory tdbs if an oddball huge record creeps in */
370 if (size
> 100 * 1024) {
371 top_size
= map_size
+ size
* 2;
373 top_size
= map_size
+ size
* 100;
376 /* always make room for at least top_size more records, and at
377 least 25% more space. if the DB is smaller than 100MiB,
378 otherwise grow it by 10% only. */
379 if (map_size
> 100 * 1024 * 1024) {
380 new_size
= map_size
* 1.10;
382 new_size
= map_size
* 1.25;
385 /* Round the database up to a multiple of the page size */
386 new_size
= MAX(top_size
, new_size
);
387 return TDB_ALIGN(new_size
, page_size
) - map_size
;
390 /* expand the database at least size bytes by expanding the underlying
391 file and doing the mmap again if necessary */
392 int tdb_expand(struct tdb_context
*tdb
, tdb_off_t size
)
394 struct tdb_record rec
;
397 if (tdb_lock(tdb
, -1, F_WRLCK
) == -1) {
398 TDB_LOG((tdb
, TDB_DEBUG_ERROR
, "lock failed in tdb_expand\n"));
402 /* must know about any previous expansions by another process */
403 tdb
->methods
->tdb_oob(tdb
, tdb
->map_size
, 1, 1);
405 size
= tdb_expand_adjust(tdb
->map_size
, size
, tdb
->page_size
);
407 /* expand the file itself */
408 if (!(tdb
->flags
& TDB_INTERNAL
)) {
409 if (tdb
->methods
->tdb_expand_file(tdb
, tdb
->map_size
, size
) != 0)
413 /* form a new freelist record */
414 offset
= tdb
->map_size
;
415 memset(&rec
,'\0',sizeof(rec
));
416 rec
.rec_len
= size
- sizeof(rec
);
418 if (tdb
->flags
& TDB_INTERNAL
) {
419 char *new_map_ptr
= (char *)realloc(tdb
->map_ptr
,
420 tdb
->map_size
+ size
);
424 tdb
->map_ptr
= new_map_ptr
;
425 tdb
->map_size
+= size
;
427 /* Explicitly remap: if we're in a transaction, this won't
428 * happen automatically! */
430 tdb
->map_size
+= size
;
431 if (tdb_mmap(tdb
) != 0) {
436 /* link it into the free list */
437 if (tdb_free(tdb
, offset
, &rec
) == -1)
440 tdb_unlock(tdb
, -1, F_WRLCK
);
443 tdb_unlock(tdb
, -1, F_WRLCK
);
447 /* read/write a tdb_off_t */
448 int tdb_ofs_read(struct tdb_context
*tdb
, tdb_off_t offset
, tdb_off_t
*d
)
450 return tdb
->methods
->tdb_read(tdb
, offset
, (char*)d
, sizeof(*d
), DOCONV());
453 int tdb_ofs_write(struct tdb_context
*tdb
, tdb_off_t offset
, tdb_off_t
*d
)
456 return tdb
->methods
->tdb_write(tdb
, offset
, CONVERT(off
), sizeof(*d
));
460 /* read a lump of data, allocating the space for it */
461 unsigned char *tdb_alloc_read(struct tdb_context
*tdb
, tdb_off_t offset
, tdb_len_t len
)
465 /* some systems don't like zero length malloc */
467 if (!(buf
= (unsigned char *)malloc(len
? len
: 1))) {
468 /* Ensure ecode is set for log fn. */
469 tdb
->ecode
= TDB_ERR_OOM
;
470 TDB_LOG((tdb
, TDB_DEBUG_ERROR
,"tdb_alloc_read malloc failed len=%u (%s)\n",
471 len
, strerror(errno
)));
474 if (tdb
->methods
->tdb_read(tdb
, offset
, buf
, len
, 0) == -1) {
481 /* Give a piece of tdb data to a parser */
483 int tdb_parse_data(struct tdb_context
*tdb
, TDB_DATA key
,
484 tdb_off_t offset
, tdb_len_t len
,
485 int (*parser
)(TDB_DATA key
, TDB_DATA data
,
494 if ((tdb
->transaction
== NULL
) && (tdb
->map_ptr
!= NULL
)) {
496 * Optimize by avoiding the malloc/memcpy/free, point the
497 * parser directly at the mmap area.
499 if (tdb
->methods
->tdb_oob(tdb
, offset
, len
, 0) != 0) {
502 data
.dptr
= offset
+ (unsigned char *)tdb
->map_ptr
;
503 return parser(key
, data
, private_data
);
506 if (!(data
.dptr
= tdb_alloc_read(tdb
, offset
, len
))) {
510 result
= parser(key
, data
, private_data
);
515 /* read/write a record */
516 int tdb_rec_read(struct tdb_context
*tdb
, tdb_off_t offset
, struct tdb_record
*rec
)
518 if (tdb
->methods
->tdb_read(tdb
, offset
, rec
, sizeof(*rec
),DOCONV()) == -1)
520 if (TDB_BAD_MAGIC(rec
)) {
521 /* Ensure ecode is set for log fn. */
522 tdb
->ecode
= TDB_ERR_CORRUPT
;
523 TDB_LOG((tdb
, TDB_DEBUG_FATAL
,"tdb_rec_read bad magic 0x%x at offset=%u\n", rec
->magic
, offset
));
526 return tdb
->methods
->tdb_oob(tdb
, rec
->next
, sizeof(*rec
), 0);
529 int tdb_rec_write(struct tdb_context
*tdb
, tdb_off_t offset
, struct tdb_record
*rec
)
531 struct tdb_record r
= *rec
;
532 return tdb
->methods
->tdb_write(tdb
, offset
, CONVERT(r
), sizeof(r
));
535 static const struct tdb_methods io_methods
= {
544 initialise the default methods table
546 void tdb_io_init(struct tdb_context
*tdb
)
548 tdb
->methods
= &io_methods
;