From 69408654b7656189de0c37b0994c668a14e518a3 Mon Sep 17 00:00:00 2001 From: Gary Lockyer Date: Mon, 1 Apr 2019 16:33:52 +1300 Subject: [PATCH] lib ldb key_value: Set index cache size on open Set the default index cache from the passed option "transaction_index_cache_size" on open. This allows the default cache size to be overridden when processing large transactions i.e. joining a large domain. Signed-off-by: Gary Lockyer Reviewed-by: Andrew Bartlett --- lib/ldb/ldb_key_value/ldb_kv.c | 35 +++++++++- lib/ldb/ldb_key_value/ldb_kv.h | 5 ++ lib/ldb/tests/ldb_key_value_test.c | 140 +++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/lib/ldb/ldb_key_value/ldb_kv.c b/lib/ldb/ldb_key_value/ldb_kv.c index 8247ada4256..bd35feed14b 100644 --- a/lib/ldb/ldb_key_value/ldb_kv.c +++ b/lib/ldb/ldb_key_value/ldb_kv.c @@ -1382,7 +1382,9 @@ static int ldb_kv_start_trans(struct ldb_module *module) return ldb_kv->kv_ops->error(ldb_kv); } - ldb_kv_index_transaction_start(module, DEFAULT_INDEX_CACHE_SIZE); + ldb_kv_index_transaction_start( + module, + ldb_kv->index_transaction_cache_size); ldb_kv->reindex_failed = false; @@ -1946,5 +1948,36 @@ int ldb_kv_init_store(struct ldb_kv_private *ldb_kv, } } + /* + * Set the size of the transaction index cache. + * If the ldb option "transaction_index_cache_size" is set use that + * otherwise use DEFAULT_INDEX_CACHE_SIZE + */ + ldb_kv->index_transaction_cache_size = DEFAULT_INDEX_CACHE_SIZE; + { + const char *size = ldb_options_find( + ldb, + options, + "transaction_index_cache_size"); + if (size != NULL) { + size_t cache_size = 0; + errno = 0; + + cache_size = strtoul( size, NULL, 0); + if (cache_size == 0 || errno == ERANGE) { + ldb_debug( + ldb, + LDB_DEBUG_WARNING, + "Invalid transaction_index_cache_size " + "value [%s], using default(%d)\n", + size, + DEFAULT_INDEX_CACHE_SIZE); + } else { + ldb_kv->index_transaction_cache_size = + cache_size; + } + } + } + return LDB_SUCCESS; } diff --git a/lib/ldb/ldb_key_value/ldb_kv.h b/lib/ldb/ldb_key_value/ldb_kv.h index 2eb6fdfed8b..a4aa5ed9e62 100644 --- a/lib/ldb/ldb_key_value/ldb_kv.h +++ b/lib/ldb/ldb_key_value/ldb_kv.h @@ -103,6 +103,11 @@ struct ldb_kv_private { * fork()ed child. */ pid_t pid; + + /* + * The size to be used for the index transaction cache + */ + size_t index_transaction_cache_size; }; struct ldb_kv_context { diff --git a/lib/ldb/tests/ldb_key_value_test.c b/lib/ldb/tests/ldb_key_value_test.c index c78b7fcb316..3f31bb98350 100644 --- a/lib/ldb/tests/ldb_key_value_test.c +++ b/lib/ldb/tests/ldb_key_value_test.c @@ -152,6 +152,7 @@ static void test_default_index_cache_size(void **state) ldb_kv = talloc_zero(test_ctx, struct ldb_kv_private); ldb_kv->pid = getpid(); ldb_kv->kv_ops = &ops; + ldb_kv->index_transaction_cache_size = DEFAULT_INDEX_CACHE_SIZE; ldb_module_set_private(module, ldb_kv); ret = ldb_kv_start_trans(module); @@ -227,6 +228,129 @@ static void test_reindex_cache_size(void **state) TALLOC_FREE(module); } +/* + * Test that ldb_kv_init_store sets the default index transaction cache size + * if the option is not supplied. + */ +static void test_init_store_default_index_cache_size(void **state) +{ + struct test_ctx *test_ctx = talloc_get_type_abort( + *state, + struct test_ctx); + struct ldb_module *module = NULL; + struct ldb_kv_private *ldb_kv = NULL; + struct ldb_context *ldb = NULL; + int ret = LDB_SUCCESS; + + module = talloc_zero(test_ctx, struct ldb_module); + ldb = talloc_zero(test_ctx, struct ldb_context); + ldb_kv = talloc_zero(test_ctx, struct ldb_kv_private); + + ret = ldb_kv_init_store(ldb_kv, "test", ldb, NULL, &module); + assert_int_equal(LDB_SUCCESS, ret); + + assert_int_equal( + DEFAULT_INDEX_CACHE_SIZE, + ldb_kv->index_transaction_cache_size); + + TALLOC_FREE(ldb_kv); + TALLOC_FREE(module); + TALLOC_FREE(ldb); +} + +/* + * Test that ldb_kv_init_store sets the index transaction cache size + * to the value specified in the option. + */ +static void test_init_store_set_index_cache_size(void **state) +{ + struct test_ctx *test_ctx = talloc_get_type_abort( + *state, + struct test_ctx); + struct ldb_module *module = NULL; + struct ldb_kv_private *ldb_kv = NULL; + struct ldb_context *ldb = NULL; + const char *options[] = {"transaction_index_cache_size:1900", NULL}; + int ret = LDB_SUCCESS; + + module = talloc_zero(test_ctx, struct ldb_module); + ldb = talloc_zero(test_ctx, struct ldb_context); + ldb_kv = talloc_zero(test_ctx, struct ldb_kv_private); + + ret = ldb_kv_init_store(ldb_kv, "test", ldb, options, &module); + assert_int_equal(LDB_SUCCESS, ret); + + assert_int_equal( 1900, ldb_kv->index_transaction_cache_size); + + TALLOC_FREE(ldb_kv); + TALLOC_FREE(module); + TALLOC_FREE(ldb); +} + +/* + * Test that ldb_kv_init_store sets the default index transaction cache size + * if the value specified in the option is not a number. + */ +static void test_init_store_set_index_cache_size_non_numeric(void **state) +{ + struct test_ctx *test_ctx = talloc_get_type_abort( + *state, + struct test_ctx); + struct ldb_module *module = NULL; + struct ldb_kv_private *ldb_kv = NULL; + struct ldb_context *ldb = NULL; + const char *options[] = {"transaction_index_cache_size:fred", NULL}; + int ret = LDB_SUCCESS; + + module = talloc_zero(test_ctx, struct ldb_module); + ldb = talloc_zero(test_ctx, struct ldb_context); + ldb_kv = talloc_zero(test_ctx, struct ldb_kv_private); + + ret = ldb_kv_init_store(ldb_kv, "test", ldb, options, &module); + assert_int_equal(LDB_SUCCESS, ret); + + assert_int_equal( + DEFAULT_INDEX_CACHE_SIZE, + ldb_kv->index_transaction_cache_size); + + TALLOC_FREE(ldb_kv); + TALLOC_FREE(module); + TALLOC_FREE(ldb); +} + +/* + * Test that ldb_kv_init_store sets the default index transaction cache size + * if the value specified is too large + */ +static void test_init_store_set_index_cache_size_range(void **state) +{ + struct test_ctx *test_ctx = talloc_get_type_abort( + *state, + struct test_ctx); + struct ldb_module *module = NULL; + struct ldb_kv_private *ldb_kv = NULL; + struct ldb_context *ldb = NULL; + const char *options[] = { + "transaction_index_cache_size:0xfffffffffffffffffffffffffffff", + NULL}; + int ret = LDB_SUCCESS; + + module = talloc_zero(test_ctx, struct ldb_module); + ldb = talloc_zero(test_ctx, struct ldb_context); + ldb_kv = talloc_zero(test_ctx, struct ldb_kv_private); + + ret = ldb_kv_init_store(ldb_kv, "test", ldb, options, &module); + assert_int_equal(LDB_SUCCESS, ret); + + assert_int_equal( + DEFAULT_INDEX_CACHE_SIZE, + ldb_kv->index_transaction_cache_size); + + TALLOC_FREE(ldb_kv); + TALLOC_FREE(module); + TALLOC_FREE(ldb); +} + int main(int argc, const char **argv) { const struct CMUnitTest tests[] = { @@ -242,6 +366,22 @@ int main(int argc, const char **argv) test_reindex_cache_size, setup, teardown), + cmocka_unit_test_setup_teardown( + test_init_store_default_index_cache_size, + setup, + teardown), + cmocka_unit_test_setup_teardown( + test_init_store_set_index_cache_size, + setup, + teardown), + cmocka_unit_test_setup_teardown( + test_init_store_set_index_cache_size_non_numeric, + setup, + teardown), + cmocka_unit_test_setup_teardown( + test_init_store_set_index_cache_size_range, + setup, + teardown), }; return cmocka_run_group_tests(tests, NULL, NULL); -- 2.11.4.GIT