4 Copyright (C) Andrew Tridgell 2004-2009
6 ** NOTE! The following LGPL license applies to the ldb
7 ** library. This does NOT imply that all of Samba is released
10 This library is free software; you can redistribute it and/or
11 modify it under the terms of the GNU Lesser General Public
12 License as published by the Free Software Foundation; either
13 version 3 of the License, or (at your option) any later version.
15 This library is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 Lesser General Public License for more details.
20 You should have received a copy of the GNU Lesser General Public
21 License along with this library; if not, see <http://www.gnu.org/licenses/>.
27 * Component: ldb tdb backend - indexing
29 * Description: indexing routines for ldb tdb backend
31 * Author: Andrew Tridgell
36 LDB Index design and choice of TDB key:
37 =======================================
39 LDB has index records held as LDB objects with a special record like:
43 value may be base64 encoded, if it is deemed not printable:
45 dn: @INDEX:attr::base64-value
47 In each record, there is two possible formats:
49 The original format is:
50 -----------------------
52 dn: @INDEX:NAME:DNSUPDATEPROXY
54 @IDX: CN=DnsUpdateProxy,CN=Users,DC=addom,DC=samba,DC=example,DC=com
56 In this format, @IDX is multi-valued, one entry for each match
58 The corrosponding entry is stored in a TDB record with key:
60 DN=CN=DNSUPDATEPROXY,CN=USERS,DC=ADDOM,DC=SAMBA,DC=EXAMPLE,DC=COM
62 (This allows a scope BASE search to directly find the record via
63 a simple casefold of the DN).
65 The original mixed-case DN is stored in the entry iself.
68 The new 'GUID index' format is:
69 -------------------------------
71 dn: @INDEX:NAME:DNSUPDATEPROXY
73 @IDX: <binary GUID>[<binary GUID>[...]]
75 The binary guid is 16 bytes, as bytes and not expanded as hexidecimal
76 or pretty-printed. The GUID is chosen from the message to be stored
77 by the @IDXGUID attribute on @INDEXLIST.
79 If there are multiple values the @IDX value simply becomes longer,
82 The corrosponding entry is stored in a TDB record with key:
86 This allows a very quick translation between the fixed-length index
87 values and the TDB key, while seperating entries from other data
88 in the TDB, should they be unlucky enough to start with the bytes of
91 Additionally, this allows a scope BASE search to directly find the
92 record via a simple match on a GUID= extended DN, controlled via
93 @IDX_DN_GUID on @INDEXLIST
95 Exception for special @ DNs:
97 @BASEINFO, @INDEXLIST and all other special DNs are stored as per the
98 original format, as they are never referenced in an index and are used
99 to bootstrap the database.
102 Control points for choice of index mode
103 ---------------------------------------
105 The choice of index and TDB key mode is made based (for example, from
106 Samba) on entries in the @INDEXLIST DN:
112 By default, the original DN format is used.
115 Control points for choosing indexed attributes
116 ----------------------------------------------
118 @IDXATTR controls if an attribute is indexed
121 @IDXATTR: samAccountName
122 @IDXATTR: nETBIOSName
128 void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
129 const char *GUID_index_attribute,
130 const char *GUID_index_dn_component)
132 This is used, particularly in combination with the below, instead of
133 the @IDXGUID and @IDX_DN_GUID values in @INDEXLIST.
135 void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
136 bool one_level_indexes);
137 void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
138 ldb_attribute_handler_override_fn_t override,
141 When the above two functions are called in combination, the @INDEXLIST
142 values are not read from the DB, so
143 ldb_schema_set_override_GUID_index() must be called.
148 #include "ldb_private.h"
149 #include "lib/util/binsearch.h"
155 * Do not optimise the intersection of this list,
156 * we must never return an entry not in this
157 * list. This allows the index for
158 * SCOPE_ONELEVEL to be trusted.
164 struct tdb_context *itdb;
168 enum key_truncation {
173 static int ltdb_write_index_dn_guid(struct ldb_module *module,
174 const struct ldb_message *msg,
176 static int ltdb_index_dn_base_dn(struct ldb_module *module,
177 struct ltdb_private *ltdb,
178 struct ldb_dn *base_dn,
179 struct dn_list *dn_list,
180 enum key_truncation *truncation);
182 static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
183 struct dn_list *list);
185 /* we put a @IDXVERSION attribute on index entries. This
186 allows us to tell if it was written by an older version
188 #define LTDB_INDEXING_VERSION 2
190 #define LTDB_GUID_INDEXING_VERSION 3
192 static unsigned ltdb_max_key_length(struct ltdb_private *ltdb) {
193 if (ltdb->max_key_length == 0){
196 return ltdb->max_key_length;
199 /* enable the idxptr mode when transactions start */
200 int ltdb_index_transaction_start(struct ldb_module *module)
202 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
203 ltdb->idxptr = talloc_zero(ltdb, struct ltdb_idxptr);
204 if (ltdb->idxptr == NULL) {
205 return ldb_oom(ldb_module_get_ctx(module));
212 see if two ldb_val structures contain exactly the same data
213 return -1 or 1 for a mismatch, 0 for match
215 static int ldb_val_equal_exact_for_qsort(const struct ldb_val *v1,
216 const struct ldb_val *v2)
218 if (v1->length > v2->length) {
221 if (v1->length < v2->length) {
224 return memcmp(v1->data, v2->data, v1->length);
228 see if two ldb_val structures contain exactly the same data
229 return -1 or 1 for a mismatch, 0 for match
231 static int ldb_val_equal_exact_ordered(const struct ldb_val v1,
232 const struct ldb_val *v2)
234 if (v1.length > v2->length) {
237 if (v1.length < v2->length) {
240 return memcmp(v1.data, v2->data, v1.length);
245 find a entry in a dn_list, using a ldb_val. Uses a case sensitive
246 binary-safe comparison for the 'dn' returns -1 if not found
248 This is therefore safe when the value is a GUID in the future
250 static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
251 const struct dn_list *list,
252 const struct ldb_val *v)
255 struct ldb_val *exact = NULL, *next = NULL;
257 if (ltdb->cache->GUID_index_attribute == NULL) {
258 for (i=0; i<list->count; i++) {
259 if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
266 BINARY_ARRAY_SEARCH_GTE(list->dn, list->count,
267 *v, ldb_val_equal_exact_ordered,
272 /* Not required, but keeps the compiler quiet */
277 i = exact - list->dn;
282 find a entry in a dn_list. Uses a case sensitive comparison with the dn
283 returns -1 if not found
285 static int ltdb_dn_list_find_msg(struct ltdb_private *ltdb,
286 struct dn_list *list,
287 const struct ldb_message *msg)
290 const struct ldb_val *key_val;
291 if (ltdb->cache->GUID_index_attribute == NULL) {
292 const char *dn_str = ldb_dn_get_linearized(msg->dn);
293 v.data = discard_const_p(unsigned char, dn_str);
294 v.length = strlen(dn_str);
296 key_val = ldb_msg_find_ldb_val(msg,
297 ltdb->cache->GUID_index_attribute);
298 if (key_val == NULL) {
303 return ltdb_dn_list_find_val(ltdb, list, &v);
307 this is effectively a cast function, but with lots of paranoia
308 checks and also copes with CPUs that are fussy about pointer
311 static struct dn_list *ltdb_index_idxptr(struct ldb_module *module, TDB_DATA rec, bool check_parent)
313 struct dn_list *list;
314 if (rec.dsize != sizeof(void *)) {
315 ldb_asprintf_errstring(ldb_module_get_ctx(module),
316 "Bad data size for idxptr %u", (unsigned)rec.dsize);
319 /* note that we can't just use a cast here, as rec.dptr may
320 not be aligned sufficiently for a pointer. A cast would cause
321 platforms like some ARM CPUs to crash */
322 memcpy(&list, rec.dptr, sizeof(void *));
323 list = talloc_get_type(list, struct dn_list);
325 ldb_asprintf_errstring(ldb_module_get_ctx(module),
326 "Bad type '%s' for idxptr",
327 talloc_get_name(list));
330 if (check_parent && list->dn && talloc_parent(list->dn) != list) {
331 ldb_asprintf_errstring(ldb_module_get_ctx(module),
332 "Bad parent '%s' for idxptr",
333 talloc_get_name(talloc_parent(list->dn)));
340 return the @IDX list in an index entry for a dn as a
343 static int ltdb_dn_list_load(struct ldb_module *module,
344 struct ltdb_private *ltdb,
345 struct ldb_dn *dn, struct dn_list *list)
347 struct ldb_message *msg;
349 struct ldb_message_element *el;
351 struct dn_list *list2;
357 /* see if we have any in-memory index entries */
358 if (ltdb->idxptr == NULL ||
359 ltdb->idxptr->itdb == NULL) {
363 key.dptr = discard_const_p(unsigned char, ldb_dn_get_linearized(dn));
364 key.dsize = strlen((char *)key.dptr);
366 rec = tdb_fetch(ltdb->idxptr->itdb, key);
367 if (rec.dptr == NULL) {
371 /* we've found an in-memory index entry */
372 list2 = ltdb_index_idxptr(module, rec, true);
375 return LDB_ERR_OPERATIONS_ERROR;
383 msg = ldb_msg_new(list);
385 return LDB_ERR_OPERATIONS_ERROR;
388 ret = ltdb_search_dn1(module, dn, msg,
389 LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
390 |LDB_UNPACK_DATA_FLAG_NO_DN);
391 if (ret != LDB_SUCCESS) {
396 el = ldb_msg_find_element(msg, LTDB_IDX);
402 version = ldb_msg_find_attr_as_int(msg, LTDB_IDXVERSION, 0);
405 * we avoid copying the strings by stealing the list. We have
406 * to steal msg onto el->values (which looks odd) because we
407 * asked for the memory to be allocated on msg, not on each
408 * value with LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC above
410 if (ltdb->cache->GUID_index_attribute == NULL) {
411 /* check indexing version number */
412 if (version != LTDB_INDEXING_VERSION) {
413 ldb_debug_set(ldb_module_get_ctx(module),
415 "Wrong DN index version %d "
416 "expected %d for %s",
417 version, LTDB_INDEXING_VERSION,
418 ldb_dn_get_linearized(dn));
419 return LDB_ERR_OPERATIONS_ERROR;
422 talloc_steal(el->values, msg);
423 list->dn = talloc_steal(list, el->values);
424 list->count = el->num_values;
427 if (version != LTDB_GUID_INDEXING_VERSION) {
428 /* This is quite likely during the DB startup
429 on first upgrade to using a GUID index */
430 ldb_debug_set(ldb_module_get_ctx(module),
432 "Wrong GUID index version %d "
433 "expected %d for %s",
434 version, LTDB_GUID_INDEXING_VERSION,
435 ldb_dn_get_linearized(dn));
436 return LDB_ERR_OPERATIONS_ERROR;
439 if (el->num_values == 0) {
440 return LDB_ERR_OPERATIONS_ERROR;
443 if ((el->values[0].length % LTDB_GUID_SIZE) != 0) {
444 return LDB_ERR_OPERATIONS_ERROR;
447 list->count = el->values[0].length / LTDB_GUID_SIZE;
448 list->dn = talloc_array(list, struct ldb_val, list->count);
451 * The actual data is on msg, due to
452 * LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
454 talloc_steal(list->dn, msg);
455 for (i = 0; i < list->count; i++) {
457 = &el->values[0].data[i * LTDB_GUID_SIZE];
458 list->dn[i].length = LTDB_GUID_SIZE;
462 /* We don't need msg->elements any more */
463 talloc_free(msg->elements);
467 int ltdb_key_dn_from_idx(struct ldb_module *module,
468 struct ltdb_private *ltdb,
473 struct ldb_context *ldb = ldb_module_get_ctx(module);
476 enum key_truncation truncation = KEY_NOT_TRUNCATED;
477 struct dn_list *list = talloc(mem_ctx, struct dn_list);
480 return LDB_ERR_OPERATIONS_ERROR;
484 ret = ltdb_index_dn_base_dn(module, ltdb, dn, list, &truncation);
485 if (ret != LDB_SUCCESS) {
490 if (list->count == 0) {
492 return LDB_ERR_NO_SUCH_OBJECT;
495 if (list->count > 1 && truncation == KEY_NOT_TRUNCATED) {
496 const char *dn_str = ldb_dn_get_linearized(dn);
497 ldb_asprintf_errstring(ldb_module_get_ctx(module),
499 ": Failed to read DN index "
500 "against %s for %s: too many "
502 ltdb->cache->GUID_index_attribute,
503 dn_str, list->count);
505 return LDB_ERR_CONSTRAINT_VIOLATION;
508 if (list->count > 0 && truncation == KEY_TRUNCATED) {
510 * DN key has been truncated, need to inspect the actual
511 * records to locate the actual DN
515 for (i=0; i < list->count; i++) {
516 uint8_t guid_key[LTDB_GUID_KEY_SIZE];
519 .dsize = sizeof(guid_key)
521 const int flags = LDB_UNPACK_DATA_FLAG_NO_ATTRS;
522 struct ldb_message *rec = ldb_msg_new(ldb);
524 return LDB_ERR_OPERATIONS_ERROR;
527 ret = ltdb_idx_to_key(module, ltdb,
530 if (ret != LDB_SUCCESS) {
536 ret = ltdb_search_key(module, ltdb, key,
538 if (key.dptr != guid_key) {
539 TALLOC_FREE(key.dptr);
541 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
543 * the record has disappeared?
544 * yes, this can happen
550 if (ret != LDB_SUCCESS) {
551 /* an internal error */
554 return LDB_ERR_OPERATIONS_ERROR;
558 * We found the actual DN that we wanted from in the
559 * multiple values that matched the index
560 * (due to truncation), so return that.
563 if (ldb_dn_compare(dn, rec->dn) == 0) {
571 * We matched the index but the actual DN we wanted
576 return LDB_ERR_NO_SUCH_OBJECT;
580 /* The tdb_key memory is allocated by the caller */
581 ret = ltdb_guid_to_key(module, ltdb,
582 &list->dn[index], tdb_key);
585 if (ret != LDB_SUCCESS) {
586 return LDB_ERR_OPERATIONS_ERROR;
595 save a dn_list into a full @IDX style record
597 static int ltdb_dn_list_store_full(struct ldb_module *module,
598 struct ltdb_private *ltdb,
600 struct dn_list *list)
602 struct ldb_message *msg;
605 msg = ldb_msg_new(module);
607 return ldb_module_oom(module);
612 if (list->count == 0) {
613 ret = ltdb_delete_noindex(module, msg);
614 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
621 if (ltdb->cache->GUID_index_attribute == NULL) {
622 ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
623 LTDB_INDEXING_VERSION);
624 if (ret != LDB_SUCCESS) {
626 return ldb_module_oom(module);
629 ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
630 LTDB_GUID_INDEXING_VERSION);
631 if (ret != LDB_SUCCESS) {
633 return ldb_module_oom(module);
637 if (list->count > 0) {
638 struct ldb_message_element *el;
640 ret = ldb_msg_add_empty(msg, LTDB_IDX, LDB_FLAG_MOD_ADD, &el);
641 if (ret != LDB_SUCCESS) {
643 return ldb_module_oom(module);
646 if (ltdb->cache->GUID_index_attribute == NULL) {
647 el->values = list->dn;
648 el->num_values = list->count;
652 el->values = talloc_array(msg,
654 if (el->values == NULL) {
656 return ldb_module_oom(module);
659 v.data = talloc_array_size(el->values,
662 if (v.data == NULL) {
664 return ldb_module_oom(module);
667 v.length = talloc_get_size(v.data);
669 for (i = 0; i < list->count; i++) {
670 if (list->dn[i].length !=
673 return ldb_module_operr(module);
675 memcpy(&v.data[LTDB_GUID_SIZE*i],
684 ret = ltdb_store(module, msg, TDB_REPLACE);
690 save a dn_list into the database, in either @IDX or internal format
692 static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
693 struct dn_list *list)
695 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
698 struct dn_list *list2;
700 if (ltdb->idxptr == NULL) {
701 return ltdb_dn_list_store_full(module, ltdb,
705 if (ltdb->idxptr->itdb == NULL) {
706 ltdb->idxptr->itdb = tdb_open(NULL, 1000, TDB_INTERNAL, O_RDWR, 0);
707 if (ltdb->idxptr->itdb == NULL) {
708 return LDB_ERR_OPERATIONS_ERROR;
712 key.dptr = discard_const_p(unsigned char, ldb_dn_get_linearized(dn));
713 key.dsize = strlen((char *)key.dptr);
715 rec = tdb_fetch(ltdb->idxptr->itdb, key);
716 if (rec.dptr != NULL) {
717 list2 = ltdb_index_idxptr(module, rec, false);
720 return LDB_ERR_OPERATIONS_ERROR;
723 list2->dn = talloc_steal(list2, list->dn);
724 list2->count = list->count;
728 list2 = talloc(ltdb->idxptr, struct dn_list);
730 return LDB_ERR_OPERATIONS_ERROR;
732 list2->dn = talloc_steal(list2, list->dn);
733 list2->count = list->count;
735 rec.dptr = (uint8_t *)&list2;
736 rec.dsize = sizeof(void *);
740 * This is not a store into the main DB, but into an in-memory
741 * TDB, so we don't need a guard on ltdb->read_only
743 ret = tdb_store(ltdb->idxptr->itdb, key, rec, TDB_INSERT);
745 return ltdb_err_map(tdb_error(ltdb->idxptr->itdb));
751 traverse function for storing the in-memory index entries on disk
753 static int ltdb_index_traverse_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *state)
755 struct ldb_module *module = state;
756 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
758 struct ldb_context *ldb = ldb_module_get_ctx(module);
760 struct dn_list *list;
762 list = ltdb_index_idxptr(module, data, true);
764 ltdb->idxptr->error = LDB_ERR_OPERATIONS_ERROR;
769 v.length = strnlen((char *)key.dptr, key.dsize);
771 dn = ldb_dn_from_ldb_val(module, ldb, &v);
773 ldb_asprintf_errstring(ldb, "Failed to parse index key %*.*s as an LDB DN", (int)v.length, (int)v.length, (const char *)v.data);
774 ltdb->idxptr->error = LDB_ERR_OPERATIONS_ERROR;
778 ltdb->idxptr->error = ltdb_dn_list_store_full(module, ltdb,
781 if (ltdb->idxptr->error != 0) {
787 /* cleanup the idxptr mode when transaction commits */
788 int ltdb_index_transaction_commit(struct ldb_module *module)
790 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
793 struct ldb_context *ldb = ldb_module_get_ctx(module);
795 ldb_reset_err_string(ldb);
797 if (ltdb->idxptr->itdb) {
798 tdb_traverse(ltdb->idxptr->itdb, ltdb_index_traverse_store, module);
799 tdb_close(ltdb->idxptr->itdb);
802 ret = ltdb->idxptr->error;
803 if (ret != LDB_SUCCESS) {
804 if (!ldb_errstring(ldb)) {
805 ldb_set_errstring(ldb, ldb_strerror(ret));
807 ldb_asprintf_errstring(ldb, "Failed to store index records in transaction commit: %s", ldb_errstring(ldb));
810 talloc_free(ltdb->idxptr);
815 /* cleanup the idxptr mode when transaction cancels */
816 int ltdb_index_transaction_cancel(struct ldb_module *module)
818 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
819 if (ltdb->idxptr && ltdb->idxptr->itdb) {
820 tdb_close(ltdb->idxptr->itdb);
822 talloc_free(ltdb->idxptr);
829 return the dn key to be used for an index
830 the caller is responsible for freeing
832 static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
833 struct ltdb_private *ltdb,
834 const char *attr, const struct ldb_val *value,
835 const struct ldb_schema_attribute **ap,
836 enum key_truncation *truncation)
840 const struct ldb_schema_attribute *a = NULL;
841 char *attr_folded = NULL;
842 const char *attr_for_dn = NULL;
844 bool should_b64_encode;
845 unsigned max_key_length = ltdb_max_key_length(ltdb);
846 unsigned key_len = 0;
847 unsigned attr_len = 0;
848 unsigned indx_len = 0;
849 unsigned frmt_len = 0;
851 if (attr[0] == '@') {
858 attr_folded = ldb_attr_casefold(ldb, attr);
863 attr_for_dn = attr_folded;
865 a = ldb_schema_attribute_by_name(ldb, attr);
869 r = a->syntax->canonicalise_fn(ldb, ldb, value, &v);
870 if (r != LDB_SUCCESS) {
871 const char *errstr = ldb_errstring(ldb);
872 /* canonicalisation can be refused. For
873 example, a attribute that takes wildcards
874 will refuse to canonicalise if the value
875 contains a wildcard */
876 ldb_asprintf_errstring(ldb,
877 "Failed to create index "
878 "key for attribute '%s':%s%s%s",
879 attr, ldb_strerror(r),
882 talloc_free(attr_folded);
886 attr_len = strlen(attr_for_dn);
887 indx_len = strlen(LTDB_INDEX);
890 * We do not base 64 encode a DN in a key, it has already been
891 * casefold and lineraized, that is good enough. That already
892 * avoids embedded NUL etc.
894 if (ltdb->cache->GUID_index_attribute != NULL) {
895 if (strcmp(attr, LTDB_IDXDN) == 0) {
896 should_b64_encode = false;
897 } else if (strcmp(attr, LTDB_IDXONE) == 0) {
899 * We can only change the behaviour for IDXONE
900 * when the GUID index is enabled
902 should_b64_encode = false;
905 = ldb_should_b64_encode(ldb, &v);
908 should_b64_encode = ldb_should_b64_encode(ldb, &v);
911 if (should_b64_encode) {
912 unsigned vstr_len = 0;
913 char *vstr = ldb_base64_encode(ldb, (char *)v.data, v.length);
915 talloc_free(attr_folded);
918 vstr_len = strlen(vstr);
919 key_len = 3 + indx_len + attr_len + vstr_len;
920 if (key_len > max_key_length) {
921 unsigned excess = key_len - max_key_length;
922 frmt_len = vstr_len - excess;
923 *truncation = KEY_TRUNCATED;
925 * Truncated keys are placed in a separate key space
926 * from the non truncated keys
927 * Note: the double hash "##" is not a typo and
928 * indicates that the following value is base64 encoded
930 ret = ldb_dn_new_fmt(ldb, ldb, "%s#%s##%.*s",
931 LTDB_INDEX, attr_for_dn,
935 *truncation = KEY_NOT_TRUNCATED;
937 * Note: the double colon "::" is not a typo and
938 * indicates that the following value is base64 encoded
940 ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s::%.*s",
941 LTDB_INDEX, attr_for_dn,
946 key_len = 2 + indx_len + attr_len + (int)v.length;
947 if (key_len > max_key_length) {
948 unsigned excess = key_len - max_key_length;
949 frmt_len = v.length - excess;
950 *truncation = KEY_TRUNCATED;
952 * Truncated keys are placed in a separate key space
953 * from the non truncated keys
955 ret = ldb_dn_new_fmt(ldb, ldb, "%s#%s#%.*s",
956 LTDB_INDEX, attr_for_dn,
957 frmt_len, (char *)v.data);
960 *truncation = KEY_NOT_TRUNCATED;
961 ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s:%.*s",
962 LTDB_INDEX, attr_for_dn,
963 frmt_len, (char *)v.data);
967 if (v.data != value->data) {
970 talloc_free(attr_folded);
976 see if a attribute value is in the list of indexed attributes
978 static bool ltdb_is_indexed(struct ldb_module *module,
979 struct ltdb_private *ltdb,
982 struct ldb_context *ldb = ldb_module_get_ctx(module);
984 struct ldb_message_element *el;
986 if ((ltdb->cache->GUID_index_attribute != NULL) &&
988 ltdb->cache->GUID_index_attribute) == 0)) {
989 /* Implicity covered, this is the index key */
992 if (ldb->schema.index_handler_override) {
993 const struct ldb_schema_attribute *a
994 = ldb_schema_attribute_by_name(ldb, attr);
1000 if (a->flags & LDB_ATTR_FLAG_INDEXED) {
1007 if (!ltdb->cache->attribute_indexes) {
1011 el = ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR);
1016 /* TODO: this is too expensive! At least use a binary search */
1017 for (i=0; i<el->num_values; i++) {
1018 if (ldb_attr_cmp((char *)el->values[i].data, attr) == 0) {
1026 in the following logic functions, the return value is treated as
1029 LDB_SUCCESS: we found some matching index values
1031 LDB_ERR_NO_SUCH_OBJECT: we know for sure that no object matches
1033 LDB_ERR_OPERATIONS_ERROR: indexing could not answer the call,
1034 we'll need a full search
1038 return a list of dn's that might match a simple indexed search (an
1039 equality search only)
1041 static int ltdb_index_dn_simple(struct ldb_module *module,
1042 struct ltdb_private *ltdb,
1043 const struct ldb_parse_tree *tree,
1044 struct dn_list *list)
1046 struct ldb_context *ldb;
1049 enum key_truncation truncation = KEY_NOT_TRUNCATED;
1051 ldb = ldb_module_get_ctx(module);
1056 /* if the attribute isn't in the list of indexed attributes then
1057 this node needs a full search */
1058 if (!ltdb_is_indexed(module, ltdb, tree->u.equality.attr)) {
1059 return LDB_ERR_OPERATIONS_ERROR;
1062 /* the attribute is indexed. Pull the list of DNs that match the
1064 dn = ltdb_index_key(ldb, ltdb,
1065 tree->u.equality.attr,
1066 &tree->u.equality.value, NULL, &truncation);
1068 * We ignore truncation here and allow multi-valued matches
1069 * as ltdb_search_indexed will filter out the wrong one in
1070 * ltdb_index_filter() which calls ldb_match_message().
1072 if (!dn) return LDB_ERR_OPERATIONS_ERROR;
1074 ret = ltdb_dn_list_load(module, ltdb, dn, list);
1080 static bool list_union(struct ldb_context *ldb,
1081 struct ltdb_private *ltdb,
1082 struct dn_list *list, struct dn_list *list2);
1085 return a list of dn's that might match a leaf indexed search
1087 static int ltdb_index_dn_leaf(struct ldb_module *module,
1088 struct ltdb_private *ltdb,
1089 const struct ldb_parse_tree *tree,
1090 struct dn_list *list)
1092 if (ltdb->disallow_dn_filter &&
1093 (ldb_attr_cmp(tree->u.equality.attr, "dn") == 0)) {
1094 /* in AD mode we do not support "(dn=...)" search filters */
1099 if (tree->u.equality.attr[0] == '@') {
1100 /* Do not allow a indexed search against an @ */
1105 if (ldb_attr_dn(tree->u.equality.attr) == 0) {
1106 enum key_truncation truncation = KEY_NOT_TRUNCATED;
1108 = ldb_dn_from_ldb_val(list,
1109 ldb_module_get_ctx(module),
1110 &tree->u.equality.value);
1112 /* If we can't parse it, no match */
1119 * Re-use the same code we use for a SCOPE_BASE
1122 * We can't call TALLOC_FREE(dn) as this must belong
1123 * to list for the memory to remain valid.
1125 return ltdb_index_dn_base_dn(module, ltdb, dn, list,
1128 * We ignore truncation here and allow multi-valued matches
1129 * as ltdb_search_indexed will filter out the wrong one in
1130 * ltdb_index_filter() which calls ldb_match_message().
1133 } else if ((ltdb->cache->GUID_index_attribute != NULL) &&
1134 (ldb_attr_cmp(tree->u.equality.attr,
1135 ltdb->cache->GUID_index_attribute) == 0)) {
1137 struct ldb_context *ldb = ldb_module_get_ctx(module);
1138 list->dn = talloc_array(list, struct ldb_val, 1);
1139 if (list->dn == NULL) {
1140 ldb_module_oom(module);
1141 return LDB_ERR_OPERATIONS_ERROR;
1144 * We need to go via the canonicalise_fn() to
1145 * ensure we get the index in binary, rather
1148 ret = ltdb->GUID_index_syntax->canonicalise_fn(ldb,
1150 &tree->u.equality.value,
1152 if (ret != LDB_SUCCESS) {
1153 return LDB_ERR_OPERATIONS_ERROR;
1159 return ltdb_index_dn_simple(module, ltdb, tree, list);
1167 static bool list_intersect(struct ldb_context *ldb,
1168 struct ltdb_private *ltdb,
1169 struct dn_list *list, const struct dn_list *list2)
1171 const struct dn_list *short_list, *long_list;
1172 struct dn_list *list3;
1175 if (list->count == 0) {
1179 if (list2->count == 0) {
1186 /* the indexing code is allowed to return a longer list than
1187 what really matches, as all results are filtered by the
1188 full expression at the end - this shortcut avoids a lot of
1189 work in some cases */
1190 if (list->count < 2 && list2->count > 10 && list2->strict == false) {
1193 if (list2->count < 2 && list->count > 10 && list->strict == false) {
1194 list->count = list2->count;
1195 list->dn = list2->dn;
1196 /* note that list2 may not be the parent of list2->dn,
1197 as list2->dn may be owned by ltdb->idxptr. In that
1198 case we expect this reparent call to fail, which is
1200 talloc_reparent(list2, list, list2->dn);
1204 if (list->count > list2->count) {
1212 list3 = talloc_zero(list, struct dn_list);
1213 if (list3 == NULL) {
1217 list3->dn = talloc_array(list3, struct ldb_val,
1218 MIN(list->count, list2->count));
1225 for (i=0;i<short_list->count;i++) {
1226 /* For the GUID index case, this is a binary search */
1227 if (ltdb_dn_list_find_val(ltdb, long_list,
1228 &short_list->dn[i]) != -1) {
1229 list3->dn[list3->count] = short_list->dn[i];
1234 list->strict |= list2->strict;
1235 list->dn = talloc_steal(list, list3->dn);
1236 list->count = list3->count;
1247 static bool list_union(struct ldb_context *ldb,
1248 struct ltdb_private *ltdb,
1249 struct dn_list *list, struct dn_list *list2)
1251 struct ldb_val *dn3;
1252 unsigned int i = 0, j = 0, k = 0;
1254 if (list2->count == 0) {
1259 if (list->count == 0) {
1261 list->count = list2->count;
1262 list->dn = list2->dn;
1263 /* note that list2 may not be the parent of list2->dn,
1264 as list2->dn may be owned by ltdb->idxptr. In that
1265 case we expect this reparent call to fail, which is
1267 talloc_reparent(list2, list, list2->dn);
1272 * Sort the lists (if not in GUID DN mode) so we can do
1273 * the de-duplication during the merge
1275 ltdb_dn_list_sort(ltdb, list);
1276 ltdb_dn_list_sort(ltdb, list2);
1278 dn3 = talloc_array(list, struct ldb_val, list->count + list2->count);
1284 while (i < list->count || j < list2->count) {
1286 if (i >= list->count) {
1288 } else if (j >= list2->count) {
1291 cmp = ldb_val_equal_exact_ordered(list->dn[i],
1297 dn3[k] = list->dn[i];
1300 } else if (cmp > 0) {
1302 dn3[k] = list2->dn[j];
1306 /* Equal, take list */
1307 dn3[k] = list->dn[i];
1320 static int ltdb_index_dn(struct ldb_module *module,
1321 struct ltdb_private *ltdb,
1322 const struct ldb_parse_tree *tree,
1323 struct dn_list *list);
1327 process an OR list (a union)
1329 static int ltdb_index_dn_or(struct ldb_module *module,
1330 struct ltdb_private *ltdb,
1331 const struct ldb_parse_tree *tree,
1332 struct dn_list *list)
1334 struct ldb_context *ldb;
1337 ldb = ldb_module_get_ctx(module);
1342 for (i=0; i<tree->u.list.num_elements; i++) {
1343 struct dn_list *list2;
1346 list2 = talloc_zero(list, struct dn_list);
1347 if (list2 == NULL) {
1348 return LDB_ERR_OPERATIONS_ERROR;
1351 ret = ltdb_index_dn(module, ltdb,
1352 tree->u.list.elements[i], list2);
1354 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
1360 if (ret != LDB_SUCCESS) {
1366 if (!list_union(ldb, ltdb, list, list2)) {
1368 return LDB_ERR_OPERATIONS_ERROR;
1372 if (list->count == 0) {
1373 return LDB_ERR_NO_SUCH_OBJECT;
1381 NOT an index results
1383 static int ltdb_index_dn_not(struct ldb_module *module,
1384 struct ltdb_private *ltdb,
1385 const struct ldb_parse_tree *tree,
1386 struct dn_list *list)
1388 /* the only way to do an indexed not would be if we could
1389 negate the not via another not or if we knew the total
1390 number of database elements so we could know that the
1391 existing expression covered the whole database.
1393 instead, we just give up, and rely on a full index scan
1394 (unless an outer & manages to reduce the list)
1396 return LDB_ERR_OPERATIONS_ERROR;
1400 * These things are unique, so avoid a full scan if this is a search
1401 * by GUID, DN or a unique attribute
1403 static bool ltdb_index_unique(struct ldb_context *ldb,
1404 struct ltdb_private *ltdb,
1407 const struct ldb_schema_attribute *a;
1408 if (ltdb->cache->GUID_index_attribute != NULL) {
1409 if (ldb_attr_cmp(attr, ltdb->cache->GUID_index_attribute) == 0) {
1413 if (ldb_attr_dn(attr) == 0) {
1417 a = ldb_schema_attribute_by_name(ldb, attr);
1418 if (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX) {
1425 process an AND expression (intersection)
1427 static int ltdb_index_dn_and(struct ldb_module *module,
1428 struct ltdb_private *ltdb,
1429 const struct ldb_parse_tree *tree,
1430 struct dn_list *list)
1432 struct ldb_context *ldb;
1436 ldb = ldb_module_get_ctx(module);
1441 /* in the first pass we only look for unique simple
1442 equality tests, in the hope of avoiding having to look
1444 for (i=0; i<tree->u.list.num_elements; i++) {
1445 const struct ldb_parse_tree *subtree = tree->u.list.elements[i];
1448 if (subtree->operation != LDB_OP_EQUALITY ||
1449 !ltdb_index_unique(ldb, ltdb,
1450 subtree->u.equality.attr)) {
1454 ret = ltdb_index_dn(module, ltdb, subtree, list);
1455 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
1457 return LDB_ERR_NO_SUCH_OBJECT;
1459 if (ret == LDB_SUCCESS) {
1460 /* a unique index match means we can
1461 * stop. Note that we don't care if we return
1462 * a few too many objects, due to later
1468 /* now do a full intersection */
1471 for (i=0; i<tree->u.list.num_elements; i++) {
1472 const struct ldb_parse_tree *subtree = tree->u.list.elements[i];
1473 struct dn_list *list2;
1476 list2 = talloc_zero(list, struct dn_list);
1477 if (list2 == NULL) {
1478 return ldb_module_oom(module);
1481 ret = ltdb_index_dn(module, ltdb, subtree, list2);
1483 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
1488 return LDB_ERR_NO_SUCH_OBJECT;
1491 if (ret != LDB_SUCCESS) {
1492 /* this didn't adding anything */
1498 talloc_reparent(list2, list, list->dn);
1499 list->dn = list2->dn;
1500 list->count = list2->count;
1502 } else if (!list_intersect(ldb, ltdb,
1505 return LDB_ERR_OPERATIONS_ERROR;
1508 if (list->count == 0) {
1510 return LDB_ERR_NO_SUCH_OBJECT;
1513 if (list->count < 2) {
1514 /* it isn't worth loading the next part of the tree */
1520 /* none of the attributes were indexed */
1521 return LDB_ERR_OPERATIONS_ERROR;
1528 return a list of matching objects using a one-level index
1530 static int ltdb_index_dn_attr(struct ldb_module *module,
1531 struct ltdb_private *ltdb,
1534 struct dn_list *list,
1535 enum key_truncation *truncation)
1537 struct ldb_context *ldb;
1542 ldb = ldb_module_get_ctx(module);
1544 /* work out the index key from the parent DN */
1545 val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
1546 val.length = strlen((char *)val.data);
1547 key = ltdb_index_key(ldb, ltdb, attr, &val, NULL, truncation);
1550 return LDB_ERR_OPERATIONS_ERROR;
1553 ret = ltdb_dn_list_load(module, ltdb, key, list);
1555 if (ret != LDB_SUCCESS) {
1559 if (list->count == 0) {
1560 return LDB_ERR_NO_SUCH_OBJECT;
1567 return a list of matching objects using a one-level index
1569 static int ltdb_index_dn_one(struct ldb_module *module,
1570 struct ltdb_private *ltdb,
1571 struct ldb_dn *parent_dn,
1572 struct dn_list *list,
1573 enum key_truncation *truncation)
1575 /* Ensure we do not shortcut on intersection for this list */
1576 list->strict = true;
1577 return ltdb_index_dn_attr(module, ltdb,
1578 LTDB_IDXONE, parent_dn, list, truncation);
1583 return a list of matching objects using the DN index
1585 static int ltdb_index_dn_base_dn(struct ldb_module *module,
1586 struct ltdb_private *ltdb,
1587 struct ldb_dn *base_dn,
1588 struct dn_list *dn_list,
1589 enum key_truncation *truncation)
1591 const struct ldb_val *guid_val = NULL;
1592 if (ltdb->cache->GUID_index_attribute == NULL) {
1593 dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
1594 if (dn_list->dn == NULL) {
1595 return ldb_module_oom(module);
1597 dn_list->dn[0].data = discard_const_p(unsigned char,
1598 ldb_dn_get_linearized(base_dn));
1599 if (dn_list->dn[0].data == NULL) {
1600 return ldb_module_oom(module);
1602 dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
1608 if (ltdb->cache->GUID_index_dn_component != NULL) {
1609 guid_val = ldb_dn_get_extended_component(base_dn,
1610 ltdb->cache->GUID_index_dn_component);
1613 if (guid_val != NULL) {
1614 dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
1615 if (dn_list->dn == NULL) {
1616 return ldb_module_oom(module);
1618 dn_list->dn[0].data = guid_val->data;
1619 dn_list->dn[0].length = guid_val->length;
1625 return ltdb_index_dn_attr(module, ltdb,
1626 LTDB_IDXDN, base_dn, dn_list, truncation);
1630 return a list of dn's that might match a indexed search or
1631 an error. return LDB_ERR_NO_SUCH_OBJECT for no matches, or LDB_SUCCESS for matches
1633 static int ltdb_index_dn(struct ldb_module *module,
1634 struct ltdb_private *ltdb,
1635 const struct ldb_parse_tree *tree,
1636 struct dn_list *list)
1638 int ret = LDB_ERR_OPERATIONS_ERROR;
1640 switch (tree->operation) {
1642 ret = ltdb_index_dn_and(module, ltdb, tree, list);
1646 ret = ltdb_index_dn_or(module, ltdb, tree, list);
1650 ret = ltdb_index_dn_not(module, ltdb, tree, list);
1653 case LDB_OP_EQUALITY:
1654 ret = ltdb_index_dn_leaf(module, ltdb, tree, list);
1657 case LDB_OP_SUBSTRING:
1658 case LDB_OP_GREATER:
1660 case LDB_OP_PRESENT:
1662 case LDB_OP_EXTENDED:
1663 /* we can't index with fancy bitops yet */
1664 ret = LDB_ERR_OPERATIONS_ERROR;
1672 filter a candidate dn_list from an indexed search into a set of results
1673 extracting just the given attributes
1675 static int ltdb_index_filter(struct ltdb_private *ltdb,
1676 const struct dn_list *dn_list,
1677 struct ltdb_context *ac,
1678 uint32_t *match_count,
1679 enum key_truncation scope_one_truncation)
1681 struct ldb_context *ldb;
1682 struct ldb_message *msg;
1683 struct ldb_message *filtered_msg;
1685 uint8_t previous_guid_key[LTDB_GUID_KEY_SIZE] = {};
1687 ldb = ldb_module_get_ctx(ac->module);
1689 for (i = 0; i < dn_list->count; i++) {
1690 uint8_t guid_key[LTDB_GUID_KEY_SIZE];
1691 TDB_DATA tdb_key = {
1693 .dsize = sizeof(guid_key)
1698 ret = ltdb_idx_to_key(ac->module, ltdb,
1699 ac, &dn_list->dn[i],
1701 if (ret != LDB_SUCCESS) {
1705 if (ltdb->cache->GUID_index_attribute != NULL) {
1707 * If we are in GUID index mode, then the dn_list is
1708 * sorted. If we got a duplicate, forget about it, as
1709 * otherwise we would send the same entry back more
1712 * This is needed in the truncated DN case, or if a
1713 * duplicate was forced in via
1714 * LDB_FLAG_INTERNAL_DISABLE_SINGLE_VALUE_CHECK
1717 if (memcmp(previous_guid_key, tdb_key.dptr,
1718 sizeof(previous_guid_key)) == 0) {
1722 memcpy(previous_guid_key, tdb_key.dptr,
1723 sizeof(previous_guid_key));
1726 msg = ldb_msg_new(ac);
1728 return LDB_ERR_OPERATIONS_ERROR;
1732 ret = ltdb_search_key(ac->module, ltdb,
1734 LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
1735 LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
1736 if (tdb_key.dptr != guid_key) {
1737 TALLOC_FREE(tdb_key.dptr);
1739 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
1740 /* the record has disappeared? yes, this can happen */
1745 if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
1746 /* an internal error */
1748 return LDB_ERR_OPERATIONS_ERROR;
1752 * We trust the index for LDB_SCOPE_ONELEVEL
1753 * unless the index key has been truncated.
1755 * LDB_SCOPE_BASE is not passed in by our only caller.
1757 if (ac->scope == LDB_SCOPE_ONELEVEL
1758 && ltdb->cache->one_level_indexes
1759 && scope_one_truncation == KEY_NOT_TRUNCATED) {
1760 ret = ldb_match_message(ldb, msg, ac->tree,
1761 ac->scope, &matched);
1763 ret = ldb_match_msg_error(ldb, msg,
1765 ac->scope, &matched);
1768 if (ret != LDB_SUCCESS) {
1777 /* filter the attributes that the user wants */
1778 ret = ltdb_filter_attrs(ac, msg, ac->attrs, &filtered_msg);
1783 return LDB_ERR_OPERATIONS_ERROR;
1786 ret = ldb_module_send_entry(ac->req, filtered_msg, NULL);
1787 if (ret != LDB_SUCCESS) {
1788 /* Regardless of success or failure, the msg
1789 * is the callbacks responsiblity, and should
1790 * not be talloc_free()'ed */
1791 ac->request_terminated = true;
1804 static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
1805 struct dn_list *list)
1807 if (list->count < 2) {
1811 /* We know the list is sorted when using the GUID index */
1812 if (ltdb->cache->GUID_index_attribute != NULL) {
1816 TYPESAFE_QSORT(list->dn, list->count,
1817 ldb_val_equal_exact_for_qsort);
1821 search the database with a LDAP-like expression using indexes
1822 returns -1 if an indexed search is not possible, in which
1823 case the caller should call ltdb_search_full()
1825 int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
1827 struct ldb_context *ldb = ldb_module_get_ctx(ac->module);
1828 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(ac->module), struct ltdb_private);
1829 struct dn_list *dn_list;
1831 enum ldb_scope index_scope;
1832 enum key_truncation scope_one_truncation = KEY_NOT_TRUNCATED;
1834 /* see if indexing is enabled */
1835 if (!ltdb->cache->attribute_indexes &&
1836 !ltdb->cache->one_level_indexes &&
1837 ac->scope != LDB_SCOPE_BASE) {
1838 /* fallback to a full search */
1839 return LDB_ERR_OPERATIONS_ERROR;
1842 dn_list = talloc_zero(ac, struct dn_list);
1843 if (dn_list == NULL) {
1844 return ldb_module_oom(ac->module);
1848 * For the purposes of selecting the switch arm below, if we
1849 * don't have a one-level index then treat it like a subtree
1852 if (ac->scope == LDB_SCOPE_ONELEVEL &&
1853 !ltdb->cache->one_level_indexes) {
1854 index_scope = LDB_SCOPE_SUBTREE;
1856 index_scope = ac->scope;
1859 switch (index_scope) {
1860 case LDB_SCOPE_BASE:
1862 * The only caller will have filtered the operation out
1863 * so we should never get here
1865 return ldb_operr(ldb);
1867 case LDB_SCOPE_ONELEVEL:
1869 * If we ever start to also load the index values for
1870 * the tree, we must ensure we strictly intersect with
1871 * this list, as we trust the ONELEVEL index
1873 ret = ltdb_index_dn_one(ac->module, ltdb, ac->base, dn_list,
1874 &scope_one_truncation);
1875 if (ret != LDB_SUCCESS) {
1876 talloc_free(dn_list);
1881 * If we have too many matches, running the filter
1882 * tree over the SCOPE_ONELEVEL can be quite expensive
1883 * so we now check the filter tree index as well.
1885 * We only do this in the GUID index mode, which is
1886 * O(n*log(m)) otherwise the intersection below will
1887 * be too costly at O(n*m).
1889 * We don't set a heuristic for 'too many' but instead
1890 * do it always and rely on the index lookup being
1891 * fast enough in the small case.
1893 if (ltdb->cache->GUID_index_attribute != NULL) {
1894 struct dn_list *idx_one_tree_list
1895 = talloc_zero(ac, struct dn_list);
1896 if (idx_one_tree_list == NULL) {
1897 return ldb_module_oom(ac->module);
1900 if (!ltdb->cache->attribute_indexes) {
1901 talloc_free(idx_one_tree_list);
1902 talloc_free(dn_list);
1903 return LDB_ERR_OPERATIONS_ERROR;
1906 * Here we load the index for the tree.
1908 ret = ltdb_index_dn(ac->module, ltdb, ac->tree,
1910 if (ret != LDB_SUCCESS) {
1911 talloc_free(idx_one_tree_list);
1912 talloc_free(dn_list);
1916 if (!list_intersect(ldb, ltdb,
1917 dn_list, idx_one_tree_list)) {
1918 talloc_free(idx_one_tree_list);
1919 talloc_free(dn_list);
1920 return LDB_ERR_OPERATIONS_ERROR;
1925 case LDB_SCOPE_SUBTREE:
1926 case LDB_SCOPE_DEFAULT:
1927 if (!ltdb->cache->attribute_indexes) {
1928 talloc_free(dn_list);
1929 return LDB_ERR_OPERATIONS_ERROR;
1932 * Here we load the index for the tree. We have no
1933 * index for the subtree.
1935 ret = ltdb_index_dn(ac->module, ltdb, ac->tree, dn_list);
1936 if (ret != LDB_SUCCESS) {
1937 talloc_free(dn_list);
1944 * It is critical that this function do the re-filter even
1945 * on things found by the index as the index can over-match
1946 * in cases of truncation (as well as when it decides it is
1947 * not worth further filtering)
1949 * If this changes, then the index code above would need to
1950 * pass up a flag to say if any index was truncated during
1951 * processing as the truncation here refers only to the
1952 * SCOPE_ONELEVEL index.
1954 ret = ltdb_index_filter(ltdb, dn_list, ac, match_count,
1955 scope_one_truncation);
1956 talloc_free(dn_list);
1961 * @brief Add a DN in the index list of a given attribute name/value pair
1963 * This function will add the DN in the index list for the index for
1964 * the given attribute name and value.
1966 * @param[in] module A ldb_module structure
1968 * @param[in] dn The string representation of the DN as it
1969 * will be stored in the index entry
1971 * @param[in] el A ldb_message_element array, one of the entry
1972 * referred by the v_idx is the attribute name and
1973 * value pair which will be used to construct the
1976 * @param[in] v_idx The index of element in the el array to use
1978 * @return An ldb error code
1980 static int ltdb_index_add1(struct ldb_module *module,
1981 struct ltdb_private *ltdb,
1982 const struct ldb_message *msg,
1983 struct ldb_message_element *el, int v_idx)
1985 struct ldb_context *ldb;
1986 struct ldb_dn *dn_key;
1988 const struct ldb_schema_attribute *a;
1989 struct dn_list *list;
1991 enum key_truncation truncation = KEY_TRUNCATED;
1994 ldb = ldb_module_get_ctx(module);
1996 list = talloc_zero(module, struct dn_list);
1998 return LDB_ERR_OPERATIONS_ERROR;
2001 dn_key = ltdb_index_key(ldb, ltdb,
2002 el->name, &el->values[v_idx], &a, &truncation);
2005 return LDB_ERR_OPERATIONS_ERROR;
2008 * Samba only maintains unique indexes on the objectSID and objectGUID
2009 * so if a unique index key exceeds the maximum length there is a
2012 if ((truncation == KEY_TRUNCATED) && (a != NULL &&
2013 (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX ||
2014 (el->flags & LDB_FLAG_INTERNAL_FORCE_UNIQUE_INDEX)))) {
2016 ldb_asprintf_errstring(
2018 __location__ ": unique index key on %s in %s, "
2019 "exceeds maximum key length of %u (encoded).",
2021 ldb_dn_get_linearized(msg->dn),
2022 ltdb->max_key_length);
2024 return LDB_ERR_CONSTRAINT_VIOLATION;
2026 talloc_steal(list, dn_key);
2028 ret = ltdb_dn_list_load(module, ltdb, dn_key, list);
2029 if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
2035 * Check for duplicates in the @IDXDN DN -> GUID record
2037 * This is very normal, it just means a duplicate DN creation
2038 * was attempted, so don't set the error string or print scary
2041 if (list->count > 0 &&
2042 ldb_attr_cmp(el->name, LTDB_IDXDN) == 0 &&
2043 truncation == KEY_NOT_TRUNCATED) {
2046 return LDB_ERR_CONSTRAINT_VIOLATION;
2048 } else if (list->count > 0
2049 && ldb_attr_cmp(el->name, LTDB_IDXDN) == 0) {
2052 * At least one existing entry in the DN->GUID index, which
2053 * arises when the DN indexes have been truncated
2055 * So need to pull the DN's to check if it's really a duplicate
2058 for (i=0; i < list->count; i++) {
2059 uint8_t guid_key[LTDB_GUID_KEY_SIZE];
2062 .dsize = sizeof(guid_key)
2064 const int flags = LDB_UNPACK_DATA_FLAG_NO_ATTRS;
2065 struct ldb_message *rec = ldb_msg_new(ldb);
2067 return LDB_ERR_OPERATIONS_ERROR;
2070 ret = ltdb_idx_to_key(module, ltdb,
2073 if (ret != LDB_SUCCESS) {
2079 ret = ltdb_search_key(module, ltdb, key,
2081 if (key.dptr != guid_key) {
2082 TALLOC_FREE(key.dptr);
2084 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
2086 * the record has disappeared?
2087 * yes, this can happen
2093 if (ret != LDB_SUCCESS) {
2094 /* an internal error */
2097 return LDB_ERR_OPERATIONS_ERROR;
2100 * The DN we are trying to add to the DB and index
2101 * is already here, so we must deny the addition
2103 if (ldb_dn_compare(msg->dn, rec->dn) == 0) {
2106 return LDB_ERR_CONSTRAINT_VIOLATION;
2112 * Check for duplicates in unique indexes
2114 * We don't need to do a loop test like the @IDXDN case
2115 * above as we have a ban on long unique index values
2116 * at the start of this function.
2118 if (list->count > 0 &&
2120 && (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX ||
2121 (el->flags & LDB_FLAG_INTERNAL_FORCE_UNIQUE_INDEX))))) {
2123 * We do not want to print info about a possibly
2124 * confidential DN that the conflict was with in the
2125 * user-visible error string
2128 if (ltdb->cache->GUID_index_attribute == NULL) {
2129 ldb_debug(ldb, LDB_DEBUG_WARNING,
2131 ": unique index violation on %s in %s, "
2132 "conficts with %*.*s in %s",
2133 el->name, ldb_dn_get_linearized(msg->dn),
2134 (int)list->dn[0].length,
2135 (int)list->dn[0].length,
2137 ldb_dn_get_linearized(dn_key));
2139 /* This can't fail, gives a default at worst */
2140 const struct ldb_schema_attribute *attr
2141 = ldb_schema_attribute_by_name(
2143 ltdb->cache->GUID_index_attribute);
2145 ret = attr->syntax->ldif_write_fn(ldb, list,
2147 if (ret == LDB_SUCCESS) {
2148 ldb_debug(ldb, LDB_DEBUG_WARNING,
2150 ": unique index violation on %s in "
2151 "%s, conficts with %s %*.*s in %s",
2153 ldb_dn_get_linearized(msg->dn),
2154 ltdb->cache->GUID_index_attribute,
2158 ldb_dn_get_linearized(dn_key));
2161 ldb_asprintf_errstring(ldb,
2162 __location__ ": unique index violation "
2165 ldb_dn_get_linearized(msg->dn));
2167 return LDB_ERR_CONSTRAINT_VIOLATION;
2170 /* overallocate the list a bit, to reduce the number of
2171 * realloc trigered copies */
2172 alloc_len = ((list->count+1)+7) & ~7;
2173 list->dn = talloc_realloc(list, list->dn, struct ldb_val, alloc_len);
2174 if (list->dn == NULL) {
2176 return LDB_ERR_OPERATIONS_ERROR;
2179 if (ltdb->cache->GUID_index_attribute == NULL) {
2180 const char *dn_str = ldb_dn_get_linearized(msg->dn);
2181 list->dn[list->count].data
2182 = (uint8_t *)talloc_strdup(list->dn, dn_str);
2183 if (list->dn[list->count].data == NULL) {
2185 return LDB_ERR_OPERATIONS_ERROR;
2187 list->dn[list->count].length = strlen(dn_str);
2189 const struct ldb_val *key_val;
2190 struct ldb_val *exact = NULL, *next = NULL;
2191 key_val = ldb_msg_find_ldb_val(msg,
2192 ltdb->cache->GUID_index_attribute);
2193 if (key_val == NULL) {
2195 return ldb_module_operr(module);
2198 if (key_val->length != LTDB_GUID_SIZE) {
2200 return ldb_module_operr(module);
2203 BINARY_ARRAY_SEARCH_GTE(list->dn, list->count,
2204 *key_val, ldb_val_equal_exact_ordered,
2208 * Give a warning rather than fail, this could be a
2209 * duplicate value in the record allowed by a caller
2210 * forcing in the value with
2211 * LDB_FLAG_INTERNAL_DISABLE_SINGLE_VALUE_CHECK
2213 if (exact != NULL && truncation == KEY_NOT_TRUNCATED) {
2214 /* This can't fail, gives a default at worst */
2215 const struct ldb_schema_attribute *attr
2216 = ldb_schema_attribute_by_name(
2218 ltdb->cache->GUID_index_attribute);
2220 ret = attr->syntax->ldif_write_fn(ldb, list,
2222 if (ret == LDB_SUCCESS) {
2223 ldb_debug(ldb, LDB_DEBUG_WARNING,
2225 ": duplicate attribute value in %s "
2227 "duplicate of %s %*.*s in %s",
2228 ldb_dn_get_linearized(msg->dn),
2230 ltdb->cache->GUID_index_attribute,
2234 ldb_dn_get_linearized(dn_key));
2239 next = &list->dn[list->count];
2241 memmove(&next[1], next,
2242 sizeof(*next) * (list->count - (next - list->dn)));
2244 *next = ldb_val_dup(list->dn, key_val);
2245 if (next->data == NULL) {
2247 return ldb_module_operr(module);
2252 ret = ltdb_dn_list_store(module, dn_key, list);
2260 add index entries for one elements in a message
2262 static int ltdb_index_add_el(struct ldb_module *module,
2263 struct ltdb_private *ltdb,
2264 const struct ldb_message *msg,
2265 struct ldb_message_element *el)
2268 for (i = 0; i < el->num_values; i++) {
2269 int ret = ltdb_index_add1(module, ltdb,
2271 if (ret != LDB_SUCCESS) {
2280 add index entries for all elements in a message
2282 static int ltdb_index_add_all(struct ldb_module *module,
2283 struct ltdb_private *ltdb,
2284 const struct ldb_message *msg)
2286 struct ldb_message_element *elements = msg->elements;
2291 if (ldb_dn_is_special(msg->dn)) {
2295 dn_str = ldb_dn_get_linearized(msg->dn);
2296 if (dn_str == NULL) {
2297 return LDB_ERR_OPERATIONS_ERROR;
2300 ret = ltdb_write_index_dn_guid(module, msg, 1);
2301 if (ret != LDB_SUCCESS) {
2305 if (!ltdb->cache->attribute_indexes) {
2306 /* no indexed fields */
2310 for (i = 0; i < msg->num_elements; i++) {
2311 if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
2314 ret = ltdb_index_add_el(module, ltdb,
2316 if (ret != LDB_SUCCESS) {
2317 struct ldb_context *ldb = ldb_module_get_ctx(module);
2318 ldb_asprintf_errstring(ldb,
2319 __location__ ": Failed to re-index %s in %s - %s",
2320 elements[i].name, dn_str,
2321 ldb_errstring(ldb));
2331 insert a DN index for a message
2333 static int ltdb_modify_index_dn(struct ldb_module *module,
2334 struct ltdb_private *ltdb,
2335 const struct ldb_message *msg,
2337 const char *index, int add)
2339 struct ldb_message_element el;
2343 val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
2344 if (val.data == NULL) {
2345 const char *dn_str = ldb_dn_get_linearized(dn);
2346 ldb_asprintf_errstring(ldb_module_get_ctx(module),
2348 ": Failed to modify %s "
2349 "against %s in %s: failed "
2350 "to get casefold DN",
2352 ltdb->cache->GUID_index_attribute,
2354 return LDB_ERR_OPERATIONS_ERROR;
2357 val.length = strlen((char *)val.data);
2363 ret = ltdb_index_add1(module, ltdb, msg, &el, 0);
2364 } else { /* delete */
2365 ret = ltdb_index_del_value(module, ltdb, msg, &el, 0);
2368 if (ret != LDB_SUCCESS) {
2369 struct ldb_context *ldb = ldb_module_get_ctx(module);
2370 const char *dn_str = ldb_dn_get_linearized(dn);
2371 ldb_asprintf_errstring(ldb,
2373 ": Failed to modify %s "
2374 "against %s in %s - %s",
2376 ltdb->cache->GUID_index_attribute,
2377 dn_str, ldb_errstring(ldb));
2384 insert a one level index for a message
2386 static int ltdb_index_onelevel(struct ldb_module *module,
2387 const struct ldb_message *msg, int add)
2389 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
2390 struct ltdb_private);
2394 /* We index for ONE Level only if requested */
2395 if (!ltdb->cache->one_level_indexes) {
2399 pdn = ldb_dn_get_parent(module, msg->dn);
2401 return LDB_ERR_OPERATIONS_ERROR;
2403 ret = ltdb_modify_index_dn(module, ltdb,
2404 msg, pdn, LTDB_IDXONE, add);
2412 insert a one level index for a message
2414 static int ltdb_write_index_dn_guid(struct ldb_module *module,
2415 const struct ldb_message *msg,
2419 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
2420 struct ltdb_private);
2422 /* We index for DN only if using a GUID index */
2423 if (ltdb->cache->GUID_index_attribute == NULL) {
2427 ret = ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
2430 if (ret == LDB_ERR_CONSTRAINT_VIOLATION) {
2431 ldb_asprintf_errstring(ldb_module_get_ctx(module),
2432 "Entry %s already exists",
2433 ldb_dn_get_linearized(msg->dn));
2434 ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
2440 add the index entries for a new element in a record
2441 The caller guarantees that these element values are not yet indexed
2443 int ltdb_index_add_element(struct ldb_module *module,
2444 struct ltdb_private *ltdb,
2445 const struct ldb_message *msg,
2446 struct ldb_message_element *el)
2448 if (ldb_dn_is_special(msg->dn)) {
2451 if (!ltdb_is_indexed(module, ltdb, el->name)) {
2454 return ltdb_index_add_el(module, ltdb, msg, el);
2458 add the index entries for a new record
2460 int ltdb_index_add_new(struct ldb_module *module,
2461 struct ltdb_private *ltdb,
2462 const struct ldb_message *msg)
2466 if (ldb_dn_is_special(msg->dn)) {
2470 ret = ltdb_index_add_all(module, ltdb, msg);
2471 if (ret != LDB_SUCCESS) {
2473 * Because we can't trust the caller to be doing
2474 * transactions properly, clean up any index for this
2475 * entry rather than relying on a transaction
2479 ltdb_index_delete(module, msg);
2483 ret = ltdb_index_onelevel(module, msg, 1);
2484 if (ret != LDB_SUCCESS) {
2486 * Because we can't trust the caller to be doing
2487 * transactions properly, clean up any index for this
2488 * entry rather than relying on a transaction
2491 ltdb_index_delete(module, msg);
2499 delete an index entry for one message element
2501 int ltdb_index_del_value(struct ldb_module *module,
2502 struct ltdb_private *ltdb,
2503 const struct ldb_message *msg,
2504 struct ldb_message_element *el, unsigned int v_idx)
2506 struct ldb_context *ldb;
2507 struct ldb_dn *dn_key;
2511 struct dn_list *list;
2512 struct ldb_dn *dn = msg->dn;
2513 enum key_truncation truncation = KEY_NOT_TRUNCATED;
2515 ldb = ldb_module_get_ctx(module);
2517 dn_str = ldb_dn_get_linearized(dn);
2518 if (dn_str == NULL) {
2519 return LDB_ERR_OPERATIONS_ERROR;
2522 if (dn_str[0] == '@') {
2526 dn_key = ltdb_index_key(ldb, ltdb,
2527 el->name, &el->values[v_idx],
2530 * We ignore key truncation in ltdb_index_add1() so
2531 * match that by ignoring it here as well
2533 * Multiple values are legitimate and accepted
2536 return LDB_ERR_OPERATIONS_ERROR;
2539 list = talloc_zero(dn_key, struct dn_list);
2541 talloc_free(dn_key);
2542 return LDB_ERR_OPERATIONS_ERROR;
2545 ret = ltdb_dn_list_load(module, ltdb, dn_key, list);
2546 if (ret == LDB_ERR_NO_SUCH_OBJECT) {
2547 /* it wasn't indexed. Did we have an earlier error? If we did then
2549 talloc_free(dn_key);
2553 if (ret != LDB_SUCCESS) {
2554 talloc_free(dn_key);
2559 * Find one of the values matching this message to remove
2561 i = ltdb_dn_list_find_msg(ltdb, list, msg);
2563 /* nothing to delete */
2564 talloc_free(dn_key);
2568 j = (unsigned int) i;
2569 if (j != list->count - 1) {
2570 memmove(&list->dn[j], &list->dn[j+1], sizeof(list->dn[0])*(list->count - (j+1)));
2573 if (list->count == 0) {
2574 talloc_free(list->dn);
2577 list->dn = talloc_realloc(list, list->dn, struct ldb_val, list->count);
2580 ret = ltdb_dn_list_store(module, dn_key, list);
2582 talloc_free(dn_key);
2588 delete the index entries for a element
2589 return -1 on failure
2591 int ltdb_index_del_element(struct ldb_module *module,
2592 struct ltdb_private *ltdb,
2593 const struct ldb_message *msg,
2594 struct ldb_message_element *el)
2600 if (!ltdb->cache->attribute_indexes) {
2601 /* no indexed fields */
2605 dn_str = ldb_dn_get_linearized(msg->dn);
2606 if (dn_str == NULL) {
2607 return LDB_ERR_OPERATIONS_ERROR;
2610 if (dn_str[0] == '@') {
2614 if (!ltdb_is_indexed(module, ltdb, el->name)) {
2617 for (i = 0; i < el->num_values; i++) {
2618 ret = ltdb_index_del_value(module, ltdb, msg, el, i);
2619 if (ret != LDB_SUCCESS) {
2628 delete the index entries for a record
2629 return -1 on failure
2631 int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
2633 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
2637 if (ldb_dn_is_special(msg->dn)) {
2641 ret = ltdb_index_onelevel(module, msg, 0);
2642 if (ret != LDB_SUCCESS) {
2646 ret = ltdb_write_index_dn_guid(module, msg, 0);
2647 if (ret != LDB_SUCCESS) {
2651 if (!ltdb->cache->attribute_indexes) {
2652 /* no indexed fields */
2656 for (i = 0; i < msg->num_elements; i++) {
2657 ret = ltdb_index_del_element(module, ltdb,
2658 msg, &msg->elements[i]);
2659 if (ret != LDB_SUCCESS) {
2669 traversal function that deletes all @INDEX records in the in-memory
2672 This does not touch the actual DB, that is done at transaction
2673 commit, which in turn greatly reduces DB churn as we will likely
2674 be able to do a direct update into the old record.
2676 static int delete_index(struct ltdb_private *ltdb, struct ldb_val key, struct ldb_val data, void *state)
2678 struct ldb_module *module = state;
2679 const char *dnstr = "DN=" LTDB_INDEX ":";
2680 struct dn_list list;
2685 if (strncmp((char *)key.data, dnstr, strlen(dnstr)) != 0) {
2688 /* we need to put a empty list in the internal tdb for this
2693 /* the offset of 3 is to remove the DN= prefix. */
2694 v.data = key.data + 3;
2695 v.length = strnlen((char *)key.data, key.length) - 3;
2697 dn = ldb_dn_from_ldb_val(ltdb, ldb_module_get_ctx(module), &v);
2700 * This does not actually touch the DB quite yet, just
2701 * the in-memory index cache
2703 ret = ltdb_dn_list_store(module, dn, &list);
2704 if (ret != LDB_SUCCESS) {
2705 ldb_asprintf_errstring(ldb_module_get_ctx(module),
2706 "Unable to store null index for %s\n",
2707 ldb_dn_get_linearized(dn));
2716 traversal function that adds @INDEX records during a re index TODO wrong comment
2718 static int re_key(struct ltdb_private *ltdb, struct ldb_val ldb_key, struct ldb_val val, void *state)
2720 struct ldb_context *ldb;
2721 struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
2722 struct ldb_module *module = ctx->module;
2723 struct ldb_message *msg;
2724 unsigned int nb_elements_in_db;
2729 .dptr = ldb_key.data,
2730 .dsize = ldb_key.length
2733 ldb = ldb_module_get_ctx(module);
2735 if (key.dsize > 4 &&
2736 memcmp(key.dptr, "DN=@", 4) == 0) {
2740 is_record = ltdb_key_is_record(key);
2741 if (is_record == false) {
2745 msg = ldb_msg_new(module);
2750 ret = ldb_unpack_data_only_attr_list_flags(ldb, &val,
2753 LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC,
2754 &nb_elements_in_db);
2756 ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %s\n",
2757 ldb_dn_get_linearized(msg->dn));
2763 if (msg->dn == NULL) {
2764 ldb_debug(ldb, LDB_DEBUG_ERROR,
2765 "Refusing to re-index as GUID "
2766 "key %*.*s with no DN\n",
2767 (int)key.dsize, (int)key.dsize,
2773 /* check if the DN key has changed, perhaps due to the case
2774 insensitivity of an element changing, or a change from DN
2776 key2 = ltdb_key_msg(module, msg, msg);
2777 if (key2.dptr == NULL) {
2778 /* probably a corrupt record ... darn */
2779 ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
2780 ldb_dn_get_linearized(msg->dn));
2784 if (key.dsize != key2.dsize ||
2785 (memcmp(key.dptr, key2.dptr, key.dsize) != 0)) {
2786 struct ldb_val ldb_key2 = {
2788 .length = key2.dsize
2790 ltdb->kv_ops->update_in_iterate(ltdb, ldb_key, ldb_key2, val, ctx);
2792 talloc_free(key2.dptr);
2797 if (ctx->count % 10000 == 0) {
2798 ldb_debug(ldb, LDB_DEBUG_WARNING,
2799 "Reindexing: re-keyed %u records so far",
2807 traversal function that adds @INDEX records during a re index
2809 static int re_index(struct ltdb_private *ltdb, struct ldb_val ldb_key, struct ldb_val val, void *state)
2811 struct ldb_context *ldb;
2812 struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
2813 struct ldb_module *module = ctx->module;
2814 struct ldb_message *msg;
2815 unsigned int nb_elements_in_db;
2817 .dptr = ldb_key.data,
2818 .dsize = ldb_key.length
2823 ldb = ldb_module_get_ctx(module);
2825 if (key.dsize > 4 &&
2826 memcmp(key.dptr, "DN=@", 4) == 0) {
2830 is_record = ltdb_key_is_record(key);
2831 if (is_record == false) {
2835 msg = ldb_msg_new(module);
2840 ret = ldb_unpack_data_only_attr_list_flags(ldb, &val,
2843 LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC,
2844 &nb_elements_in_db);
2846 ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %s\n",
2847 ldb_dn_get_linearized(msg->dn));
2853 if (msg->dn == NULL) {
2854 ldb_debug(ldb, LDB_DEBUG_ERROR,
2855 "Refusing to re-index as GUID "
2856 "key %*.*s with no DN\n",
2857 (int)key.dsize, (int)key.dsize,
2863 ret = ltdb_index_onelevel(module, msg, 1);
2864 if (ret != LDB_SUCCESS) {
2865 ldb_debug(ldb, LDB_DEBUG_ERROR,
2866 "Adding special ONE LEVEL index failed (%s)!",
2867 ldb_dn_get_linearized(msg->dn));
2872 ret = ltdb_index_add_all(module, ltdb, msg);
2874 if (ret != LDB_SUCCESS) {
2883 if (ctx->count % 10000 == 0) {
2884 ldb_debug(ldb, LDB_DEBUG_WARNING,
2885 "Reindexing: re-indexed %u records so far",
2893 force a complete reindex of the database
2895 int ltdb_reindex(struct ldb_module *module)
2897 struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
2899 struct ltdb_reindex_context ctx;
2902 * Only triggered after a modification, but make clear we do
2903 * not re-index a read-only DB
2905 if (ltdb->read_only) {
2906 return LDB_ERR_UNWILLING_TO_PERFORM;
2909 if (ltdb_cache_reload(module) != 0) {
2910 return LDB_ERR_OPERATIONS_ERROR;
2914 * Ensure we read (and so remove) the entries from the real
2915 * DB, no values stored so far are any use as we want to do a
2918 ltdb_index_transaction_cancel(module);
2920 ret = ltdb_index_transaction_start(module);
2921 if (ret != LDB_SUCCESS) {
2925 /* first traverse the database deleting any @INDEX records by
2926 * putting NULL entries in the in-memory tdb
2928 ret = ltdb->kv_ops->iterate(ltdb, delete_index, module);
2930 struct ldb_context *ldb = ldb_module_get_ctx(module);
2931 ldb_asprintf_errstring(ldb, "index deletion traverse failed: %s",
2932 ldb_errstring(ldb));
2933 return LDB_ERR_OPERATIONS_ERROR;
2936 ctx.module = module;
2940 ret = ltdb->kv_ops->iterate(ltdb, re_key, &ctx);
2942 struct ldb_context *ldb = ldb_module_get_ctx(module);
2943 ldb_asprintf_errstring(ldb, "key correction traverse failed: %s",
2944 ldb_errstring(ldb));
2945 return LDB_ERR_OPERATIONS_ERROR;
2948 if (ctx.error != LDB_SUCCESS) {
2949 struct ldb_context *ldb = ldb_module_get_ctx(module);
2950 ldb_asprintf_errstring(ldb, "reindexing failed: %s", ldb_errstring(ldb));
2957 /* now traverse adding any indexes for normal LDB records */
2958 ret = ltdb->kv_ops->iterate(ltdb, re_index, &ctx);
2960 struct ldb_context *ldb = ldb_module_get_ctx(module);
2961 ldb_asprintf_errstring(ldb, "reindexing traverse failed: %s",
2962 ldb_errstring(ldb));
2963 return LDB_ERR_OPERATIONS_ERROR;
2966 if (ctx.error != LDB_SUCCESS) {
2967 struct ldb_context *ldb = ldb_module_get_ctx(module);
2968 ldb_asprintf_errstring(ldb, "reindexing failed: %s", ldb_errstring(ldb));
2972 if (ctx.count > 10000) {
2973 ldb_debug(ldb_module_get_ctx(module),
2974 LDB_DEBUG_WARNING, "Reindexing: re_index successful on %s, "
2975 "final index write-out will be in transaction commit",
2976 ltdb->kv_ops->name(ltdb));