ldb: Add ldb_unpack_data_only_attr_list_flags()

This function allows us to control allocation of memory during parse
of the packed ldb data.

This in turn can have an important performance impact as each
small allocation can have a large overhead

Signed-off-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>

diff --git a/lib/ldb/ABI/ldb-1.1.26.sigs b/lib/ldb/ABI/ldb-1.1.26.sigs
index 3f33df96f745ec241160238e60756a99299933e0..4fa30d88ec980b6baa0827aa0e408e17f7b420e1 100644 (file)
--- a/lib/ldb/ABI/ldb-1.1.26.sigs
+++ b/lib/ldb/ABI/ldb-1.1.26.sigs
@@ -254,6 +254,7 @@ ldb_transaction_prepare_commit: int (struct ldb_context *)
 ldb_transaction_start: int (struct ldb_context *)
 ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
 ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
 ldb_transaction_start: int (struct ldb_context *)
 ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
 ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)

+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)

 ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
 ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
 ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
 ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
 ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
 ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)

diff --git a/lib/ldb/common/ldb_pack.c b/lib/ldb/common/ldb_pack.c
index 7970b9d3ecb8325b4a46f1da46236810009cbb50..cc3a552758386d7c9116535171ae7b6fb73a9549 100644 (file)
--- a/lib/ldb/common/ldb_pack.c
+++ b/lib/ldb/common/ldb_pack.c
@@ -211,20 +211,20 @@ static bool ldb_consume_element_data(uint8_t **pp, size_t *premaining)
        return true;
 }
 
        return true;
 }
 

+

 /*
  * Unpack a ldb message from a linear buffer in ldb_val
  *
  * Providing a list of attributes to this function allows selective unpacking.
  * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
 /*
  * Unpack a ldb message from a linear buffer in ldb_val
  *
  * Providing a list of attributes to this function allows selective unpacking.
  * Giving a NULL list (or a list_size of 0) unpacks all the attributes.

- *
- * Free with ldb_unpack_data_free()

  */
  */

-int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
-                                  const struct ldb_val *data,
-                                  struct ldb_message *message,
-                                  const char * const *list,
-                                  unsigned int list_size,
-                                  unsigned int *nb_elements_in_db)
+int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb,
+                                        const struct ldb_val *data,
+                                        struct ldb_message *message,
+                                        const char * const *list,
+                                        unsigned int list_size,
+                                        unsigned int flags,
+                                        unsigned int *nb_elements_in_db)

 {
        uint8_t *p;
        size_t remaining;
 {
        uint8_t *p;
        size_t remaining;


@@ -271,10 +271,14 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
                        errno = EIO;
                        goto failed;
                }
                        errno = EIO;
                        goto failed;
                }

-               message->dn = ldb_dn_new(message, ldb, (char *)p);
-               if (message->dn == NULL) {
-                       errno = ENOMEM;
-                       goto failed;
+               if (flags & LDB_UNPACK_DATA_FLAG_NO_DN) {
+                       message->dn = NULL;
+               } else {
+                       message->dn = ldb_dn_new(message, ldb, (char *)p);
+                       if (message->dn == NULL) {
+                               errno = ENOMEM;
+                               goto failed;
+                       }

                }
                /*
                 * Redundant: by definition, remaining must be more
                }
                /*
                 * Redundant: by definition, remaining must be more


@@ -373,11 +377,15 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
                        }
                }
                element = &message->elements[nelem];
                        }
                }
                element = &message->elements[nelem];

-               element->name = talloc_memdup(message->elements, attr, attr_len+1);
+               if (flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
+                       element->name = attr;
+               } else {
+                       element->name = talloc_memdup(message->elements, attr, attr_len+1);

 
 

-               if (element->name == NULL) {
-                       errno = ENOMEM;
-                       goto failed;
+                       if (element->name == NULL) {
+                               errno = ENOMEM;
+                               goto failed;
+                       }

                }
                element->flags = 0;
 
                }
                element->flags = 0;
 


@@ -422,15 +430,18 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
                        }
 
                        element->values[j].length = len;
                        }
 
                        element->values[j].length = len;

-                       element->values[j].data = talloc_size(element->values, len+1);
-                       if (element->values[j].data == NULL) {
-                               errno = ENOMEM;
-                               goto failed;
+                       if (flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
+                               element->values[j].data = p + 4;
+                       } else {
+                               element->values[j].data = talloc_size(element->values, len+1);
+                               if (element->values[j].data == NULL) {
+                                       errno = ENOMEM;
+                                       goto failed;
+                               }
+                               memcpy(element->values[j].data, p + 4,
+                                      len);
+                               element->values[j].data[len] = 0;

                        }
                        }

-                       memcpy(element->values[j].data, p + 4,
-                              len);
-                       element->values[j].data[len] = 0;
-

                        remaining -= len;
                        p += len+4+1;
                }
                        remaining -= len;
                        p += len+4+1;
                }


@@ -463,6 +474,30 @@ failed:
        return -1;
 }
 
        return -1;
 }
 

+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Providing a list of attributes to this function allows selective unpacking.
+ * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
+ *
+ * Free with ldb_unpack_data_free()
+ */
+int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
+                                  const struct ldb_val *data,
+                                  struct ldb_message *message,
+                                  const char * const *list,
+                                  unsigned int list_size,
+                                  unsigned int *nb_elements_in_db)
+{
+       return ldb_unpack_data_only_attr_list_flags(ldb,
+                                                   data,
+                                                   message,
+                                                   list,
+                                                   list_size,
+                                                   0,
+                                                   nb_elements_in_db);
+}
+

 int ldb_unpack_data(struct ldb_context *ldb,
                    const struct ldb_val *data,
                    struct ldb_message *message)
 int ldb_unpack_data(struct ldb_context *ldb,
                    const struct ldb_val *data,
                    struct ldb_message *message)

diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h
index c6a24d35e537e2fbe778e80f447daa4998f3043f..1c48590a3812c2ff93aebffc350c4671310e572b 100644 (file)
--- a/lib/ldb/include/ldb_module.h
+++ b/lib/ldb/include/ldb_module.h
@@ -390,6 +390,12 @@ int ldb_register_extended_match_rule(struct ldb_context *ldb,
 int ldb_pack_data(struct ldb_context *ldb,
                  const struct ldb_message *message,
                  struct ldb_val *data);
 int ldb_pack_data(struct ldb_context *ldb,
                  const struct ldb_message *message,
                  struct ldb_val *data);

+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Providing a list of attributes to this function allows selective unpacking.
+ * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
+ */

 int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
                                   const struct ldb_val *data,
                                   struct ldb_message *message,
 int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
                                   const struct ldb_val *data,
                                   struct ldb_message *message,


@@ -399,5 +405,27 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
 int ldb_unpack_data(struct ldb_context *ldb,
                    const struct ldb_val *data,
                    struct ldb_message *message);
 int ldb_unpack_data(struct ldb_context *ldb,
                    const struct ldb_val *data,
                    struct ldb_message *message);

+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Providing a list of attributes to this function allows selective unpacking.
+ * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
+ *
+ * Flags allow control of allocation, so that if
+ * LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC is specified, then values are
+ * not allocate, instead they point into the supplier constant buffer.
+ *
+ * Likewise if LDB_UNPACK_DATA_FLAG_NO_DN is specified, the DN is omitted.
+ */
+int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb,
+                                        const struct ldb_val *data,
+                                        struct ldb_message *message,
+                                        const char * const *list,
+                                        unsigned int list_size,
+                                        unsigned int flags,
+                                        unsigned int *nb_elements_in_db);
+
+#define LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC 0x0001
+#define LDB_UNPACK_DATA_FLAG_NO_DN         0x0002

 
 #endif
 
 #endif

diff --git a/source4/torture/ldb/ldb.c b/source4/torture/ldb/ldb.c
index 7ea9726d520d56f71a19bf79d3e242caf735d265..f7f04db0242f24c1ae85ae8e3cc3aac651a2d77d 100644 (file)
--- a/source4/torture/ldb/ldb.c
+++ b/source4/torture/ldb/ldb.c
@@ -1102,6 +1102,54 @@ static bool torture_ldb_unpack(struct torture_context *torture)
        return true;
 }
 
        return true;
 }
 

+static bool torture_ldb_unpack_flags(struct torture_context *torture)
+{
+       TALLOC_CTX *mem_ctx = talloc_new(torture);
+       struct ldb_context *ldb;
+       struct ldb_val data = data_blob_const(dda1d01d_bin, sizeof(dda1d01d_bin));
+       struct ldb_message *msg = ldb_msg_new(mem_ctx);
+       const char *ldif_text = dda1d01d_ldif;
+       struct ldb_ldif ldif;
+       unsigned int nb_elements_in_db;
+
+       ldb = samba_ldb_init(mem_ctx, torture->ev, NULL, NULL, NULL);
+       torture_assert(torture,
+                      ldb != NULL,
+                      "Failed to init ldb");
+
+       torture_assert_int_equal(torture,
+                                ldb_unpack_data_only_attr_list_flags(ldb, &data,
+                                                                     msg,
+                                                                     NULL, 0,
+                                                                     LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC,
+                                                                     &nb_elements_in_db),
+                                0,
+                                "ldb_unpack_data failed");
+
+       ldif.changetype = LDB_CHANGETYPE_NONE;
+       ldif.msg = msg;
+       ldif_text = ldb_ldif_write_string(ldb, mem_ctx, &ldif);
+
+       torture_assert_int_equal(torture,
+                                strcmp(ldif_text, dda1d01d_ldif), 0,
+                                "ldif form differs from binary form");
+
+       torture_assert_int_equal(torture,
+                                ldb_unpack_data_only_attr_list_flags(ldb, &data,
+                                                                     msg,
+                                                                     NULL, 0,
+                                                                     LDB_UNPACK_DATA_FLAG_NO_DN,
+                                                                     &nb_elements_in_db),
+                                0,
+                                "ldb_unpack_data failed");
+
+       torture_assert(torture,
+                      msg->dn == NULL,
+                      "msg->dn should be NULL");
+
+       return true;
+}
+

 static bool torture_ldb_parse_ldif(struct torture_context *torture)
 {
        TALLOC_CTX *mem_ctx = talloc_new(torture);
 static bool torture_ldb_parse_ldif(struct torture_context *torture)
 {
        TALLOC_CTX *mem_ctx = talloc_new(torture);


@@ -1265,6 +1313,8 @@ struct torture_suite *torture_ldb(TALLOC_CTX *mem_ctx)
        torture_suite_add_simple_test(suite, "dn", torture_ldb_dn);
        torture_suite_add_simple_test(suite, "unpack-data",
                                      torture_ldb_unpack);
        torture_suite_add_simple_test(suite, "dn", torture_ldb_dn);
        torture_suite_add_simple_test(suite, "unpack-data",
                                      torture_ldb_unpack);

+       torture_suite_add_simple_test(suite, "unpack-data-flags",
+                                     torture_ldb_unpack_flags);

        torture_suite_add_simple_test(suite, "parse-ldif",
                                      torture_ldb_parse_ldif);
        torture_suite_add_simple_test(suite, "unpack-data-only-attr-list",
        torture_suite_add_simple_test(suite, "parse-ldif",
                                      torture_ldb_parse_ldif);
        torture_suite_add_simple_test(suite, "unpack-data-only-attr-list",