librpc/ndr: add new LIBNDR_FLAG_STR_RAW8 for ndr_pull_string
authorSean Finney <seanius@seanius.net>
Tue, 31 May 2011 07:49:19 +0000 (09:49 +0200)
committerAndrew Bartlett <abartlet@samba.org>
Tue, 31 May 2011 22:30:40 +0000 (00:30 +0200)
Introduce a new flag, LIBNDR_FLAG_STR_RAW8, which indicates that libndr
should not attempt to convert the corresponding byte sequence, and place
the responsibility on the caller to do so later.

This is needed in cases where the string is known to be 8-bit and either
NULL terminated or of known length, but in an unspecified character set.
For example, when pulling PT_STRING8 properties from an exchange server
via libmapi + libndr, the codepage is neither known nor in the control
of the caller, and is determined by subsequent properties requested from
the server.  Therefore the client would like to fetch all properties in
one large batch, and convert the resulting strings locally.

This commit also includes some (basic) tests of each of the flags'
respective behaviors with the ndr push/pull string functions, in a new
source4 torture test suite ndr.ndr_string.

Signed-off-by: Sean Finney <seanius@seanius.net>
librpc/idl/idl_types.h
librpc/ndr/libndr.h
librpc/ndr/ndr_string.c
source4/torture/ndr/ndr.c
source4/torture/ndr/string.c [new file with mode: 0644]
source4/torture/wscript_build

index 023c04020ea9528c7590c77d670c30643bb8b8ea..c50eface0d1282d311bd82602e07ca36a380bcc5 100644 (file)
@@ -8,6 +8,7 @@
 #define STR_CONFORMANT  LIBNDR_FLAG_STR_CONFORMANT
 #define STR_CHARLEN    LIBNDR_FLAG_STR_CHARLEN
 #define STR_UTF8       LIBNDR_FLAG_STR_UTF8
 #define STR_CONFORMANT  LIBNDR_FLAG_STR_CONFORMANT
 #define STR_CHARLEN    LIBNDR_FLAG_STR_CHARLEN
 #define STR_UTF8       LIBNDR_FLAG_STR_UTF8
+#define STR_RAW8       LIBNDR_FLAG_STR_RAW8
 
 /*
   a null terminated UCS2 string
 
 /*
   a null terminated UCS2 string
 */
 #define utf8string     [flag(STR_UTF8|STR_NULLTERM)] string
 
 */
 #define utf8string     [flag(STR_UTF8|STR_NULLTERM)] string
 
+/*
+  a null terminated "raw" string (null terminated byte sequence)
+*/
+#define raw8string     [flag(STR_RAW8|STR_NULLTERM)] string
+
 /*
   a null terminated UCS2 string
 */
 /*
   a null terminated UCS2 string
 */
index cbe9b40440e33e3979a2dc6f5271fbcdda592d03..ca3710bc986a95c96cc924adfe184cd1cf277b23 100644 (file)
@@ -122,6 +122,7 @@ struct ndr_print {
 #define LIBNDR_FLAG_STR_CONFORMANT     (1<<10)
 #define LIBNDR_FLAG_STR_CHARLEN                (1<<11)
 #define LIBNDR_FLAG_STR_UTF8           (1<<12)
 #define LIBNDR_FLAG_STR_CONFORMANT     (1<<10)
 #define LIBNDR_FLAG_STR_CHARLEN                (1<<11)
 #define LIBNDR_FLAG_STR_UTF8           (1<<12)
+#define LIBNDR_FLAG_STR_RAW8           (1<<13)
 #define LIBNDR_STRING_FLAGS            (0x7FFC)
 
 /* set if relative pointers should *not* be marshalled in reverse order */
 #define LIBNDR_STRING_FLAGS            (0x7FFC)
 
 /* set if relative pointers should *not* be marshalled in reverse order */
index d0d8303240918b3de1e2740eed6a89d6147cc524..207d55bb1b15ae902c0b24c576b8141251d50371 100644 (file)
@@ -31,7 +31,7 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
        uint32_t len1, ofs, len2;
        uint16_t len3;
        size_t conv_src_len = 0, converted_size;
        uint32_t len1, ofs, len2;
        uint16_t len3;
        size_t conv_src_len = 0, converted_size;
-       int chset = CH_UTF16;
+       int do_convert = 1, chset = CH_UTF16;
        unsigned byte_mul = 2;
        unsigned flags = ndr->flags;
        unsigned c_len_term = 0;
        unsigned byte_mul = 2;
        unsigned flags = ndr->flags;
        unsigned c_len_term = 0;
@@ -56,6 +56,12 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
                flags &= ~LIBNDR_FLAG_STR_UTF8;
        }
 
                flags &= ~LIBNDR_FLAG_STR_UTF8;
        }
 
+       if (flags & LIBNDR_FLAG_STR_RAW8) {
+               do_convert = 0;
+               byte_mul = 1;
+               flags &= ~LIBNDR_FLAG_STR_RAW8;
+       }
+
        flags &= ~LIBNDR_FLAG_STR_CONFORMANT;
        if (flags & LIBNDR_FLAG_STR_CHARLEN) {
                c_len_term = 1;
        flags &= ~LIBNDR_FLAG_STR_CONFORMANT;
        if (flags & LIBNDR_FLAG_STR_CHARLEN) {
                c_len_term = 1;
@@ -138,7 +144,11 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string(struct ndr_pull *ndr, int ndr_flags,
        if (conv_src_len == 0) {
                as = talloc_strdup(ndr->current_mem_ctx, "");
        } else {
        if (conv_src_len == 0) {
                as = talloc_strdup(ndr->current_mem_ctx, "");
        } else {
-               if (!convert_string_talloc(ndr->current_mem_ctx, chset,
+               if (!do_convert) {
+                       as = talloc_strndup(ndr->current_mem_ctx,
+                                           ndr->data + ndr->offset,
+                                           conv_src_len);
+               } else if (!convert_string_talloc(ndr->current_mem_ctx, chset,
                                           CH_UNIX, ndr->data + ndr->offset,
                                           conv_src_len * byte_mul,
                                           (void **)(void *)&as,
                                           CH_UNIX, ndr->data + ndr->offset,
                                           conv_src_len * byte_mul,
                                           (void **)(void *)&as,
@@ -174,7 +184,7 @@ _PUBLIC_ enum ndr_err_code ndr_push_string(struct ndr_push *ndr, int ndr_flags,
 {
        ssize_t s_len, c_len;
        size_t d_len;
 {
        ssize_t s_len, c_len;
        size_t d_len;
-       int chset = CH_UTF16;
+       int do_convert = 1, chset = CH_UTF16;
        unsigned flags = ndr->flags;
        unsigned byte_mul = 2;
        uint8_t *dest = NULL;
        unsigned flags = ndr->flags;
        unsigned byte_mul = 2;
        uint8_t *dest = NULL;
@@ -201,12 +211,22 @@ _PUBLIC_ enum ndr_err_code ndr_push_string(struct ndr_push *ndr, int ndr_flags,
                flags &= ~LIBNDR_FLAG_STR_UTF8;
        }
 
                flags &= ~LIBNDR_FLAG_STR_UTF8;
        }
 
+       if (flags & LIBNDR_FLAG_STR_RAW8) {
+               do_convert = 0;
+               byte_mul = 1;
+               flags &= ~LIBNDR_FLAG_STR_RAW8;
+       }
+
        flags &= ~LIBNDR_FLAG_STR_CONFORMANT;
 
        if (!(flags & LIBNDR_FLAG_STR_NOTERM)) {
                s_len++;
        }
        flags &= ~LIBNDR_FLAG_STR_CONFORMANT;
 
        if (!(flags & LIBNDR_FLAG_STR_NOTERM)) {
                s_len++;
        }
-       if (!convert_string_talloc(ndr, CH_UNIX, chset, s, s_len,
+
+       if (!do_convert) {
+               d_len = s_len;
+               dest = talloc_strndup(ndr, s, s_len);
+       } else if (!convert_string_talloc(ndr, CH_UNIX, chset, s, s_len,
                                   (void **)(void *)&dest, &d_len))
        {
                return ndr_push_error(ndr, NDR_ERR_CHARCNV, 
                                   (void **)(void *)&dest, &d_len))
        {
                return ndr_push_error(ndr, NDR_ERR_CHARCNV, 
@@ -276,9 +296,13 @@ _PUBLIC_ size_t ndr_string_array_size(struct ndr_push *ndr, const char *s)
        unsigned byte_mul = 2;
        unsigned c_len_term = 1;
 
        unsigned byte_mul = 2;
        unsigned c_len_term = 1;
 
-       c_len = s?strlen_m(s):0;
+       if (flags & LIBNDR_FLAG_STR_RAW8) {
+               c_len = s?strlen(s):0;
+       } else {
+               c_len = s?strlen_m(s):0;
+       }
 
 
-       if (flags & (LIBNDR_FLAG_STR_ASCII|LIBNDR_FLAG_STR_UTF8)) {
+       if (flags & (LIBNDR_FLAG_STR_ASCII|LIBNDR_FLAG_STR_RAW8|LIBNDR_FLAG_STR_UTF8)) {
                byte_mul = 1;
        }
 
                byte_mul = 1;
        }
 
@@ -484,16 +508,22 @@ _PUBLIC_ size_t ndr_size_string_array(const char **a, uint32_t count, int flags)
 {
        uint32_t i;
        size_t size = 0;
 {
        uint32_t i;
        size_t size = 0;
+       int rawbytes = 0;
+
+       if (flags & LIBNDR_FLAG_STR_RAW8) {
+               rawbytes = 1;
+               flags &= ~LIBNDR_FLAG_STR_RAW8;
+       }
 
        switch (flags & LIBNDR_STRING_FLAGS) {
        case LIBNDR_FLAG_STR_NULLTERM:
                for (i = 0; i < count; i++) {
 
        switch (flags & LIBNDR_STRING_FLAGS) {
        case LIBNDR_FLAG_STR_NULLTERM:
                for (i = 0; i < count; i++) {
-                       size += strlen_m_term(a[i]);
+                       size += rawbytes?strlen(a[i]) + 1:strlen_m_term(a[i]);
                }
                break;
        case LIBNDR_FLAG_STR_NOTERM:
                for (i = 0; i < count; i++) {
                }
                break;
        case LIBNDR_FLAG_STR_NOTERM:
                for (i = 0; i < count; i++) {
-                       size += strlen_m(a[i]);
+                       size += rawbytes?strlen(a[i]):strlen_m(a[i]);
                }
                break;
        default:
                }
                break;
        default:
index 36b2b5540c0f70ea1f14e5745ad0ea6967ff8f9b..6c564d3310a12cd6714c55d17a09f365393b873c 100644 (file)
@@ -355,6 +355,7 @@ struct torture_suite *torture_local_ndr(TALLOC_CTX *mem_ctx)
        torture_suite_add_suite(suite, ndr_nbt_suite(suite));
        torture_suite_add_suite(suite, ndr_ntlmssp_suite(suite));
        torture_suite_add_suite(suite, ndr_backupkey_suite(suite));
        torture_suite_add_suite(suite, ndr_nbt_suite(suite));
        torture_suite_add_suite(suite, ndr_ntlmssp_suite(suite));
        torture_suite_add_suite(suite, ndr_backupkey_suite(suite));
+       torture_suite_add_suite(suite, ndr_string_suite(suite));
 
        torture_suite_add_simple_test(suite, "string terminator",
                                      test_check_string_terminator);
 
        torture_suite_add_simple_test(suite, "string terminator",
                                      test_check_string_terminator);
diff --git a/source4/torture/ndr/string.c b/source4/torture/ndr/string.c
new file mode 100644 (file)
index 0000000..9214b59
--- /dev/null
@@ -0,0 +1,199 @@
+#include "includes.h"
+#include "torture/ndr/ndr.h"
+#include "torture/ndr/proto.h"
+#include "../lib/util/dlinklist.h"
+#include "param/param.h"
+
+static const char const *ascii = "ascii";
+/* the following is equivalent to "kamelåså öäüÿéèóò" in latin1 */
+static const char const latin1[] = { 0x6b, 0x61, 0x6d, 0x65, 0x6c, 0xe5, 0x73,
+                                     0xe5, 0x20, 0xF6, 0xE4, 0xFC, 0xFF, 0xE9,
+                                     0xE8, 0xF3, 0xF2, 0x00 };
+/* the following is equivalent to "kamelåså ☺☺☺ öäüÿéèóò" in utf8 */
+static const char const utf8[] = { 0x6b, 0x61, 0x6d, 0x65, 0x6c, 0xc3, 0xa5,
+                                   0x73, 0xc3, 0xa5, 0x20, 0xE2, 0x98, 0xBA,
+                                   0xE2, 0x98, 0xBA, 0xE2, 0x98, 0xBA, 0x20,
+                                   0xc3, 0xb6, 0xc3, 0xa4, 0xc3, 0xbc, 0xc3,
+                                   0xbf, 0xc3, 0xa9, 0xc3, 0xa8, 0xc3, 0xb3,
+                                   0xc3, 0xb2, 0x00 };
+
+/* purely for convenience */
+static int fl_ascii_null = LIBNDR_FLAG_STR_ASCII|LIBNDR_FLAG_STR_NULLTERM;
+static int fl_utf8_null = LIBNDR_FLAG_STR_UTF8|LIBNDR_FLAG_STR_NULLTERM;
+static int fl_raw8_null = LIBNDR_FLAG_STR_RAW8|LIBNDR_FLAG_STR_NULLTERM;
+
+static bool
+test_ndr_push_string (struct torture_context *tctx, const char *string,
+                      int flags, enum ndr_err_code exp_ndr_err,
+                      bool strcmp_pass)
+{
+       TALLOC_CTX *mem_ctx;
+       struct ndr_push *ndr;
+       enum ndr_err_code err;
+
+       torture_comment(tctx,
+                        "test_ndr_push_string %s flags 0x%x expecting "
+                       "err 0x%x and strcmp %s\n", string, flags, exp_ndr_err,
+                       strcmp_pass?"pass":"fail");
+       if (exp_ndr_err != NDR_ERR_SUCCESS) {
+               torture_comment(tctx, "(ignore any Conversion error) ");
+       }
+
+       mem_ctx = talloc_named (NULL, 0, "test_ndr_push_string");
+       ndr = talloc_zero (mem_ctx, struct ndr_push);
+       ndr_set_flags (&ndr->flags, flags);
+
+       err = ndr_push_string (ndr, NDR_SCALARS, string);
+       torture_assert(tctx, err == exp_ndr_err,
+                      "ndr_push_string: unexpected return code");
+
+       if (exp_ndr_err == NDR_ERR_SUCCESS) {
+               torture_assert(tctx, ndr->data != NULL,
+                              "ndr_push_string: succeeded but NULL data");
+
+               torture_assert(tctx, strcmp_pass == !strcmp(string, ndr->data),
+                              "ndr_push_string: post-push strcmp");
+       }
+
+       talloc_free(mem_ctx);
+       return true;
+}
+
+static bool
+test_ndr_pull_string (struct torture_context *tctx, const char *string,
+                      int flags, enum ndr_err_code exp_ndr_err,
+                      bool strcmp_pass)
+{
+       TALLOC_CTX *mem_ctx;
+       DATA_BLOB blob;
+       struct ndr_pull *ndr;
+       enum ndr_err_code err;
+       const char *result = NULL;
+
+       torture_comment(tctx,
+                        "test_ndr_pull_string '%s' flags 0x%x expecting "
+                       "err 0x%x and strcmp %s\n", string, flags, exp_ndr_err,
+                       strcmp_pass?"pass":"fail");
+       if (exp_ndr_err != NDR_ERR_SUCCESS) {
+               torture_comment(tctx, "(ignore any Conversion error) ");
+       }
+
+       mem_ctx = talloc_named (NULL, 0, "test_ndr_pull_string");
+
+       blob = data_blob_string_const(string);
+       ndr = ndr_pull_init_blob(&blob, mem_ctx);
+       ndr_set_flags (&ndr->flags, flags);
+
+       err = ndr_pull_string (ndr, NDR_SCALARS, &result);
+       torture_assert(tctx, err == exp_ndr_err,
+                      "ndr_pull_string: unexpected return code");
+
+       if (exp_ndr_err == NDR_ERR_SUCCESS) {
+               torture_assert(tctx, result != NULL,
+                              "ndr_pull_string: NULL data");
+               torture_assert(tctx, strcmp_pass == !strcmp(string, result),
+                              "ndr_pull_string: post-pull strcmp");
+               torture_assert(tctx, result != NULL,
+                              "ndr_pull_string succeeded but result NULL");
+       }
+
+       talloc_free(mem_ctx);
+       return true;
+}
+
+static bool
+torture_ndr_string(struct torture_context *torture)
+{
+       bool ok;
+       int i;
+       const char *saved_dos_cp = lpcfg_dos_charset(torture->lp_ctx);
+
+       torture_assert(torture,
+                      test_ndr_push_string (torture, ascii, fl_ascii_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_push_string(ASCII, STR_ASCII|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_push_string (torture, utf8, fl_utf8_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_push_string(UTF8, STR_UTF8|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_push_string (torture, utf8, fl_raw8_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_push_string(UTF8, STR_RAW8|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_push_string (torture, latin1, fl_raw8_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_push_string(LATIN1, STR_RAW8|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_push_string (torture, utf8, fl_ascii_null,
+                                            NDR_ERR_CHARCNV, false),
+                      "test_ndr_push_string(UTF8, STR_ASCII|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_push_string (torture, latin1, fl_ascii_null,
+                                            NDR_ERR_CHARCNV, false),
+                      "test_ndr_push_string(LATIN1, STR_ASCII|STR_NULL)");
+
+
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, ascii, fl_ascii_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_pull_string(ASCII, STR_ASCII|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, utf8, fl_utf8_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_pull_string(UTF8, STR_UTF8|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, utf8, fl_raw8_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_pull_string(UTF8, STR_RAW8|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, latin1, fl_raw8_null,
+                                            NDR_ERR_SUCCESS, true),
+                      "test_ndr_pull_string(LATIN1, STR_RAW8|STR_NULL)");
+
+       /* Depending on runtime config, the behavior of ndr_pull_string on
+        * incorrect combinations of strings and flags (latin1 with ASCII
+        * flags, for example) may differ; it may return NDR_ERR_CHARCNV, or
+        * it may return NDR_ERR_SUCCESS but with a string that has been
+        * mutilated, depending on the value of "dos charset".  We test for
+        * both cases here. */
+
+       lpcfg_do_global_parameter(torture->lp_ctx, "dos charset", "ASCII");
+       reload_charcnv(torture->lp_ctx);
+
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, latin1, fl_ascii_null,
+                                            NDR_ERR_CHARCNV, false),
+                      "test_ndr_pull_string(LATIN1, STR_ASCII|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, utf8, fl_ascii_null,
+                                            NDR_ERR_CHARCNV, false),
+                      "test_ndr_pull_string(UTF8, STR_ASCII|STR_NULL)");
+
+       lpcfg_do_global_parameter(torture->lp_ctx, "dos charset", "CP850");
+       reload_charcnv(torture->lp_ctx);
+
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, latin1, fl_ascii_null,
+                                            NDR_ERR_SUCCESS, false),
+                      "test_ndr_pull_string(LATIN1, STR_ASCII|STR_NULL)");
+       torture_assert(torture,
+                      test_ndr_pull_string (torture, utf8, fl_ascii_null,
+                                            NDR_ERR_SUCCESS, false),
+                      "test_ndr_pull_string(UTF8, STR_ASCII|STR_NULL)");
+
+       lpcfg_do_global_parameter(torture->lp_ctx, "dos charset", saved_dos_cp);
+       reload_charcnv(torture->lp_ctx);
+
+       return true;
+}
+
+struct torture_suite *ndr_string_suite(TALLOC_CTX *ctx)
+{
+       struct torture_suite *suite = torture_suite_create(ctx, "ndr_string");
+
+       torture_suite_add_simple_test(suite, "ndr_string", torture_ndr_string);
+       suite->description = talloc_strdup(suite, "NDR - string-conversion focused push/pull tests");
+
+       return suite;
+}
index 68ec4e622095c89e9322c1d9b98b98e07fc3071f..106cc64280cb4a0d81189017852dd67206ffda12 100644 (file)
@@ -33,7 +33,7 @@ bld.RECURSE('libnetapi')
 bld.RECURSE('libsmbclient')
 
 bld.SAMBA_SUBSYSTEM('TORTURE_NDR',
 bld.RECURSE('libsmbclient')
 
 bld.SAMBA_SUBSYSTEM('TORTURE_NDR',
-       source='ndr/ndr.c ndr/winreg.c ndr/atsvc.c ndr/lsa.c ndr/epmap.c ndr/dfs.c ndr/netlogon.c ndr/drsuapi.c ndr/spoolss.c ndr/samr.c ndr/dfsblob.c ndr/drsblobs.c ndr/nbt.c ndr/ntlmssp.c ndr/backupkey.c',
+       source='ndr/ndr.c ndr/winreg.c ndr/atsvc.c ndr/lsa.c ndr/epmap.c ndr/dfs.c ndr/netlogon.c ndr/drsuapi.c ndr/spoolss.c ndr/samr.c ndr/dfsblob.c ndr/drsblobs.c ndr/nbt.c ndr/ntlmssp.c ndr/backupkey.c ndr/string.c',
        autoproto='ndr/proto.h',
        deps='torture'
        )
        autoproto='ndr/proto.h',
        deps='torture'
        )