pidl/python: also add a ndr_PyLong_FromLongLong() for symnetric reasons
[samba.git] / pidl / lib / Parse / Pidl / Samba4 / Python.pm
index 801c0bf9729ccb7d3f28d98315330c072d2685b1..180b6b20089953641dc2d564556f99e6820e125f 100644 (file)
@@ -10,9 +10,9 @@ use Exporter;
 
 use strict;
 use Parse::Pidl qw(warning fatal error);
-use Parse::Pidl::Typelist qw(hasType resolveType getType mapTypeName expandAlias);
+use Parse::Pidl::Typelist qw(hasType resolveType getType mapTypeName expandAlias bitmap_type_fn enum_type_fn);
 use Parse::Pidl::Util qw(has_property ParseExpr unmake_str);
-use Parse::Pidl::NDR qw(GetPrevLevel GetNextLevel ContainsDeferred is_charset_array);
+use Parse::Pidl::NDR qw(GetPrevLevel GetNextLevel ContainsDeferred ContainsPipe is_charset_array);
 use Parse::Pidl::CUtil qw(get_value_of get_pointer_to);
 use Parse::Pidl::Samba4 qw(ArrayDynamicallyAllocated);
 use Parse::Pidl::Samba4::Header qw(GenerateFunctionInEnv GenerateFunctionOutEnv EnvSubstituteValue GenerateStructEnv);
@@ -22,9 +22,11 @@ $VERSION = '0.01';
 
 sub new($) {
        my ($class) = @_;
-       my $self = { res => "", res_hdr => "", tabs => "", constants => {},
+       my $self = { res => "", res_hdr => "", tabs => "",
+                                constants => [], constants_uniq => {},
                     module_methods => [], module_objects => [], ready_types => [],
-                                module_imports => {}, type_imports => {},
+                                module_imports => [], module_imports_uniq => {},
+                                type_imports => [], type_imports_uniq => {},
                                 patch_type_calls => [], prereadycode => [],
                                 postreadycode => []};
        bless($self, $class);
@@ -94,7 +96,11 @@ sub register_constant($$$$)
 {
        my ($self, $name, $type, $value) = @_;
 
-       $self->{constants}->{$name} = [$type, $value];
+       unless (defined $self->{constants_uniq}->{$name}) {
+               my $h = {"key" => $name, "val" => [$type, $value]};
+               push @{$self->{constants}}, $h;
+               $self->{constants_uniq}->{$name} = $h;
+       }
 }
 
 sub EnumAndBitmapConsts($$$)
@@ -106,7 +112,7 @@ sub EnumAndBitmapConsts($$$)
        foreach my $e (@{$d->{ELEMENTS}}) {
                $e =~ /^([A-Za-z0-9_]+)/;
                my $cname = $1;
-               
+
                $self->register_constant($cname, $d, $cname);
        }
 }
@@ -200,9 +206,9 @@ sub PythonStruct($$$$$$)
                        $self->pidl("static PyObject *py_$name\_get_$e->{NAME}(PyObject *obj, void *closure)");
                        $self->pidl("{");
                        $self->indent;
-                       $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(obj);");
+                       $self->pidl("$cname *object = ($cname *)pytalloc_get_ptr(obj);");
                        $self->pidl("PyObject *py_$e->{NAME};");
-                       $self->ConvertObjectToPython("py_talloc_get_mem_ctx(obj)", $env, $e, $varname, "py_$e->{NAME}", "return NULL;");
+                       $self->ConvertObjectToPython("pytalloc_get_mem_ctx(obj)", $env, $e, $varname, "py_$e->{NAME}", "return NULL;");
                        $self->pidl("return py_$e->{NAME};");
                        $self->deindent;
                        $self->pidl("}");
@@ -211,14 +217,14 @@ sub PythonStruct($$$$$$)
                        $self->pidl("static int py_$name\_set_$e->{NAME}(PyObject *py_obj, PyObject *value, void *closure)");
                        $self->pidl("{");
                        $self->indent;
-                       $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
-                       my $mem_ctx = "py_talloc_get_mem_ctx(py_obj)";
+                       $self->pidl("$cname *object = ($cname *)pytalloc_get_ptr(py_obj);");
+                       my $mem_ctx = "pytalloc_get_mem_ctx(py_obj)";
                        my $l = $e->{LEVELS}[0];
                        my $nl = GetNextLevel($e, $l);
-                       if ($l->{TYPE} eq "POINTER" and 
+                       if ($l->{TYPE} eq "POINTER" and
                                not ($nl->{TYPE} eq "ARRAY" and ($nl->{IS_FIXED} or is_charset_array($e, $nl))) and
                                not ($nl->{TYPE} eq "DATA" and Parse::Pidl::Typelist::scalar_is_reference($nl->{DATA_TYPE}))) {
-                               $self->pidl("talloc_unlink(py_talloc_get_mem_ctx(py_obj), $varname);");
+                               $self->pidl("talloc_unlink(pytalloc_get_mem_ctx(py_obj), discard_const($varname));");
                        }
                        $self->ConvertObjectFromPython($env, $mem_ctx, $e, "value", $varname, "return -1;");
                        $self->pidl("return 0;");
@@ -242,23 +248,23 @@ sub PythonStruct($$$$$$)
        $self->pidl("static PyObject *py_$name\_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)");
        $self->pidl("{");
        $self->indent;
-       $self->pidl("return py_talloc_new($cname, type);");
+       $self->pidl("return pytalloc_new($cname, type);");
        $self->deindent;
        $self->pidl("}");
        $self->pidl("");
 
        my $py_methods = "NULL";
 
-       # If the struct is not public there ndr_pull/ndr_push functions will 
+       # If the struct is not public there ndr_pull/ndr_push functions will
        # be static so not callable from here
        if (has_property($d, "public")) {
                $self->pidl("static PyObject *py_$name\_ndr_pack(PyObject *py_obj)");
                $self->pidl("{");
                $self->indent;
-               $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
+               $self->pidl("$cname *object = ($cname *)pytalloc_get_ptr(py_obj);");
                $self->pidl("DATA_BLOB blob;");
                $self->pidl("enum ndr_err_code err;");
-               $self->pidl("err = ndr_push_struct_blob(&blob, py_talloc_get_mem_ctx(py_obj), object, (ndr_push_flags_fn_t)ndr_push_$name);");
+               $self->pidl("err = ndr_push_struct_blob(&blob, pytalloc_get_mem_ctx(py_obj), object, (ndr_push_flags_fn_t)ndr_push_$name);");
                $self->pidl("if (err != NDR_ERR_SUCCESS) {");
                $self->indent;
                $self->pidl("PyErr_SetNdrError(err);");
@@ -271,16 +277,44 @@ sub PythonStruct($$$$$$)
                $self->pidl("}");
                $self->pidl("");
 
-               $self->pidl("static PyObject *py_$name\_ndr_unpack(PyObject *py_obj, PyObject *args)");
+               $self->pidl("static PyObject *py_$name\_ndr_unpack(PyObject *py_obj, PyObject *args, PyObject *kwargs)");
                $self->pidl("{");
                $self->indent;
-               $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
+               $self->pidl("$cname *object = ($cname *)pytalloc_get_ptr(py_obj);");
                $self->pidl("DATA_BLOB blob;");
+               $self->pidl("int blob_length = 0;");
                $self->pidl("enum ndr_err_code err;");
-               $self->pidl("if (!PyArg_ParseTuple(args, \"s#:__ndr_unpack__\", &blob.data, &blob.length))");
-               $self->pidl("\treturn NULL;");
+               $self->pidl("const char * const kwnames[] = { \"data_blob\", \"allow_remaining\", NULL };");
+               $self->pidl("PyObject *allow_remaining_obj = NULL;");
+               $self->pidl("bool allow_remaining = false;");
+               $self->pidl("");
+               $self->pidl("if (!PyArg_ParseTupleAndKeywords(args, kwargs, \"s#|O:__ndr_unpack__\",");
+               $self->indent;
+               $self->pidl("discard_const_p(char *, kwnames),");
+               $self->pidl("&blob.data, &blob_length,");
+               $self->pidl("&allow_remaining_obj)) {");
+               $self->deindent;
+               $self->indent;
+               $self->pidl("return NULL;");
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("blob.length = blob_length;");
+               $self->pidl("");
+               $self->pidl("if (allow_remaining_obj && PyObject_IsTrue(allow_remaining_obj)) {");
+               $self->indent;
+               $self->pidl("allow_remaining = true;");
+               $self->deindent;
+               $self->pidl("}");
                $self->pidl("");
-               $self->pidl("err = ndr_pull_struct_blob_all(&blob, py_talloc_get_mem_ctx(py_obj), object, (ndr_pull_flags_fn_t)ndr_pull_$name);");
+               $self->pidl("if (allow_remaining) {");
+               $self->indent;
+               $self->pidl("err = ndr_pull_struct_blob(&blob, pytalloc_get_mem_ctx(py_obj), object, (ndr_pull_flags_fn_t)ndr_pull_$name);");
+               $self->deindent;
+               $self->pidl("} else {");
+               $self->indent;
+               $self->pidl("err = ndr_pull_struct_blob_all(&blob, pytalloc_get_mem_ctx(py_obj), object, (ndr_pull_flags_fn_t)ndr_pull_$name);");
+               $self->deindent;
+               $self->pidl("}");
                $self->pidl("if (err != NDR_ERR_SUCCESS) {");
                $self->indent;
                $self->pidl("PyErr_SetNdrError(err);");
@@ -296,10 +330,15 @@ sub PythonStruct($$$$$$)
                $self->pidl("static PyObject *py_$name\_ndr_print(PyObject *py_obj)");
                $self->pidl("{");
                $self->indent;
-               $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
+               $self->pidl("$cname *object = ($cname *)pytalloc_get_ptr(py_obj);");
+               $self->pidl("PyObject *ret;");
                $self->pidl("char *retstr;");
-               $self->pidl("retstr = ndr_print_struct_string(py_talloc_get_mem_ctx(py_obj), (ndr_print_fn_t)ndr_print_$name, \"$name\", object);");
-               $self->pidl("return PyString_FromString(retstr);");
+               $self->pidl("");
+               $self->pidl("retstr = ndr_print_struct_string(pytalloc_get_mem_ctx(py_obj), (ndr_print_fn_t)ndr_print_$name, \"$name\", object);");
+               $self->pidl("ret = PyString_FromString(retstr);");
+               $self->pidl("talloc_free(retstr);");
+               $self->pidl("");
+               $self->pidl("return ret;");
                $self->deindent;
                $self->pidl("}");
                $self->pidl("");
@@ -308,7 +347,7 @@ sub PythonStruct($$$$$$)
                $self->pidl("static PyMethodDef $py_methods\[] = {");
                $self->indent;
                $self->pidl("{ \"__ndr_pack__\", (PyCFunction)py_$name\_ndr_pack, METH_NOARGS, \"S.ndr_pack(object) -> blob\\nNDR pack\" },");
-               $self->pidl("{ \"__ndr_unpack__\", (PyCFunction)py_$name\_ndr_unpack, METH_VARARGS, \"S.ndr_unpack(class, blob) -> None\\nNDR unpack\" },");
+               $self->pidl("{ \"__ndr_unpack__\", (PyCFunction)py_$name\_ndr_unpack, METH_VARARGS|METH_KEYWORDS, \"S.ndr_unpack(class, blob, allow_remaining=False) -> None\\nNDR unpack\" },");
                $self->pidl("{ \"__ndr_print__\", (PyCFunction)py_$name\_ndr_print, METH_VARARGS, \"S.ndr_print(object) -> None\\nNDR print\" },");
                $self->pidl("{ NULL, NULL, 0, NULL }");
                $self->deindent;
@@ -316,7 +355,7 @@ sub PythonStruct($$$$$$)
                $self->pidl("");
        }
 
-       $self->pidl_hdr("staticforward PyTypeObject $name\_Type;\n");
+       $self->pidl_hdr("static PyTypeObject $name\_Type;\n");
        $self->pidl("");
        my $docstring = $self->DocString($d, $name);
        my $typeobject = "$name\_Type";
@@ -330,7 +369,7 @@ sub PythonStruct($$$$$$)
        }
        $self->pidl(".tp_methods = $py_methods,");
        $self->pidl(".tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,");
-       $self->pidl(".tp_basicsize = sizeof(py_talloc_Object),");
+       $self->pidl(".tp_basicsize = sizeof(pytalloc_Object),");
        $self->pidl(".tp_new = py_$name\_new,");
        $self->deindent;
        $self->pidl("};");
@@ -366,8 +405,8 @@ sub find_metadata_args($)
        foreach my $e (@{$fn->{ELEMENTS}}) {
                foreach my $dir (@{$e->{DIRECTION}}) {
                         my $main = get_metadata_var($e);
-                        if ($main) { 
-                                $metadata_args->{$dir}->{$main} = $e->{NAME}; 
+                        if ($main) {
+                                $metadata_args->{$dir}->{$main} = $e->{NAME};
                         }
                 }
        }
@@ -393,7 +432,7 @@ sub PythonFunctionUnpackOut($$$)
        $self->pidl("PyObject *result;");
        foreach my $e (@{$fn->{ELEMENTS}}) {
                next unless (grep(/out/,@{$e->{DIRECTION}}));
-               next if (($metadata_args->{in}->{$e->{NAME}} and grep(/in/, @{$e->{DIRECTION}})) or 
+               next if (($metadata_args->{in}->{$e->{NAME}} and grep(/in/, @{$e->{DIRECTION}})) or
                         ($metadata_args->{out}->{$e->{NAME}}) and grep(/out/, @{$e->{DIRECTION}}));
                $self->pidl("PyObject *py_$e->{NAME};");
                $result_size++;
@@ -476,7 +515,7 @@ sub PythonFunctionPackIn($$$)
 
        foreach my $e (@{$fn->{ELEMENTS}}) {
                next unless (grep(/in/,@{$e->{DIRECTION}}));
-               next if (($metadata_args->{in}->{$e->{NAME}} and grep(/in/, @{$e->{DIRECTION}})) or 
+               next if (($metadata_args->{in}->{$e->{NAME}} and grep(/in/, @{$e->{DIRECTION}})) or
                                 ($metadata_args->{out}->{$e->{NAME}}) and grep(/out/, @{$e->{DIRECTION}}));
                $self->pidl("PyObject *py_$e->{NAME};");
                $args_format .= "O";
@@ -606,6 +645,7 @@ sub PythonType($$$$)
 
        if ($actual_ctype->{TYPE} eq "UNION" and defined($actual_ctype->{ELEMENTS})) {
                $self->pidl("PyObject *py_import_$d->{NAME}(TALLOC_CTX *mem_ctx, int level, " .mapTypeName($d) . " *in)");
+               $self->pidl_hdr("PyObject *py_import_$d->{NAME}(TALLOC_CTX *mem_ctx, int level, " .mapTypeName($d) . " *in);\n");
                $self->pidl("{");
                $self->indent;
                $self->FromUnionToPythonFunction("mem_ctx", $actual_ctype, "level", "in") if ($actual_ctype->{TYPE} eq "UNION");
@@ -614,6 +654,7 @@ sub PythonType($$$$)
                $self->pidl("");
 
                $self->pidl(mapTypeName($d) . " *py_export_$d->{NAME}(TALLOC_CTX *mem_ctx, int level, PyObject *in)");
+               $self->pidl_hdr(mapTypeName($d) . " *py_export_$d->{NAME}(TALLOC_CTX *mem_ctx, int level, PyObject *in);\n");
                $self->pidl("{");
                $self->indent;
                $self->FromPythonToUnionFunction($actual_ctype, mapTypeName($d), "level", "mem_ctx", "in") if ($actual_ctype->{TYPE} eq "UNION");
@@ -652,7 +693,7 @@ sub Interface($$$)
        }
 
        if (defined $interface->{PROPERTIES}->{uuid}) {
-               $self->pidl_hdr("staticforward PyTypeObject $interface->{NAME}_InterfaceType;\n");
+               $self->pidl_hdr("static PyTypeObject $interface->{NAME}_InterfaceType;\n");
                $self->pidl("");
 
                my @fns = ();
@@ -662,6 +703,15 @@ sub Interface($$$)
                        next if has_property($d, "nopython");
                        next if has_property($d, "todo");
 
+                       my $skip = 0;
+                       foreach my $e (@{$d->{ELEMENTS}}) {
+                               if (ContainsPipe($e, $e->{LEVELS}[0])) {
+                                       $skip = 1;
+                                       last;
+                               }
+                       }
+                       next if $skip;
+
                        my $prettyname = $d->{NAME};
 
                        $prettyname =~ s/^$interface->{NAME}_//;
@@ -689,10 +739,10 @@ sub Interface($$$)
                $self->pidl("return py_dcerpc_interface_init_helper(type, args, kwargs, &ndr_table_$interface->{NAME});");
                $self->deindent;
                $self->pidl("}");
-       
+
                $self->pidl("");
 
-               my $signature = 
+               my $signature =
 "\"$interface->{NAME}(binding, lp_ctx=None, credentials=None) -> connection\\n\"
 \"\\n\"
 \"binding should be a DCE/RPC binding string (for example: ncacn_ip_tcp:127.0.0.1)\\n\"
@@ -726,6 +776,45 @@ sub Interface($$$)
                my $dcerpc_typename = $self->import_type_variable("samba.dcerpc.base", "ClientConnection");
                $self->register_module_prereadycode(["$if_typename.tp_base = $dcerpc_typename;", ""]);
                $self->register_module_postreadycode(["if (!PyInterface_AddNdrRpcMethods(&$if_typename, py_ndr_$interface->{NAME}\_methods))", "\treturn;", ""]);
+
+
+               $self->pidl("static PyObject *syntax_$interface->{NAME}_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)");
+               $self->pidl("{");
+               $self->indent;
+               $self->pidl("return py_dcerpc_syntax_init_helper(type, args, kwargs, &ndr_table_$interface->{NAME}.syntax_id);");
+               $self->deindent;
+               $self->pidl("}");
+
+               $self->pidl("");
+
+               my $signature = "\"abstract_syntax()\\n\"";
+
+               my $docstring = $self->DocString($interface, $interface->{NAME}."_syntax");
+
+               if ($docstring) {
+                       $docstring = "$signature$docstring";
+               } else {
+                       $docstring = $signature;
+               }
+
+               my $syntax_typename = "$interface->{NAME}_SyntaxType";
+
+               $self->pidl("static PyTypeObject $syntax_typename = {");
+               $self->indent;
+               $self->pidl("PyObject_HEAD_INIT(NULL) 0,");
+               $self->pidl(".tp_name = \"$basename.$interface->{NAME}\",");
+               $self->pidl(".tp_basicsize = sizeof(pytalloc_Object),");
+               $self->pidl(".tp_doc = $docstring,");
+               $self->pidl(".tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,");
+               $self->pidl(".tp_new = syntax_$interface->{NAME}_new,");
+               $self->deindent;
+               $self->pidl("};");
+
+               $self->pidl("");
+
+               $self->register_module_typeobject("abstract_syntax", "&$syntax_typename");
+               my $ndr_typename = $self->import_type_variable("samba.dcerpc.misc", "ndr_syntax_id");
+               $self->register_module_prereadycode(["$syntax_typename.tp_base = $ndr_typename;", ""]);
        }
 
        $self->pidl_hdr("\n");
@@ -763,8 +852,11 @@ sub register_module_import($$)
        $var_name =~ s/\./_/g;
        $var_name = "dep_$var_name";
 
-       $self->{module_imports}->{$var_name} = $module_path;
-
+       unless (defined $self->{module_imports_uniq}->{$var_name}) {
+               my $h = { "key" => $var_name, "val" => $module_path};
+               push @{$self->{module_imports}}, $h;
+               $self->{module_imports_uniq}->{$var_name} = $h;
+       }
        return $var_name;
 }
 
@@ -773,8 +865,10 @@ sub import_type_variable($$$)
        my ($self, $module, $name) = @_;
 
        $self->register_module_import($module);
-       unless (defined($self->{type_imports}->{$name})) {
-               $self->{type_imports}->{$name} = $module;
+       unless (defined $self->{type_imports_uniq}->{$name}) {
+               my $h = { "key" => $name, "val" => $module};
+               push @{$self->{type_imports}}, $h;
+               $self->{type_imports_uniq}->{$name} = $h;
        }
        return "$name\_Type";
 }
@@ -782,11 +876,17 @@ sub import_type_variable($$$)
 sub use_type_variable($$)
 {
        my ($self, $orig_ctype) = @_;
-       # FIXME: Have a global lookup table for types that look different on the 
+       # FIXME: Have a global lookup table for types that look different on the
        # wire than they are named in C?
-       if ($orig_ctype->{NAME} eq "dom_sid2" or $orig_ctype->{NAME} eq "dom_sid28") {
+       if ($orig_ctype->{NAME} eq "dom_sid2" or
+           $orig_ctype->{NAME} eq "dom_sid28" or
+           $orig_ctype->{NAME} eq "dom_sid0") {
                $orig_ctype->{NAME} = "dom_sid";
        }
+       if ($orig_ctype->{NAME} eq "spoolss_security_descriptor") {
+               $orig_ctype->{NAME} = "security_descriptor";
+       }
+
        my $ctype = resolveType($orig_ctype);
        unless (defined($ctype->{BASEFILE})) {
                return undef;
@@ -853,14 +953,58 @@ sub ConvertObjectFromPythonData($$$$$$;$)
                $actual_ctype = $actual_ctype->{DATA};
        }
 
-       if ($actual_ctype->{TYPE} eq "ENUM" or $actual_ctype->{TYPE} eq "BITMAP") {
+       # We need to cover ENUMs, BITMAPS and SCALAR values here, as
+       # all could otherwise be assigned invalid integer values
+       my $ctype_alias = "";
+       my $uint_max = "";
+       if ($actual_ctype->{TYPE} eq "ENUM") {
+               # Importantly, ENUM values are unsigned in pidl, and
+               # typically map to uint32
+               $ctype_alias = enum_type_fn($actual_ctype);
+       } elsif ($actual_ctype->{TYPE} eq "BITMAP") {
+               $ctype_alias = bitmap_type_fn($actual_ctype);
+       } elsif ($actual_ctype->{TYPE} eq "SCALAR") {
+               $ctype_alias = expandAlias($actual_ctype->{NAME});
+       }
+
+       # This is the unsigned Python Integer -> C integer validation
+       # case.  The signed case is below.
+       if ($ctype_alias  =~ /^(uint[0-9]*|hyper|udlong|udlongr
+                                |NTTIME_hyper|NTTIME|NTTIME_1sec
+                                |uid_t|gid_t)$/x) {
+               $self->pidl("{");
+               $self->indent;
+               $self->pidl("const unsigned long long uint_max = ndr_sizeof2uintmax(sizeof($target));");
                $self->pidl("if (PyLong_Check($cvar)) {");
                $self->indent;
-               $self->pidl("$target = PyLong_AsLongLong($cvar);");
+               $self->pidl("unsigned long long test_var;");
+               $self->pidl("test_var = PyLong_AsUnsignedLongLong($cvar);");
+               $self->pidl("if (PyErr_Occurred() != NULL) {");
+               $self->indent;
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("if (test_var > uint_max) {");
+               $self->indent;
+               $self->pidl("PyErr_Format(PyExc_OverflowError, \"Expected type %s or %s within range 0 - %llu, got %llu\",\\");
+               $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name, uint_max, test_var);");
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("$target = test_var;");
                $self->deindent;
                $self->pidl("} else if (PyInt_Check($cvar)) {");
                $self->indent;
-               $self->pidl("$target = PyInt_AsLong($cvar);");
+               $self->pidl("long test_var;");
+               $self->pidl("test_var = PyInt_AsLong($cvar);");
+               $self->pidl("if (test_var < 0 || test_var > uint_max) {");
+               $self->indent;
+               $self->pidl("PyErr_Format(PyExc_OverflowError, \"Expected type %s or %s within range 0 - %llu, got %ld\",\\");
+               $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name, uint_max, test_var);");
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("$target = test_var;");
                $self->deindent;
                $self->pidl("} else {");
                $self->indent;
@@ -869,32 +1013,61 @@ sub ConvertObjectFromPythonData($$$$$$;$)
                $self->pidl($fail);
                $self->deindent;
                $self->pidl("}");
+               $self->deindent;
+               $self->pidl("}");
                return;
        }
-       if ($actual_ctype->{TYPE} eq "SCALAR" ) {
-               if (expandAlias($actual_ctype->{NAME}) =~ /^(u?int64|hyper|dlong|udlong|udlongr|NTTIME_hyper|NTTIME|NTTIME_1sec)$/) {
-                       $self->pidl("if (PyLong_Check($cvar)) {");
-                       $self->indent;
-                       $self->pidl("$target = PyLong_AsLongLong($cvar);");
-                       $self->deindent;
-                       $self->pidl("} else if (PyInt_Check($cvar)) {");
-                       $self->indent;
-                       $self->pidl("$target = PyInt_AsLong($cvar);");
-                       $self->deindent;
-                       $self->pidl("} else {");
-                       $self->indent;
-                       $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected type %s or %s\",\\");
-                       $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name);");
-                       $self->pidl($fail);
-                       $self->deindent;
-                       $self->pidl("}");
-                       return;
-               }
-               if (expandAlias($actual_ctype->{NAME}) =~ /^(char|u?int[0-9]*|time_t)$/) {
-                       $self->pidl("PY_CHECK_TYPE(&PyInt_Type, $cvar, $fail);");
-                       $self->pidl("$target = PyInt_AsLong($cvar);");
-                       return;
-               }
+
+       # Confirm the signed python integer fits in the C type
+       # correctly.  It is subtly different from the unsigned case
+       # above, so while it looks like a duplicate, it is not
+       # actually a duplicate.
+       if ($ctype_alias  =~ /^(dlong|char|int[0-9]*|time_t)$/x) {
+               $self->pidl("{");
+               $self->indent;
+               $self->pidl("const long long int_max = ndr_sizeof2intmax(sizeof($target));");
+               $self->pidl("const long long int_min = -int_max - 1;");
+               $self->pidl("if (PyLong_Check($cvar)) {");
+               $self->indent;
+               $self->pidl("long long test_var;");
+               $self->pidl("test_var = PyLong_AsLongLong($cvar);");
+               $self->pidl("if (PyErr_Occurred() != NULL) {");
+               $self->indent;
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("if (test_var < int_min || test_var > int_max) {");
+               $self->indent;
+               $self->pidl("PyErr_Format(PyExc_OverflowError, \"Expected type %s or %s within range %lld - %lld, got %lld\",\\");
+               $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name, int_min, int_max, test_var);");
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("$target = test_var;");
+               $self->deindent;
+               $self->pidl("} else if (PyInt_Check($cvar)) {");
+               $self->indent;
+               $self->pidl("long test_var;");
+               $self->pidl("test_var = PyInt_AsLong($cvar);");
+               $self->pidl("if (test_var < int_min || test_var > int_max) {");
+               $self->indent;
+               $self->pidl("PyErr_Format(PyExc_OverflowError, \"Expected type %s or %s within range %lld - %lld, got %ld\",\\");
+               $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name, int_min, int_max, test_var);");
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("$target = test_var;");
+               $self->deindent;
+               $self->pidl("} else {");
+               $self->indent;
+               $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected type %s or %s\",\\");
+               $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name);");
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
+               $self->deindent;
+               $self->pidl("}");
+               return;
        }
 
        if ($actual_ctype->{TYPE} eq "STRUCT" or $actual_ctype->{TYPE} eq "INTERFACE") {
@@ -905,13 +1078,13 @@ sub ConvertObjectFromPythonData($$$$$$;$)
                        return;
                }
                $self->pidl("PY_CHECK_TYPE($ctype_name, $cvar, $fail);");
-               $self->pidl("if (talloc_reference($mem_ctx, py_talloc_get_mem_ctx($cvar)) == NULL) {");
+               $self->pidl("if (talloc_reference($mem_ctx, pytalloc_get_mem_ctx($cvar)) == NULL) {");
                $self->indent;
                $self->pidl("PyErr_NoMemory();");
                $self->pidl("$fail");
                $self->deindent;
                $self->pidl("}");
-               $self->assign($target, "(".mapTypeName($ctype)." *)py_talloc_get_ptr($cvar)");
+               $self->assign($target, "(".mapTypeName($ctype)." *)pytalloc_get_ptr($cvar)");
                return;
        }
 
@@ -920,7 +1093,7 @@ sub ConvertObjectFromPythonData($$$$$$;$)
                return;
        }
 
-       if ($actual_ctype->{TYPE} eq "SCALAR" and 
+       if ($actual_ctype->{TYPE} eq "SCALAR" and
                ($actual_ctype->{NAME} eq "string" or $actual_ctype->{NAME} eq "nbt_string" or $actual_ctype->{NAME} eq "nbt_name" or $actual_ctype->{NAME} eq "wrepl_nbt_name")) {
                $self->pidl("$target = talloc_strdup($mem_ctx, PyString_AS_STRING($cvar));");
                return;
@@ -961,6 +1134,11 @@ sub ConvertObjectFromPythonData($$$$$$;$)
                return;
        }
 
+       if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "HRESULT") {
+               $self->pidl("$target = HRES_ERROR(PyInt_AsLong($cvar));");
+               return;
+       }
+
        if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "string_array") {
                $self->pidl("$target = PyCObject_AsVoidPtr($cvar);");
                return;
@@ -979,6 +1157,13 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
 {
        my ($self, $env, $mem_ctx, $py_var, $e, $l, $var_name, $fail) = @_;
        my $nl = GetNextLevel($e, $l);
+       if ($nl and $nl->{TYPE} eq "SUBCONTEXT") {
+               $nl = GetNextLevel($e, $nl);
+       }
+       my $pl = GetPrevLevel($e, $l);
+       if ($pl and $pl->{TYPE} eq "SUBCONTEXT") {
+               $pl = GetPrevLevel($e, $pl);
+       }
 
        if ($l->{TYPE} eq "POINTER") {
                if ($l->{POINTER_TYPE} ne "ref") {
@@ -993,6 +1178,9 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                # then this is where we would need to allocate it
                if ($l->{POINTER_TYPE} eq "ref") {
                        $self->pidl("$var_name = talloc_ptrtype($mem_ctx, $var_name);");
+               } elsif ($nl->{TYPE} eq "DATA" and Parse::Pidl::Typelist::is_scalar($nl->{DATA_TYPE})
+                        and not Parse::Pidl::Typelist::scalar_is_reference($nl->{DATA_TYPE})) {
+                       $self->pidl("$var_name = talloc_ptrtype($mem_ctx, $var_name);");
                } else {
                        $self->pidl("$var_name = NULL;");
                }
@@ -1005,7 +1193,6 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                        $self->pidl("}");
                }
        } elsif ($l->{TYPE} eq "ARRAY") {
-               my $pl = GetPrevLevel($e, $l);
                if ($pl && $pl->{TYPE} eq "POINTER") {
                        $var_name = get_pointer_to($var_name);
                }
@@ -1023,7 +1210,7 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                        $self->pidl("} else {");
                        $self->indent;
                        $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected string or unicode object, got %s\", Py_TYPE($py_var)->tp_name);");
-                       $self->pidl("$fail;");
+                       $self->pidl("$fail");
                        $self->deindent;
                        $self->pidl("}");
                } else {
@@ -1036,10 +1223,17 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                                $self->pidl("$var_name = talloc_array_ptrtype($mem_ctx, $var_name, PyList_GET_SIZE($py_var));");
                                $self->pidl("if (!$var_name) { $fail; }");
                                $self->pidl("talloc_set_name_const($var_name, \"ARRAY: $var_name\");");
+                       } else {
+                               $self->pidl("if (ARRAY_SIZE($var_name) != PyList_GET_SIZE($py_var)) {");
+                               $self->indent;
+                               $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected list of type %s, length %zu, got %zd\", Py_TYPE($py_var)->tp_name, ARRAY_SIZE($var_name),  PyList_GET_SIZE($py_var));");
+                               $self->pidl("$fail");
+                               $self->deindent;
+                               $self->pidl("}");
                        }
                        $self->pidl("for ($counter = 0; $counter < PyList_GET_SIZE($py_var); $counter++) {");
                        $self->indent;
-                       $self->ConvertObjectFromPythonLevel($env, $var_name, "PyList_GET_ITEM($py_var, $counter)", $e, GetNextLevel($e, $l), $var_name."[$counter]", $fail);
+                       $self->ConvertObjectFromPythonLevel($env, $var_name, "PyList_GET_ITEM($py_var, $counter)", $e, $nl, $var_name."[$counter]", $fail);
                        $self->deindent;
                        $self->pidl("}");
                        $self->deindent;
@@ -1056,15 +1250,15 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                my $switch_ptr = "$e->{NAME}_switch_$l->{LEVEL_INDEX}";
                $self->pidl("{");
                $self->indent;
-               my $union_type = mapTypeName(GetNextLevel($e, $l)->{DATA_TYPE});
+               my $union_type = mapTypeName($nl->{DATA_TYPE});
                $self->pidl("$union_type *$switch_ptr;");
-               $self->pidl("$switch_ptr = py_export_" . GetNextLevel($e, $l)->{DATA_TYPE} . "($mem_ctx, $switch, $py_var);");
-               $self->pidl("if ($switch_ptr == NULL) { $fail }");
+               $self->pidl("$switch_ptr = py_export_" . $nl->{DATA_TYPE} . "($mem_ctx, $switch, $py_var);");
+               $self->fail_on_null($switch_ptr, $fail);
                $self->assign($var_name, "$switch_ptr");
                $self->deindent;
                $self->pidl("}");
        } elsif ($l->{TYPE} eq "SUBCONTEXT") {
-               $self->ConvertObjectFromPythonLevel($env, $mem_ctx, $py_var, $e, GetNextLevel($e, $l), $var_name, $fail);
+               $self->ConvertObjectFromPythonLevel($env, $mem_ctx, $py_var, $e, $nl, $var_name, $fail);
        } else {
                fatal($e->{ORIGINAL}, "unknown level type $l->{TYPE}");
        }
@@ -1085,14 +1279,30 @@ sub ConvertScalarToPython($$$)
 
        $ctypename = expandAlias($ctypename);
 
-       if ($ctypename =~ /^(u?int64|hyper|dlong|udlong|udlongr|NTTIME_hyper|NTTIME|NTTIME_1sec)$/) {
-               return "PyLong_FromLongLong($cvar)";
+       if ($ctypename =~ /^(int64|dlong)$/) {
+               return "ndr_PyLong_FromLongLong($cvar)";
+       }
+
+       if ($ctypename =~ /^(uint64|hyper|NTTIME_hyper|NTTIME|NTTIME_1sec|udlong|udlongr|uid_t|gid_t)$/) {
+               return "ndr_PyLong_FromUnsignedLongLong($cvar)";
        }
 
-       if ($ctypename =~ /^(char|u?int[0-9]*|time_t)$/) {
+       if ($ctypename =~ /^(char|int|int8|int16|int32|time_t)$/) {
                return "PyInt_FromLong($cvar)";
        }
 
+       # Needed to ensure unsigned values in a 32 or 16 bit enum is
+       # cast correctly to a uint32_t, not sign extended to a a
+       # possibly 64 bit unsigned long.  (enums are signed in C,
+       # unsigned in NDR)
+       if ($ctypename =~ /^(uint32|uint3264)$/) {
+               return "ndr_PyLong_FromUnsignedLongLong((uint32_t)$cvar)";
+       }
+
+       if ($ctypename =~ /^(uint|uint8|uint16|uint1632)$/) {
+               return "PyInt_FromLong((uint16_t)$cvar)";
+       }
+
        if ($ctypename eq "DATA_BLOB") {
                return "PyString_FromStringAndSize((char *)($cvar).data, ($cvar).length)";
        }
@@ -1105,6 +1315,10 @@ sub ConvertScalarToPython($$$)
                return "PyErr_FromWERROR($cvar)";
        }
 
+       if ($ctypename eq "HRESULT") {
+               return "PyErr_FromHRESULT($cvar)";
+       }
+
        if (($ctypename eq "string" or $ctypename eq "nbt_string" or $ctypename eq "nbt_name" or $ctypename eq "wrepl_nbt_name")) {
                return "PyString_FromStringOrNULL($cvar)";
        }
@@ -1114,13 +1328,13 @@ sub ConvertScalarToPython($$$)
        }
 
        # Not yet supported
-       if ($ctypename eq "string_array") { return "PyCObject_FromTallocPtr($cvar)"; }
+       if ($ctypename eq "string_array") { return "pytalloc_CObject_FromTallocPtr($cvar)"; }
        if ($ctypename eq "ipv4address") { return "PyString_FromStringOrNULL($cvar)"; }
        if ($ctypename eq "ipv6address") { return "PyString_FromStringOrNULL($cvar)"; }
        if ($ctypename eq "dnsp_name") { return "PyString_FromStringOrNULL($cvar)"; }
        if ($ctypename eq "dnsp_string") { return "PyString_FromStringOrNULL($cvar)"; }
        if ($ctypename eq "pointer") {
-               return "PyCObject_FromTallocPtr($cvar)";
+               return "pytalloc_CObject_FromTallocPtr($cvar)";
        }
 
        die("Unknown scalar type $ctypename");
@@ -1137,8 +1351,8 @@ sub ConvertObjectToPythonData($$$$$;$)
        my $actual_ctype = $ctype;
        if ($actual_ctype->{TYPE} eq "TYPEDEF") {
                $actual_ctype = $actual_ctype->{DATA};
-       } 
-       
+       }
+
        if ($actual_ctype->{TYPE} eq "ENUM") {
                return $self->ConvertScalarToPython(Parse::Pidl::Typelist::enum_type_fn($actual_ctype), $cvar);
        } elsif ($actual_ctype->{TYPE} eq "BITMAP") {
@@ -1153,7 +1367,7 @@ sub ConvertObjectToPythonData($$$$$;$)
                        error($location, "Unable to determine origin of type `" . mapTypeName($ctype) . "'");
                        return "NULL"; # FIXME!
                }
-               return "py_talloc_reference_ex($ctype_name, $mem_ctx, $cvar)";
+               return "pytalloc_reference_ex($ctype_name, $mem_ctx, $cvar)";
        }
 
        fatal($location, "unknown type $actual_ctype->{TYPE} for ".mapTypeName($ctype) . ": $cvar");
@@ -1173,6 +1387,13 @@ sub ConvertObjectToPythonLevel($$$$$$)
 {
        my ($self, $mem_ctx, $env, $e, $l, $var_name, $py_var, $fail) = @_;
        my $nl = GetNextLevel($e, $l);
+       if ($nl and $nl->{TYPE} eq "SUBCONTEXT") {
+               $nl = GetNextLevel($e, $nl);
+       }
+       my $pl = GetPrevLevel($e, $l);
+       if ($pl and $pl->{TYPE} eq "SUBCONTEXT") {
+               $pl = GetPrevLevel($e, $pl);
+       }
 
        if ($l->{TYPE} eq "POINTER") {
                if ($l->{POINTER_TYPE} ne "ref") {
@@ -1194,7 +1415,6 @@ sub ConvertObjectToPythonLevel($$$$$$)
                        $self->pidl("}");
                }
        } elsif ($l->{TYPE} eq "ARRAY") {
-               my $pl = GetPrevLevel($e, $l);
                if ($pl && $pl->{TYPE} eq "POINTER") {
                        $var_name = get_pointer_to($var_name);
                }
@@ -1225,11 +1445,11 @@ sub ConvertObjectToPythonLevel($$$$$$)
                        $self->indent;
                        my $counter = "$e->{NAME}_cntr_$l->{LEVEL_INDEX}";
                        $self->pidl("int $counter;");
-                       $self->pidl("for ($counter = 0; $counter < $length; $counter++) {");
+                       $self->pidl("for ($counter = 0; $counter < ($length); $counter++) {");
                        $self->indent;
                        my $member_var = "py_$e->{NAME}_$l->{LEVEL_INDEX}";
                        $self->pidl("PyObject *$member_var;");
-                       $self->ConvertObjectToPythonLevel($var_name, $env, $e, GetNextLevel($e, $l), $var_name."[$counter]", $member_var, $fail);
+                       $self->ConvertObjectToPythonLevel($var_name, $env, $e, $nl, $var_name."[$counter]", $member_var, $fail);
                        $self->pidl("PyList_SetItem($py_var, $counter, $member_var);");
                        $self->deindent;
                        $self->pidl("}");
@@ -1239,7 +1459,7 @@ sub ConvertObjectToPythonLevel($$$$$$)
        } elsif ($l->{TYPE} eq "SWITCH") {
                $var_name = get_pointer_to($var_name);
                my $switch = ParseExpr($l->{SWITCH_IS}, $env, $e);
-               $self->pidl("$py_var = py_import_" . GetNextLevel($e, $l)->{DATA_TYPE} . "($mem_ctx, $switch, $var_name);");
+               $self->pidl("$py_var = py_import_" . $nl->{DATA_TYPE} . "($mem_ctx, $switch, $var_name);");
                $self->fail_on_null($py_var, $fail);
 
        } elsif ($l->{TYPE} eq "DATA") {
@@ -1249,7 +1469,7 @@ sub ConvertObjectToPythonLevel($$$$$$)
                my $conv = $self->ConvertObjectToPythonData($mem_ctx, $l->{DATA_TYPE}, $var_name, $e->{ORIGINAL});
                $self->pidl("$py_var = $conv;");
        } elsif ($l->{TYPE} eq "SUBCONTEXT") {
-               $self->ConvertObjectToPythonLevel($mem_ctx, $env, $e, GetNextLevel($e, $l), $var_name, $py_var, $fail);
+               $self->ConvertObjectToPythonLevel($mem_ctx, $env, $e, $nl, $var_name, $py_var, $fail);
        } else {
                fatal($e->{ORIGINAL}, "Unknown level type $l->{TYPE} $var_name");
        }
@@ -1272,19 +1492,74 @@ sub Parse($$$$$)
 /* Python wrapper functions auto-generated by pidl */
 #include <Python.h>
 #include \"includes.h\"
-#include \"lib/talloc/pytalloc.h\"
+#include <pytalloc.h>
 #include \"librpc/rpc/pyrpc.h\"
 #include \"librpc/rpc/pyrpc_util.h\"
 #include \"$hdr\"
 #include \"$ndr_hdr\"
 
+/*
+ * These functions are here to ensure they can be optomised out by
+ * the compiler based on the constant input values
+ */
+
+static inline unsigned long long ndr_sizeof2uintmax(size_t var_size)
+{
+       switch (var_size) {
+       case 8:
+               return UINT64_MAX;
+       case 4:
+               return UINT32_MAX;
+       case 2:
+               return UINT16_MAX;
+       case 1:
+               return UINT8_MAX;
+       }
+
+       return 0;
+}
+
+static inline long long ndr_sizeof2intmax(size_t var_size)
+{
+       switch (var_size) {
+       case 8:
+               return INT64_MAX;
+       case 4:
+               return INT32_MAX;
+       case 2:
+               return INT16_MAX;
+       case 1:
+               return INT8_MAX;
+       }
+
+       return 0;
+}
+
+static inline PyObject *ndr_PyLong_FromLongLong(long long v)
+{
+       if (v > LONG_MAX || v < LONG_MIN) {
+               return PyLong_FromLongLong(v);
+       } else {
+               return PyInt_FromLong(v);
+       }
+}
+
+static inline PyObject *ndr_PyLong_FromUnsignedLongLong(unsigned long long v)
+{
+       if (v > LONG_MAX) {
+               return PyLong_FromUnsignedLongLong(v);
+       } else {
+               return PyInt_FromLong(v);
+       }
+}
+
 ");
 
        foreach my $x (@$ndr) {
                ($x->{TYPE} eq "IMPORT") && $self->Import(@{$x->{PATHS}});
            ($x->{TYPE} eq "INTERFACE") && $self->Interface($x, $basename);
        }
-       
+
        $self->pidl("static PyMethodDef $basename\_methods[] = {");
        $self->indent;
        foreach (@{$self->{module_methods}}) {
@@ -1298,29 +1573,30 @@ sub Parse($$$$$)
 
        $self->pidl("");
 
+       $self->pidl_hdr("void init$basename(void);");
        $self->pidl("void init$basename(void)");
        $self->pidl("{");
        $self->indent;
        $self->pidl("PyObject *m;");
-       foreach (keys %{$self->{module_imports}}) {
-               $self->pidl("PyObject *$_;");
+       foreach my $h (@{$self->{module_imports}}) {
+               $self->pidl("PyObject *$h->{'key'};");
        }
        $self->pidl("");
 
-       foreach (keys %{$self->{module_imports}}) {
-               my $var_name = $_;
-               my $module_path = $self->{module_imports}->{$var_name};
+       foreach my $h (@{$self->{module_imports}}) {
+               my $var_name = $h->{'key'};
+               my $module_path = $h->{'val'};
                $self->pidl("$var_name = PyImport_ImportModule(\"$module_path\");");
                $self->pidl("if ($var_name == NULL)");
                $self->pidl("\treturn;");
                $self->pidl("");
        }
 
-       foreach (keys %{$self->{type_imports}}) {
-               my $type_var = "$_\_Type";
-               my $module_path = $self->{type_imports}->{$_};
+       foreach my $h (@{$self->{type_imports}}) {
+               my $type_var = "$h->{'key'}\_Type";
+               my $module_path = $h->{'val'};
                $self->pidl_hdr("static PyTypeObject *$type_var;\n");
-               my $pretty_name = PrettifyTypeName($_, $module_path);
+               my $pretty_name = PrettifyTypeName($h->{'key'}, $module_path);
                my $module_var = "dep_$module_path";
                $module_var =~ s/\./_/g;
                $self->pidl("$type_var = (PyTypeObject *)PyObject_GetAttrString($module_var, \"$pretty_name\");");
@@ -1351,11 +1627,12 @@ sub Parse($$$$$)
        $self->pidl("if (m == NULL)");
        $self->pidl("\treturn;");
        $self->pidl("");
-       foreach my $name (keys %{$self->{constants}}) {
+       foreach my $h (@{$self->{constants}}) {
+               my $name = $h->{'key'};
                my $py_obj;
-               my ($ctype, $cvar) = @{$self->{constants}->{$name}};
+               my ($ctype, $cvar) = @{$h->{'val'}};
                if ($cvar =~ /^[0-9]+$/ or $cvar =~ /^0x[0-9a-fA-F]+$/) {
-                       $py_obj = "PyInt_FromLong($cvar)";
+                       $py_obj = "ndr_PyLong_FromUnsignedLongLong($cvar)";
                } elsif ($cvar =~ /^".*"$/) {
                        $py_obj = "PyString_FromString($cvar)";
                } else {