pidl: Allow more flexibility importing modules using convenience function.
[kai/samba.git] / pidl / lib / Parse / Pidl / Samba4 / Python.pm
index dd93605527b90e780d32f2a928a1196b5cfe7a36..ebd2a66af38dda29abf1202ad7ceb3ec6e8bd267 100644 (file)
@@ -24,8 +24,9 @@ sub new($) {
        my ($class) = @_;
        my $self = { res => "", res_hdr => "", tabs => "", constants => {},
                     module_methods => [], module_objects => [], ready_types => [],
-                                module_imports => [], type_imports => {},
-                                patch_type_calls => [], readycode => [] };
+                                module_imports => {}, type_imports => {},
+                                patch_type_calls => [], prereadycode => [],
+                                postreadycode => []};
        bless($self, $class);
 }
 
@@ -126,6 +127,7 @@ sub FromUnionToPythonFunction($$$$)
                        $self->ConvertObjectToPython($mem_ctx, {}, $e, "$name->$e->{NAME}", "ret", "return NULL;");
                } else {
                        $self->pidl("ret = Py_None;");
+                       $self->pidl("Py_INCREF(ret);");
                }
 
                $self->pidl("return ret;");
@@ -195,7 +197,7 @@ sub PythonStruct($$$$$$)
                        $self->pidl("static PyObject *py_$name\_get_$e->{NAME}(PyObject *obj, void *closure)");
                        $self->pidl("{");
                        $self->indent;
-                       $self->pidl("$cname *object = py_talloc_get_ptr(obj);");
+                       $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(obj);");
                        $self->pidl("PyObject *py_$e->{NAME};");
                        $self->ConvertObjectToPython("py_talloc_get_mem_ctx(obj)", $env, $e, $varname, "py_$e->{NAME}", "return NULL;");
                        $self->pidl("return py_$e->{NAME};");
@@ -206,14 +208,14 @@ sub PythonStruct($$$$$$)
                        $self->pidl("static int py_$name\_set_$e->{NAME}(PyObject *py_obj, PyObject *value, void *closure)");
                        $self->pidl("{");
                        $self->indent;
-                       $self->pidl("$cname *object = py_talloc_get_ptr(py_obj);");
+                       $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
                        my $mem_ctx = "py_talloc_get_mem_ctx(py_obj)";
                        my $l = $e->{LEVELS}[0];
                        my $nl = GetNextLevel($e, $l);
                        if ($l->{TYPE} eq "POINTER" and 
                                not ($nl->{TYPE} eq "ARRAY" and ($nl->{IS_FIXED} or is_charset_array($e, $nl))) and
                                not ($nl->{TYPE} eq "DATA" and Parse::Pidl::Typelist::scalar_is_reference($nl->{DATA_TYPE}))) {
-                               $self->pidl("talloc_free($varname);");
+                               $self->pidl("talloc_unlink(py_talloc_get_mem_ctx(py_obj), $varname);");
                        }
                        $self->ConvertObjectFromPython($env, $mem_ctx, $e, "value", $varname, "return -1;");
                        $self->pidl("return 0;");
@@ -250,10 +252,10 @@ sub PythonStruct($$$$$$)
                $self->pidl("static PyObject *py_$name\_ndr_pack(PyObject *py_obj)");
                $self->pidl("{");
                $self->indent;
-               $self->pidl("$cname *object = py_talloc_get_ptr(py_obj);");
+               $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
                $self->pidl("DATA_BLOB blob;");
                $self->pidl("enum ndr_err_code err;");
-               $self->pidl("err = ndr_push_struct_blob(&blob, py_talloc_get_mem_ctx(py_obj), NULL, object, (ndr_push_flags_fn_t)ndr_push_$name);");
+               $self->pidl("err = ndr_push_struct_blob(&blob, py_talloc_get_mem_ctx(py_obj), object, (ndr_push_flags_fn_t)ndr_push_$name);");
                $self->pidl("if (err != NDR_ERR_SUCCESS) {");
                $self->indent;
                $self->pidl("PyErr_SetNdrError(err);");
@@ -269,13 +271,13 @@ sub PythonStruct($$$$$$)
                $self->pidl("static PyObject *py_$name\_ndr_unpack(PyObject *py_obj, PyObject *args)");
                $self->pidl("{");
                $self->indent;
-               $self->pidl("$cname *object = py_talloc_get_ptr(py_obj);");
+               $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
                $self->pidl("DATA_BLOB blob;");
                $self->pidl("enum ndr_err_code err;");
                $self->pidl("if (!PyArg_ParseTuple(args, \"s#:__ndr_unpack__\", &blob.data, &blob.length))");
                $self->pidl("\treturn NULL;");
                $self->pidl("");
-               $self->pidl("err = ndr_pull_struct_blob_all(&blob, py_talloc_get_mem_ctx(py_obj), NULL, object, (ndr_pull_flags_fn_t)ndr_pull_$name);");
+               $self->pidl("err = ndr_pull_struct_blob_all(&blob, py_talloc_get_mem_ctx(py_obj), object, (ndr_pull_flags_fn_t)ndr_pull_$name);");
                $self->pidl("if (err != NDR_ERR_SUCCESS) {");
                $self->indent;
                $self->pidl("PyErr_SetNdrError(err);");
@@ -287,11 +289,24 @@ sub PythonStruct($$$$$$)
                $self->deindent;
                $self->pidl("}");
                $self->pidl("");
+
+               $self->pidl("static PyObject *py_$name\_ndr_print(PyObject *py_obj)");
+               $self->pidl("{");
+               $self->indent;
+               $self->pidl("$cname *object = ($cname *)py_talloc_get_ptr(py_obj);");
+               $self->pidl("char *retstr;");
+               $self->pidl("retstr = ndr_print_struct_string(py_talloc_get_mem_ctx(py_obj), (ndr_print_fn_t)ndr_print_$name, \"$name\", object);");
+               $self->pidl("return PyString_FromString(retstr);");
+               $self->deindent;
+               $self->pidl("}");
+               $self->pidl("");
+
                $py_methods = "py_$name\_methods";
                $self->pidl("static PyMethodDef $py_methods\[] = {");
                $self->indent;
-               $self->pidl("{ \"__ndr_pack__\", (PyCFunction)py_$name\_ndr_pack, METH_NOARGS, \"S.pack() -> blob\\nNDR pack\" },");
-               $self->pidl("{ \"__ndr_unpack__\", (PyCFunction)py_$name\_ndr_unpack, METH_VARARGS, \"S.unpack(blob) -> None\\nNDR unpack\" },");
+               $self->pidl("{ \"__ndr_pack__\", (PyCFunction)py_$name\_ndr_pack, METH_NOARGS, \"S.ndr_pack(object) -> blob\\nNDR pack\" },");
+               $self->pidl("{ \"__ndr_unpack__\", (PyCFunction)py_$name\_ndr_unpack, METH_VARARGS, \"S.ndr_unpack(class, blob) -> None\\nNDR unpack\" },");
+               $self->pidl("{ \"__ndr_print__\", (PyCFunction)py_$name\_ndr_print, METH_VARARGS, \"S.ndr_print(object) -> None\\nNDR print\" },");
                $self->pidl("{ NULL, NULL, 0, NULL }");
                $self->deindent;
                $self->pidl("};");
@@ -306,21 +321,22 @@ sub PythonStruct($$$$$$)
        $self->indent;
        $self->pidl("PyObject_HEAD_INIT(NULL) 0,");
        $self->pidl(".tp_name = \"$modulename.$prettyname\",");
-       $self->pidl(".tp_basicsize = sizeof(py_talloc_Object),");
-       $self->pidl(".tp_dealloc = py_talloc_dealloc,");
        $self->pidl(".tp_getset = $getsetters,");
-       $self->pidl(".tp_repr = py_talloc_default_repr,");
        if ($docstring) {
                $self->pidl(".tp_doc = $docstring,");
        }
        $self->pidl(".tp_methods = $py_methods,");
        $self->pidl(".tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,");
+       $self->pidl(".tp_basicsize = sizeof(py_talloc_Object),");
        $self->pidl(".tp_new = py_$name\_new,");
        $self->deindent;
        $self->pidl("};");
 
        $self->pidl("");
 
+       my $talloc_typename = $self->import_type_variable("talloc", "Object");
+       $self->register_module_prereadycode(["$name\_Type.tp_base = $talloc_typename;", ""]);
+
        return "&$typeobject";
 }
 
@@ -371,7 +387,7 @@ sub PythonFunctionUnpackOut($$$)
        $self->pidl("static PyObject *$outfnname(struct $fn->{NAME} *r)");
        $self->pidl("{");
        $self->indent;
-       $self->pidl("PyObject *result = Py_None;");
+       $self->pidl("PyObject *result;");
        foreach my $e (@{$fn->{ELEMENTS}}) {
                next unless (grep(/out/,@{$e->{DIRECTION}}));
                next if (($metadata_args->{in}->{$e->{NAME}} and grep(/in/, @{$e->{DIRECTION}})) or 
@@ -390,6 +406,8 @@ sub PythonFunctionUnpackOut($$$)
                $self->pidl("result = PyTuple_New($result_size);");
                $signature .= "(";
        } elsif ($result_size == 0) {
+               $self->pidl("result = Py_None;");
+               $self->pidl("Py_INCREF(result);");
                $signature .= "None";
        }
 
@@ -414,7 +432,7 @@ sub PythonFunctionUnpackOut($$$)
        } elsif (defined($fn->{RETURN_TYPE}) and $fn->{RETURN_TYPE} eq "WERROR") {
                $self->handle_werror("r->out.result", "NULL", undef);
        } elsif (defined($fn->{RETURN_TYPE})) {
-               my $conv = $self->ConvertObjectToPythonData("r", $fn->{RETURN_TYPE}, "r->out.result");
+               my $conv = $self->ConvertObjectToPythonData("r", $fn->{RETURN_TYPE}, "r->out.result", $fn);
                if ($result_size > 1) {
                        $self->pidl("PyTuple_SetItem(result, $i, $conv);");
                } else {
@@ -488,7 +506,7 @@ sub PythonFunctionPackIn($$$)
                if ($metadata_args->{in}->{$e->{NAME}}) {
                        my $py_var = "py_".$metadata_args->{in}->{$e->{NAME}};
                        $self->pidl("PY_CHECK_TYPE(&PyList_Type, $py_var, $fail);");
-                       my $val = "PyList_Size($py_var)";
+                       my $val = "PyList_GET_SIZE($py_var)";
                        if ($e->{LEVELS}[0]->{TYPE} eq "POINTER") {
                                $self->pidl("r->in.$e->{NAME} = talloc_ptrtype(r, r->in.$e->{NAME});");
                                $self->pidl("*r->in.$e->{NAME} = $val;");
@@ -513,22 +531,13 @@ sub PythonFunction($$$)
        my $fnname = "py_$fn->{NAME}";
        my $docstring = $self->DocString($fn, $fn->{NAME});
 
-       my ($insignature, $outsignature);
-       my ($infn, $outfn);
-
-       if (has_property($fn, "todo")) {
-               unless ($docstring) { $docstring = "NULL"; }
-               $infn = "NULL";
-               $outfn = "NULL";
+       my ($infn, $insignature) = $self->PythonFunctionPackIn($fn, $fnname);
+       my ($outfn, $outsignature) = $self->PythonFunctionUnpackOut($fn, $fnname);
+       my $signature = "S.$prettyname($insignature) -> $outsignature";
+       if ($docstring) {
+               $docstring = "\"$signature\\n\\n\"$docstring";
        } else {
-               ($infn, $insignature) = $self->PythonFunctionPackIn($fn, $fnname);
-               ($outfn, $outsignature) = $self->PythonFunctionUnpackOut($fn, $fnname);
-               my $signature = "S.$prettyname($insignature) -> $outsignature";
-               if ($docstring) {
-                       $docstring = "\"$signature\\n\\n\"$docstring";
-               } else {
-                       $docstring = "\"$signature\"";
-               }
+               $docstring = "\"$signature\"";
        }
 
        return ($infn, $outfn, $docstring);
@@ -646,8 +655,9 @@ sub Interface($$$)
                my @fns = ();
 
                foreach my $d (@{$interface->{FUNCTIONS}}) {
-                       next if not defined($d->{OPNUM});
+                       next if has_property($d, "noopnum");
                        next if has_property($d, "nopython");
+                       next if has_property($d, "todo");
 
                        my $prettyname = $d->{NAME};
 
@@ -656,14 +666,14 @@ sub Interface($$$)
 
                        my ($infn, $outfn, $fndocstring) = $self->PythonFunction($d, $interface->{NAME}, $prettyname);
 
-                       push (@fns, [$infn, $outfn, "dcerpc_$d->{NAME}", $prettyname, $fndocstring, $d->{OPNUM}]);
+                       push (@fns, [$infn, $outfn, "dcerpc_$d->{NAME}_r", $prettyname, $fndocstring, $d->{OPNUM}]);
                }
 
                $self->pidl("const struct PyNdrRpcMethodDef py_ndr_$interface->{NAME}\_methods[] = {");
                $self->indent;
                foreach my $d (@fns) {
                        my ($infn, $outfn, $callfn, $prettyname, $docstring, $opnum) = @$d;
-                       $self->pidl("{ \"$prettyname\", $docstring, (dcerpc_call_fn)$callfn, (py_data_pack_fn)$infn, (py_data_unpack_fn)$outfn, $opnum, &ndr_table_$interface->{NAME} },");
+                       $self->pidl("{ \"$prettyname\", $docstring, (py_dcerpc_call_fn)$callfn, (py_data_pack_fn)$infn, (py_data_unpack_fn)$outfn, $opnum, &ndr_table_$interface->{NAME} },");
                }
                $self->pidl("{ NULL }");
                $self->deindent;
@@ -694,12 +704,13 @@ sub Interface($$$)
                        $docstring = $signature;
                }
 
-               $self->pidl("static PyTypeObject $interface->{NAME}_InterfaceType = {");
+               my $if_typename = "$interface->{NAME}_InterfaceType";
+
+               $self->pidl("static PyTypeObject $if_typename = {");
                $self->indent;
                $self->pidl("PyObject_HEAD_INIT(NULL) 0,");
                $self->pidl(".tp_name = \"$basename.$interface->{NAME}\",");
                $self->pidl(".tp_basicsize = sizeof(dcerpc_InterfaceObject),");
-               $self->pidl(".tp_base = &dcerpc_InterfaceType,");
                $self->pidl(".tp_doc = $docstring,");
                $self->pidl(".tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,");
                $self->pidl(".tp_new = interface_$interface->{NAME}_new,");
@@ -708,8 +719,10 @@ sub Interface($$$)
 
                $self->pidl("");
 
-               $self->register_module_typeobject($interface->{NAME}, "&$interface->{NAME}_InterfaceType");
-               $self->register_module_readycode(["if (!PyInterface_AddNdrRpcMethods(&$interface->{NAME}_InterfaceType, py_ndr_$interface->{NAME}\_methods))", "\treturn;", ""]);
+               $self->register_module_typeobject($interface->{NAME}, "&$if_typename");
+               my $dcerpc_typename = $self->import_type_variable("base", "ClientConnection");
+               $self->register_module_prereadycode(["$if_typename.tp_base = $dcerpc_typename;", ""]);
+               $self->register_module_postreadycode(["if (!PyInterface_AddNdrRpcMethods(&$if_typename, py_ndr_$interface->{NAME}\_methods))", "\treturn;", ""]);
        }
 
        $self->pidl_hdr("\n");
@@ -726,7 +739,7 @@ sub register_module_typeobject($$$)
 {
        my ($self, $name, $py_name) = @_;
 
-       $self->register_module_object($name, "(PyObject *)$py_name");
+       $self->register_module_object($name, "(PyObject *)(void *)$py_name");
 
        $self->check_ready_type($py_name);
 
@@ -743,23 +756,35 @@ sub register_module_import($$)
 {
        my ($self, $basename) = @_;
 
-       push (@{$self->{module_imports}}, $basename) unless (grep(/^$basename$/,@{$self->{module_imports}}));
+       $self->{module_imports}->{"dep_$basename"} = "samba.dcerpc.$basename";
+}
+
+sub import_type_variable($$$)
+{
+       my ($self, $module, $name) = @_;
+
+       $self->register_module_import($module);
+       unless (defined($self->{type_imports}->{$name})) {
+               $self->{type_imports}->{$name} = $module;
+       }
+       return "$name\_Type";
 }
 
 sub use_type_variable($$)
 {
        my ($self, $orig_ctype) = @_;
+       # FIXME: Have a global lookup table for types that look different on the 
+       # wire than they are named in C?
+       if ($orig_ctype->{NAME} eq "dom_sid2" or $orig_ctype->{NAME} eq "dom_sid28") {
+               $orig_ctype->{NAME} = "dom_sid";
+       }
        my $ctype = resolveType($orig_ctype);
        unless (defined($ctype->{BASEFILE})) {
                return undef;
        }
        # If this is an external type, make sure we do the right imports.
        if (($ctype->{BASEFILE} ne $self->{BASENAME})) {
-               $self->register_module_import($ctype->{BASEFILE});
-               unless (defined($self->{type_imports}->{$ctype->{NAME}})) {
-                       $self->{type_imports}->{$ctype->{NAME}} = $ctype->{BASEFILE};
-               }
-               return "$ctype->{NAME}_Type";
+               return $self->import_type_variable($ctype->{BASEFILE}, $ctype->{NAME});
        }
        return "&$ctype->{NAME}_Type";
 }
@@ -772,11 +797,18 @@ sub register_patch_type_call($$$)
 
 }
 
-sub register_module_readycode($$)
+sub register_module_prereadycode($$)
 {
        my ($self, $code) = @_;
 
-       push (@{$self->{readycode}}, @$code);
+       push (@{$self->{prereadycode}}, @$code);
+}
+
+sub register_module_postreadycode($$)
+{
+       my ($self, $code) = @_;
+
+       push (@{$self->{postreadycode}}, @$code);
 }
 
 sub register_module_object($$$)
@@ -792,17 +824,18 @@ sub assign($$$)
        if ($dest =~ /^\&/ and $src eq "NULL") {
                $self->pidl("memset($dest, 0, sizeof(" . get_value_of($dest) . "));");
        } elsif ($dest =~ /^\&/) {
-               $self->pidl("memcpy($dest, $src, sizeof(" . get_value_of($dest) . "));");
+               my $destvar = get_value_of($dest);
+               $self->pidl("$destvar = *$src;");
        } else {
                $self->pidl("$dest = $src;");
        }
 }
 
-sub ConvertObjectFromPythonData($$$$$$)
+sub ConvertObjectFromPythonData($$$$$$;$)
 {
-       my ($self, $mem_ctx, $cvar, $ctype, $target, $fail) = @_;
+       my ($self, $mem_ctx, $cvar, $ctype, $target, $fail, $location) = @_;
 
-       die("undef type for $cvar") unless(defined($ctype));
+       fatal($location, "undef type for $cvar") unless(defined($ctype));
 
        $ctype = resolveType($ctype);
 
@@ -811,43 +844,103 @@ sub ConvertObjectFromPythonData($$$$$$)
                $actual_ctype = $actual_ctype->{DATA};
        }
 
-       if ($actual_ctype->{TYPE} eq "ENUM" or $actual_ctype->{TYPE} eq "BITMAP" or 
-               $actual_ctype->{TYPE} eq "SCALAR" and (
-               expandAlias($actual_ctype->{NAME}) =~ /^(u?int[0-9]*|hyper|NTTIME|time_t|NTTIME_hyper|NTTIME_1sec|dlong|udlong|udlongr)$/)) {
-               $self->pidl("PY_CHECK_TYPE(&PyInt_Type, $cvar, $fail);");
+       if ($actual_ctype->{TYPE} eq "ENUM" or $actual_ctype->{TYPE} eq "BITMAP") {
+               $self->pidl("if (PyLong_Check($cvar)) {");
+               $self->indent;
+               $self->pidl("$target = PyLong_AsLongLong($cvar);");
+               $self->deindent;
+               $self->pidl("} else if (PyInt_Check($cvar)) {");
+               $self->indent;
                $self->pidl("$target = PyInt_AsLong($cvar);");
+               $self->deindent;
+               $self->pidl("} else {");
+               $self->indent;
+               $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected type %s or %s\",\\");
+               $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name);");
+               $self->pidl($fail);
+               $self->deindent;
+               $self->pidl("}");
                return;
        }
+       if ($actual_ctype->{TYPE} eq "SCALAR" ) {
+               if (expandAlias($actual_ctype->{NAME}) =~ /^(u?int64|hyper|dlong|udlong|udlongr|NTTIME_hyper|NTTIME|NTTIME_1sec)$/) {
+                       $self->pidl("if (PyLong_Check($cvar)) {");
+                       $self->indent;
+                       $self->pidl("$target = PyLong_AsLongLong($cvar);");
+                       $self->deindent;
+                       $self->pidl("} else if (PyInt_Check($cvar)) {");
+                       $self->indent;
+                       $self->pidl("$target = PyInt_AsLong($cvar);");
+                       $self->deindent;
+                       $self->pidl("} else {");
+                       $self->indent;
+                       $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected type %s or %s\",\\");
+                       $self->pidl("  PyInt_Type.tp_name, PyLong_Type.tp_name);");
+                       $self->pidl($fail);
+                       $self->deindent;
+                       $self->pidl("}");
+                       return;
+               }
+               if (expandAlias($actual_ctype->{NAME}) =~ /^(char|u?int[0-9]*|time_t)$/) {
+                       $self->pidl("PY_CHECK_TYPE(&PyInt_Type, $cvar, $fail);");
+                       $self->pidl("$target = PyInt_AsLong($cvar);");
+                       return;
+               }
+       }
 
        if ($actual_ctype->{TYPE} eq "STRUCT" or $actual_ctype->{TYPE} eq "INTERFACE") {
                my $ctype_name = $self->use_type_variable($ctype);
                unless (defined ($ctype_name)) {
-                       error(undef, "Unable to determine origin of type " . mapTypeName($ctype));
-                       $self->assign($target, "NULL");
-                       # FIXME:
+                       error($location, "Unable to determine origin of type `" . mapTypeName($ctype) . "'");
+                       $self->pidl("PyErr_SetString(PyExc_TypeError, \"Can not convert C Type " . mapTypeName($ctype) . " from Python\");");
                        return;
                }
                $self->pidl("PY_CHECK_TYPE($ctype_name, $cvar, $fail);");
-               $self->assign($target, "py_talloc_get_ptr($cvar)");
+               $self->pidl("if (talloc_reference($mem_ctx, py_talloc_get_mem_ctx($cvar)) == NULL) {");
+               $self->indent;
+               $self->pidl("PyErr_NoMemory();");
+               $self->pidl("$fail");
+               $self->deindent;
+               $self->pidl("}");
+               $self->assign($target, "(".mapTypeName($ctype)." *)py_talloc_get_ptr($cvar)");
                return;
        }
 
        if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "DATA_BLOB") {
-               $self->pidl("$target = data_blob_talloc($mem_ctx, PyString_AsString($cvar), PyString_Size($cvar));");
+               $self->pidl("$target = data_blob_talloc($mem_ctx, PyString_AS_STRING($cvar), PyString_GET_SIZE($cvar));");
                return;
        }
 
        if ($actual_ctype->{TYPE} eq "SCALAR" and 
                ($actual_ctype->{NAME} eq "string" or $actual_ctype->{NAME} eq "nbt_string" or $actual_ctype->{NAME} eq "nbt_name" or $actual_ctype->{NAME} eq "wrepl_nbt_name")) {
-               $self->pidl("$target = talloc_strdup($mem_ctx, PyString_AsString($cvar));");
+               $self->pidl("$target = talloc_strdup($mem_ctx, PyString_AS_STRING($cvar));");
+               return;
+       }
+
+       if ($actual_ctype->{TYPE} eq "SCALAR" and ($actual_ctype->{NAME} eq "dns_string" or $actual_ctype->{NAME} eq "dns_name")) {
+               $self->pidl("$target = talloc_strdup($mem_ctx, PyString_AS_STRING($cvar));");
                return;
        }
 
        if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "ipv4address") {
+               $self->pidl("$target = PyString_AS_STRING($cvar);");
+               return;
+       }
+
+       if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "ipv6address") {
                $self->pidl("$target = PyString_AsString($cvar);");
                return;
-               }
+       }
+
+       if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "dnsp_name") {
+               $self->pidl("$target = PyString_AS_STRING($cvar);");
+               return;
+       }
 
+       if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "dnsp_string") {
+               $self->pidl("$target = PyString_AS_STRING($cvar);");
+               return;
+       }
 
        if ($actual_ctype->{TYPE} eq "SCALAR" and $actual_ctype->{NAME} eq "NTSTATUS") {
                $self->pidl("$target = NT_STATUS(PyInt_AsLong($cvar));");
@@ -869,7 +962,7 @@ sub ConvertObjectFromPythonData($$$$$$)
                return;
        }
 
-       fatal($ctype, "unknown type $actual_ctype->{TYPE} for ".mapTypeName($ctype) . ": $cvar");
+       fatal($location, "unknown type `$actual_ctype->{TYPE}' for ".mapTypeName($ctype) . ": $cvar");
 
 }
 
@@ -891,7 +984,13 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                        $self->pidl("} else {");
                        $self->indent;
                }
-               $self->pidl("$var_name = talloc_ptrtype($mem_ctx, $var_name);");
+               # if we want to handle more than one level of pointer in python interfaces
+               # then this is where we would need to allocate it
+               if ($l->{POINTER_TYPE} eq "ref") {
+                       $self->pidl("$var_name = talloc_ptrtype($mem_ctx, $var_name);");
+               } else {
+                       $self->pidl("$var_name = NULL;");
+               }
                $self->ConvertObjectFromPythonLevel($env, $mem_ctx, $py_var, $e, $nl, get_value_of($var_name), $fail);
                if ($l->{POINTER_TYPE} ne "ref") {
                        $self->deindent;
@@ -904,9 +1003,21 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                }
 
                if (is_charset_array($e, $l)) {
-                       $self->pidl("PY_CHECK_TYPE(&PyUnicode_Type, $py_var, $fail);");
+                       $self->pidl("if (PyUnicode_Check($py_var)) {");
+                       $self->indent;
                        # FIXME: Use Unix charset setting rather than utf-8
-                       $self->pidl($var_name . " = PyString_AsString(PyUnicode_AsEncodedString($py_var, \"utf-8\", \"ignore\"));");
+                       $self->pidl($var_name . " = PyString_AS_STRING(PyUnicode_AsEncodedString($py_var, \"utf-8\", \"ignore\"));");
+                       $self->deindent;
+                       $self->pidl("} else if (PyString_Check($py_var)) {");
+                       $self->indent;
+                       $self->pidl($var_name . " = PyString_AS_STRING($py_var);");
+                       $self->deindent;
+                       $self->pidl("} else {");
+                       $self->indent;
+                       $self->pidl("PyErr_Format(PyExc_TypeError, \"Expected string or unicode object, got %s\", Py_TYPE($py_var)->tp_name);");
+                       $self->pidl("$fail;");
+                       $self->deindent;
+                       $self->pidl("}");
                } else {
                        my $counter = "$e->{NAME}_cntr_$l->{LEVEL_INDEX}";
                        $self->pidl("PY_CHECK_TYPE(&PyList_Type, $py_var, $fail);");
@@ -914,30 +1025,40 @@ sub ConvertObjectFromPythonLevel($$$$$$$$)
                        $self->indent;
                        $self->pidl("int $counter;");
                        if (ArrayDynamicallyAllocated($e, $l)) {
-                               $self->pidl("$var_name = talloc_array_ptrtype($mem_ctx, $var_name, PyList_Size($py_var));");
+                               $self->pidl("$var_name = talloc_array_ptrtype($mem_ctx, $var_name, PyList_GET_SIZE($py_var));");
+                               $self->pidl("if (!$var_name) { $fail; }");
+                               $self->pidl("talloc_set_name_const($var_name, \"ARRAY: $var_name\");");
                        }
-                       $self->pidl("for ($counter = 0; $counter < PyList_Size($py_var); $counter++) {");
+                       $self->pidl("for ($counter = 0; $counter < PyList_GET_SIZE($py_var); $counter++) {");
                        $self->indent;
-                       $self->ConvertObjectFromPythonLevel($env, $var_name, "PyList_GetItem($py_var, $counter)", $e, GetNextLevel($e, $l), $var_name."[$counter]", $fail);
+                       $self->ConvertObjectFromPythonLevel($env, $var_name, "PyList_GET_ITEM($py_var, $counter)", $e, GetNextLevel($e, $l), $var_name."[$counter]", $fail);
                        $self->deindent;
                        $self->pidl("}");
                        $self->deindent;
                        $self->pidl("}");
                }
        } elsif ($l->{TYPE} eq "DATA") {
-
                if (not Parse::Pidl::Typelist::is_scalar($l->{DATA_TYPE})) {
                        $var_name = get_pointer_to($var_name);
                }
-               $self->ConvertObjectFromPythonData($mem_ctx, $py_var, $l->{DATA_TYPE}, $var_name, $fail);
+               $self->ConvertObjectFromPythonData($mem_ctx, $py_var, $l->{DATA_TYPE}, $var_name, $fail, $e->{ORIGINAL});
        } elsif ($l->{TYPE} eq "SWITCH") {
                $var_name = get_pointer_to($var_name);
                my $switch = ParseExpr($l->{SWITCH_IS}, $env, $e);
-               $self->assign($var_name, "py_export_" . GetNextLevel($e, $l)->{DATA_TYPE} . "($mem_ctx, $switch, $py_var)");
+               my $switch_ptr = "$e->{NAME}_switch_$l->{LEVEL_INDEX}";
+               $self->pidl("{");
+               $self->indent;
+               my $union_type = mapTypeName(GetNextLevel($e, $l)->{DATA_TYPE});
+               $self->pidl("$union_type *$switch_ptr;");
+               $self->pidl("$switch_ptr = py_export_" . GetNextLevel($e, $l)->{DATA_TYPE} . "($mem_ctx, $switch, $py_var);");
+               $self->pidl("if ($switch_ptr == NULL) { $fail }");
+               $self->assign($var_name, "$switch_ptr");
+               $self->deindent;
+               $self->pidl("}");
        } elsif ($l->{TYPE} eq "SUBCONTEXT") {
                $self->ConvertObjectFromPythonLevel($env, $mem_ctx, $py_var, $e, GetNextLevel($e, $l), $var_name, $fail);
        } else {
-               die("unknown level type $l->{TYPE}");
+               fatal($e->{ORIGINAL}, "unknown level type $l->{TYPE}");
        }
 }
 
@@ -956,7 +1077,11 @@ sub ConvertScalarToPython($$$)
 
        $ctypename = expandAlias($ctypename);
 
-       if ($ctypename =~ /^(char|u?int[0-9]*|hyper|dlong|udlong|udlongr|time_t|NTTIME_hyper|NTTIME|NTTIME_1sec)$/) {
+       if ($ctypename =~ /^(u?int64|hyper|dlong|udlong|udlongr|NTTIME_hyper|NTTIME|NTTIME_1sec)$/) {
+               return "PyLong_FromLongLong($cvar)";
+       }
+
+       if ($ctypename =~ /^(char|u?int[0-9]*|time_t)$/) {
                return "PyInt_FromLong($cvar)";
        }
 
@@ -973,22 +1098,29 @@ sub ConvertScalarToPython($$$)
        }
 
        if (($ctypename eq "string" or $ctypename eq "nbt_string" or $ctypename eq "nbt_name" or $ctypename eq "wrepl_nbt_name")) {
-               return "PyString_FromString($cvar)";
+               return "PyString_FromString_check_null($cvar)";
+       }
+
+       if (($ctypename eq "dns_string" or $ctypename eq "dns_name")) {
+               return "PyString_FromString_check_null($cvar)";
        }
 
        # Not yet supported
-       if ($ctypename eq "string_array") { return "PyCObject_FromVoidPtr($cvar)"; }
-       if ($ctypename eq "ipv4address") { return "PyString_FromString($cvar)"; }
+       if ($ctypename eq "string_array") { return "PyCObject_FromTallocPtr($cvar)"; }
+       if ($ctypename eq "ipv4address") { return "PyString_FromString_check_null($cvar)"; }
+       if ($ctypename eq "ipv6address") { return "PyString_FromString_check_null($cvar)"; }
+       if ($ctypename eq "dnsp_name") { return "PyString_FromString_check_null($cvar)"; }
+       if ($ctypename eq "dnsp_string") { return "PyString_FromString_check_null($cvar)"; }
        if ($ctypename eq "pointer") {
-               return "PyCObject_FromVoidPtr($cvar, talloc_free)";
+               return "PyCObject_FromTallocPtr($cvar)";
        }
 
        die("Unknown scalar type $ctypename");
 }
 
-sub ConvertObjectToPythonData($$$$$)
+sub ConvertObjectToPythonData($$$$$;$)
 {
-       my ($self, $mem_ctx, $ctype, $cvar) = @_;
+       my ($self, $mem_ctx, $ctype, $cvar, $location) = @_;
 
        die("undef type for $cvar") unless(defined($ctype));
 
@@ -1010,13 +1142,13 @@ sub ConvertObjectToPythonData($$$$$)
        } elsif ($actual_ctype->{TYPE} eq "STRUCT" or $actual_ctype->{TYPE} eq "INTERFACE") {
                my $ctype_name = $self->use_type_variable($ctype);
                unless (defined($ctype_name)) {
-                       error(undef, "Unable to determine origin of type " . mapTypeName($ctype));
+                       error($location, "Unable to determine origin of type `" . mapTypeName($ctype) . "'");
                        return "NULL"; # FIXME!
                }
-               return "py_talloc_import_ex($ctype_name, $mem_ctx, $cvar)";
+               return "py_talloc_reference_ex($ctype_name, $mem_ctx, $cvar)";
        }
 
-       fatal($ctype, "unknown type $actual_ctype->{TYPE} for ".mapTypeName($ctype) . ": $cvar");
+       fatal($location, "unknown type $actual_ctype->{TYPE} for ".mapTypeName($ctype) . ": $cvar");
 }
 
 sub fail_on_null($$$)
@@ -1043,6 +1175,7 @@ sub ConvertObjectToPythonLevel($$$$$$)
                        $self->pidl("if ($var_name == NULL) {");
                        $self->indent;
                        $self->pidl("$py_var = Py_None;");
+                       $self->pidl("Py_INCREF($py_var);");
                        $self->deindent;
                        $self->pidl("} else {");
                        $self->indent;
@@ -1060,7 +1193,16 @@ sub ConvertObjectToPythonLevel($$$$$$)
 
                if (is_charset_array($e, $l)) {
                        # FIXME: Use Unix charset setting rather than utf-8
+                       $self->pidl("if ($var_name == NULL) {");
+                       $self->indent;
+                       $self->pidl("$py_var = Py_None;");
+                       $self->pidl("Py_INCREF($py_var);");
+                       $self->deindent;
+                       $self->pidl("} else {");
+                       $self->indent;
                        $self->pidl("$py_var = PyUnicode_Decode($var_name, strlen($var_name), \"utf-8\", \"ignore\");");
+                       $self->deindent;
+                       $self->pidl("}");
                } else {
                        die("No SIZE_IS for array $var_name") unless (defined($l->{SIZE_IS}));
                        my $length = $l->{SIZE_IS};
@@ -1096,12 +1238,12 @@ sub ConvertObjectToPythonLevel($$$$$$)
                if (not Parse::Pidl::Typelist::is_scalar($l->{DATA_TYPE})) {
                        $var_name = get_pointer_to($var_name);
                }
-               my $conv = $self->ConvertObjectToPythonData($mem_ctx, $l->{DATA_TYPE}, $var_name);
+               my $conv = $self->ConvertObjectToPythonData($mem_ctx, $l->{DATA_TYPE}, $var_name, $e->{ORIGINAL});
                $self->pidl("$py_var = $conv;");
        } elsif ($l->{TYPE} eq "SUBCONTEXT") {
                $self->ConvertObjectToPythonLevel($mem_ctx, $env, $e, GetNextLevel($e, $l), $var_name, $py_var, $fail);
        } else {
-               die("Unknown level type $l->{TYPE} $var_name");
+               fatal($e->{ORIGINAL}, "Unknown level type $l->{TYPE} $var_name");
        }
 }
 
@@ -1120,11 +1262,11 @@ sub Parse($$$$$)
 
     $self->pidl_hdr("
 /* Python wrapper functions auto-generated by pidl */
+#include <Python.h>
 #include \"includes.h\"
-#include \"../lib/util/python_util.h\"
-#include \"librpc/rpc/dcerpc.h\"
 #include \"lib/talloc/pytalloc.h\"
 #include \"librpc/rpc/pyrpc.h\"
+#include \"librpc/rpc/pyrpc_util.h\"
 #include \"$hdr\"
 #include \"$ndr_hdr\"
 
@@ -1152,14 +1294,16 @@ sub Parse($$$$$)
        $self->pidl("{");
        $self->indent;
        $self->pidl("PyObject *m;");
-       foreach (@{$self->{module_imports}}) {
-               $self->pidl("PyObject *dep_$_;");
+       foreach (keys %{$self->{module_imports}}) {
+               $self->pidl("PyObject *$_;");
        }
        $self->pidl("");
 
-       foreach (@{$self->{module_imports}}) {
-               $self->pidl("dep_$_ = PyImport_ImportModule(\"samba.dcerpc.$_\");");
-               $self->pidl("if (dep_$_ == NULL)");
+       foreach (keys %{$self->{module_imports}}) {
+               my $var_name = $_;
+               my $module_path = $self->{module_imports}->{$var_name};
+               $self->pidl("$var_name = PyImport_ImportModule(\"$module_path\");");
+               $self->pidl("if ($var_name == NULL)");
                $self->pidl("\treturn;");
                $self->pidl("");
        }
@@ -1174,12 +1318,14 @@ sub Parse($$$$$)
                $self->pidl("");
        }
 
+       $self->pidl($_) foreach (@{$self->{prereadycode}});
+
        foreach (@{$self->{ready_types}}) {
                $self->pidl("if (PyType_Ready($_) < 0)");
                $self->pidl("\treturn;");
        }
 
-       $self->pidl($_) foreach (@{$self->{readycode}});
+       $self->pidl($_) foreach (@{$self->{postreadycode}});
 
        foreach (@{$self->{patch_type_calls}}) {
                my ($typename, $cvar) = @$_;
@@ -1202,7 +1348,7 @@ sub Parse($$$$$)
                } elsif ($cvar =~ /^".*"$/) {
                        $py_obj = "PyString_FromString($cvar)";
                } else {
-                       $py_obj = $self->ConvertObjectToPythonData("NULL", expandAlias($ctype), $cvar);
+                       $py_obj = $self->ConvertObjectToPythonData("NULL", expandAlias($ctype), $cvar, undef);
                }
 
                $self->pidl("PyModule_AddObject(m, \"$name\", $py_obj);");