Merge Herb's idmap endian fix.

Jeremy.
(This used to be commit 7ddad4061a1b7ed25e4d6471c7a1f8f97a98ed37)

diff --git a/source3/nsswitch/winbindd_idmap.c b/source3/nsswitch/winbindd_idmap.c
index 60ea188b1d33060309f8761c36a09732ba47201e..0594f61680173a8dd0f6cacc01b7716ebd61ae16 100644 (file)
--- a/source3/nsswitch/winbindd_idmap.c
+++ b/source3/nsswitch/winbindd_idmap.c
@@ -364,6 +364,7 @@ fail:
 static BOOL idmap_convert(const char *idmap_name)
 {
        int32 vers = tdb_fetch_int32(idmap_tdb, "IDMAP_VERSION");
+       BOOL bigendianheader = (idmap_tdb->flags & TDB_BIGENDIAN) ? True : False;
 
        if (vers == IDMAP_VERSION)
                return True;
@@ -374,23 +375,20 @@ static BOOL idmap_convert(const char *idmap_name)
                return False;
 #endif
 
-       if ((vers == -1) || (IREV(vers) == IDMAP_VERSION)) {
-               /* Arrggghh ! Bytereversed or missing - make order independent ! */
+       if (((vers == -1) && bigendianheader) || (IREV(vers) == IDMAP_VERSION)) {
+               /* Arrggghh ! Bytereversed or old big-endian - make order independent ! */
                /*
-                * If the header needed to be converted then the
-                * high and low records may have been created on a
-                * foreign endian machine and will need byte-reversing.
+                * high and low records were created on a
+                * big endian machine and will need byte-reversing.
                 */
 
-               BOOL bytereverse_needed = (idmap_tdb->flags & TDB_CONVERT);
                int32 wm;
 
                wm = tdb_fetch_int32(idmap_tdb, HWM_USER);
 
-               if (wm != -1 && bytereverse_needed) {
-                       /* A record existed and it was from a foreign endian machine. */
+               if (wm != -1) {
                        wm = IREV(wm);
-               } else if (wm == -1)
+               }  else
                        wm = server_state.uid_low;
 
                if (tdb_store_int32(idmap_tdb, HWM_USER, wm) == -1) {
@@ -399,10 +397,9 @@ static BOOL idmap_convert(const char *idmap_name)
                }
 
                wm = tdb_fetch_int32(idmap_tdb, HWM_GROUP);
-               if (wm != -1 && bytereverse_needed) {
-                       /* A record existed and it was from a foreign endian machine. */
+               if (wm != -1) {
                        wm = IREV(wm);
-               } else if (wm == -1)
+               } else
                        wm = server_state.gid_low;
 
                if (tdb_store_int32(idmap_tdb, HWM_GROUP, wm) == -1) {

diff --git a/source3/tdb/tdb.c b/source3/tdb/tdb.c
index 5c0fd436e3a1edf7fc3c53fa80b2ae44693d3462..c937090de4dca333799e533d5e3609800563e8b8 100644 (file)
--- a/source3/tdb/tdb.c
+++ b/source3/tdb/tdb.c
@@ -1471,6 +1471,8 @@ TDB_CONTEXT *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
        TDB_CONTEXT *tdb;
        struct stat st;
        int rev = 0, locked;
+       unsigned char *vp;
+       u32 vertest;
 
        if (!(tdb = calloc(1, sizeof *tdb))) {
                /* Can't log this */
@@ -1548,6 +1550,10 @@ TDB_CONTEXT *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
                }
                rev = (tdb->flags & TDB_CONVERT);
        }
+       vp = (unsigned char *)&tdb->header.version;
+       vertest = (((u32)vp[0]) << 24) | (((u32)vp[1]) << 16) |
+                 (((u32)vp[2]) << 8) | (u32)vp[3];
+       tdb->flags |= (vertest==TDB_VERSION) ? TDB_BIGENDIAN : 0;
        if (!rev)
                tdb->flags &= ~TDB_CONVERT;
        else {

diff --git a/source3/tdb/tdb.h b/source3/tdb/tdb.h
index 54cde10d9506a9fd2ab9567eb91ebe80898fe78c..8cc908703f8209f4da251f8f8086c4c920b388d5 100644 (file)
--- a/source3/tdb/tdb.h
+++ b/source3/tdb/tdb.h
@@ -38,6 +38,7 @@ extern "C" {
 #define TDB_NOLOCK   4 /* don't do any locking */
 #define TDB_NOMMAP   8 /* don't use mmap */
 #define TDB_CONVERT 16 /* convert endian (internal use) */
+#define TDB_BIGENDIAN 32 /* header is big-endian (internal use) */
 
 #define TDB_ERRCODE(code, ret) ((tdb->ecode = (code)), ret)