gsskrb5: try to be compatible with windows for gss_wrap* and cfx

The good thing is that windows and heimdal both use EC=0
in the non DCE_STYLE case, so we need the windows compat hack
only in DCE_STYLE mode.

metze
(This used to be commit 0fa41a94e466d5e11bcf362ccd8ff41b72733d1a)

diff --git a/source4/heimdal/lib/gssapi/krb5/cfx.c b/source4/heimdal/lib/gssapi/krb5/cfx.c
index 6452f802ab84be5bc7a9431d15cc135a3e9bca2f..bc0d736e81ac8230bb7e97da71722a2fc0c80c2e 100755 (executable)
--- a/source4/heimdal/lib/gssapi/krb5/cfx.c
+++ b/source4/heimdal/lib/gssapi/krb5/cfx.c
@@ -43,7 +43,8 @@ RCSID("$Id: cfx.c 19031 2006-11-13 18:02:57Z lha $");
 #define CFXAcceptorSubkey      (1 << 2)
 
 krb5_error_code
-_gsskrb5cfx_wrap_length_cfx(krb5_context context,
+_gsskrb5cfx_wrap_length_cfx(const gsskrb5_ctx context_handle,
+                           krb5_context context,
                            krb5_crypto crypto,
                            int conf_req_flag,
                            size_t input_length,
@@ -72,7 +73,11 @@ _gsskrb5cfx_wrap_length_cfx(krb5_context context,
        /* Header is concatenated with data before encryption */
        input_length += sizeof(gss_cfx_wrap_token_desc);
 
-       ret = krb5_crypto_getpadsize(context, crypto, &padsize);
+       if (IS_DCE_STYLE(context_handle)) {
+               ret = krb5_crypto_getblocksize(context, crypto, &padsize);
+       } else {
+               ret = krb5_crypto_getpadsize(context, crypto, &padsize);
+       }
        if (ret) {
            return ret;
        }
@@ -258,7 +263,7 @@ OM_uint32 _gssapi_wrap_cfx(OM_uint32 *minor_status,
        return GSS_S_FAILURE;
     }
 
-    ret = _gsskrb5cfx_wrap_length_cfx(context,
+    ret = _gsskrb5cfx_wrap_length_cfx(context_handle, context,
                                      crypto, conf_req_flag, 
                                      input_message_buffer->length,
                                      &wrapped_len, &cksumsize, &padlength);
@@ -380,7 +385,15 @@ OM_uint32 _gssapi_wrap_cfx(OM_uint32 *minor_status,
        token->RRC[0] = (rrc >> 8) & 0xFF;  
        token->RRC[1] = (rrc >> 0) & 0xFF;
 
-       ret = rrc_rotate(cipher.data, cipher.length, rrc, FALSE);
+       /*
+        * this is really ugly, but needed against windows
+        * for DCERPC, as windows rotates by EC+RRC.
+        */
+       if (IS_DCE_STYLE(context_handle)) {
+               ret = rrc_rotate(cipher.data, cipher.length, rrc+padlength, FALSE);
+       } else {
+               ret = rrc_rotate(cipher.data, cipher.length, rrc, FALSE);
+       }
        if (ret != 0) {
            *minor_status = ret;
            krb5_crypto_destroy(context, crypto);
@@ -553,14 +566,21 @@ OM_uint32 _gssapi_unwrap_cfx(OM_uint32 *minor_status,
     len = input_message_buffer->length;
     len -= (p - (u_char *)input_message_buffer->value);
 
-    /* Rotate by RRC; bogus to do this in-place XXX */
-    *minor_status = rrc_rotate(p, len, rrc, TRUE);
-    if (*minor_status != 0) {
-       krb5_crypto_destroy(context, crypto);
-       return GSS_S_FAILURE;
-    }
-
     if (token_flags & CFXSealed) {
+       /*
+        * this is really ugly, but needed against windows
+        * for DCERPC, as windows rotates by EC+RRC.
+        */
+       if (IS_DCE_STYLE(context_handle)) {
+               *minor_status = rrc_rotate(p, len, rrc+ec, TRUE);
+       } else {
+               *minor_status = rrc_rotate(p, len, rrc, TRUE);
+       }
+       if (*minor_status != 0) {
+           krb5_crypto_destroy(context, crypto);
+           return GSS_S_FAILURE;
+       }
+
        ret = krb5_decrypt(context, crypto, usage,
            p, len, &data);
        if (ret != 0) {
@@ -594,6 +614,13 @@ OM_uint32 _gssapi_unwrap_cfx(OM_uint32 *minor_status,
     } else {
        Checksum cksum;
 
+       /* Rotate by RRC; bogus to do this in-place XXX */
+       *minor_status = rrc_rotate(p, len, rrc, TRUE);
+       if (*minor_status != 0) {
+           krb5_crypto_destroy(context, crypto);
+           return GSS_S_FAILURE;
+       }
+
        /* Determine checksum type */
        ret = krb5_crypto_get_checksum_type(context,
                                            crypto, &cksum.cksumtype);

diff --git a/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h b/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h
index 64a0dd36b19d9cd6d0ed083915ac6a6f4a9a266a..f6edb8b247630b8e2ac3bf861355358ac3f8cb02 100644 (file)
--- a/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h
+++ b/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h
@@ -661,6 +661,7 @@ _gsskrb5cfx_max_wrap_length_cfx (
 
 krb5_error_code
 _gsskrb5cfx_wrap_length_cfx (
+       const gsskrb5_ctx /*context_handle*/,
        krb5_context /*context*/,
        krb5_crypto /*crypto*/,
        int /*conf_req_flag*/,