libcli/auth: rewrite schannel sign/seal code to be more generic
[gd/samba/.git] / libcli / auth / schannel_sign.c
1 /*
2    Unix SMB/CIFS implementation.
3
4    schannel library code
5
6    Copyright (C) Andrew Tridgell 2004
7    Copyright (C) Andrew Bartlett <abartlet@samba.org> 2005
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 3 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program.  If not, see <http://www.gnu.org/licenses/>.
21 */
22
23 #include "includes.h"
24 #include "../libcli/auth/schannel.h"
25 #include "../lib/crypto/crypto.h"
26
27 static void netsec_offset_and_sizes(struct schannel_state *state,
28                                     bool do_seal,
29                                     uint32_t *_min_sig_size,
30                                     uint32_t *_used_sig_size,
31                                     uint32_t *_checksum_length,
32                                     uint32_t *_confounder_ofs)
33 {
34         uint32_t min_sig_size = 24;
35         uint32_t used_sig_size = 32;
36         uint32_t checksum_length = 8;
37         uint32_t confounder_ofs = 24;
38
39         if (do_seal) {
40                 min_sig_size += 8;
41         }
42
43         if (_min_sig_size) {
44                 *_min_sig_size = min_sig_size;
45         }
46
47         if (_used_sig_size) {
48                 *_used_sig_size = used_sig_size;
49         }
50
51         if (_checksum_length) {
52                 *_checksum_length = checksum_length;
53         }
54
55         if (_confounder_ofs) {
56                 *_confounder_ofs = confounder_ofs;
57         }
58 }
59
60 /*******************************************************************
61  Encode or Decode the sequence number (which is symmetric)
62  ********************************************************************/
63 static void netsec_do_seq_num(struct schannel_state *state,
64                               const uint8_t *checksum,
65                               uint32_t checksum_length,
66                               uint8_t seq_num[8])
67 {
68         static const uint8_t zeros[4];
69         uint8_t sequence_key[16];
70         uint8_t digest1[16];
71
72         hmac_md5(state->creds->session_key, zeros, sizeof(zeros), digest1);
73         hmac_md5(digest1, checksum, checksum_length, sequence_key);
74         arcfour_crypt(seq_num, sequence_key, 8);
75
76         state->seq_num++;
77 }
78
79 static void netsec_do_seal(struct schannel_state *state,
80                            const uint8_t seq_num[8],
81                            uint8_t confounder[8],
82                            uint8_t *data, uint32_t length)
83 {
84         uint8_t sealing_key[16];
85         static const uint8_t zeros[4];
86         uint8_t digest2[16];
87         uint8_t sess_kf0[16];
88         int i;
89
90         for (i = 0; i < 16; i++) {
91                 sess_kf0[i] = state->creds->session_key[i] ^ 0xf0;
92         }
93
94         hmac_md5(sess_kf0, zeros, 4, digest2);
95         hmac_md5(digest2, seq_num, 8, sealing_key);
96
97         arcfour_crypt(confounder, sealing_key, 8);
98         arcfour_crypt(data, sealing_key, length);
99 }
100
101 /*******************************************************************
102  Create a digest over the entire packet (including the data), and
103  MD5 it with the session key.
104  ********************************************************************/
105 static void netsec_do_sign(struct schannel_state *state,
106                            const uint8_t *confounder,
107                            const uint8_t *data, size_t data_len,
108                            uint8_t header[8],
109                            uint8_t *checksum)
110 {
111         uint8_t packet_digest[16];
112         static const uint8_t zeros[4];
113         struct MD5Context ctx;
114
115         MD5Init(&ctx);
116         MD5Update(&ctx, zeros, 4);
117         if (confounder) {
118                 SSVAL(header, 0, NL_SIGN_HMAC_MD5);
119                 SSVAL(header, 2, NL_SEAL_RC4);
120                 SSVAL(header, 4, 0xFFFF);
121                 SSVAL(header, 6, 0x0000);
122
123                 MD5Update(&ctx, header, 8);
124                 MD5Update(&ctx, confounder, 8);
125         } else {
126                 SSVAL(header, 0, NL_SIGN_HMAC_MD5);
127                 SSVAL(header, 2, NL_SEAL_NONE);
128                 SSVAL(header, 4, 0xFFFF);
129                 SSVAL(header, 6, 0x0000);
130
131                 MD5Update(&ctx, header, 8);
132         }
133         MD5Update(&ctx, data, data_len);
134         MD5Final(packet_digest, &ctx);
135
136         hmac_md5(state->creds->session_key,
137                  packet_digest, sizeof(packet_digest),
138                  checksum);
139 }
140
141 NTSTATUS netsec_incoming_packet(struct schannel_state *state,
142                                 TALLOC_CTX *mem_ctx,
143                                 bool do_unseal,
144                                 uint8_t *data, size_t length,
145                                 const DATA_BLOB *sig)
146 {
147         uint32_t min_sig_size = 0;
148         uint8_t header[8];
149         uint8_t checksum[32];
150         uint32_t checksum_length = sizeof(checksum_length);
151         uint8_t _confounder[8];
152         uint8_t *confounder = NULL;
153         uint32_t confounder_ofs = 0;
154         uint8_t seq_num[8];
155         int ret;
156
157         netsec_offset_and_sizes(state,
158                                 do_unseal,
159                                 &min_sig_size,
160                                 NULL,
161                                 &checksum_length,
162                                 &confounder_ofs);
163
164         if (sig->length < min_sig_size) {
165                 return NT_STATUS_ACCESS_DENIED;
166         }
167
168         if (do_unseal) {
169                 confounder = _confounder;
170                 memcpy(confounder, sig->data+confounder_ofs, 8);
171         } else {
172                 confounder = NULL;
173         }
174
175         RSIVAL(seq_num, 0, state->seq_num);
176         SIVAL(seq_num, 4, state->initiator?0:0x80);
177
178         if (do_unseal) {
179                 netsec_do_seal(state, seq_num,
180                                confounder,
181                                data, length);
182         }
183
184         netsec_do_sign(state, confounder,
185                        data, length,
186                        header, checksum);
187
188         ret = memcmp(checksum, sig->data+16, checksum_length);
189         if (ret != 0) {
190                 dump_data_pw("calc digest:", checksum, checksum_length);
191                 dump_data_pw("wire digest:", sig->data+16, checksum_length);
192                 return NT_STATUS_ACCESS_DENIED;
193         }
194
195         netsec_do_seq_num(state, checksum, checksum_length, seq_num);
196
197         ret = memcmp(seq_num, sig->data+8, 8);
198         if (ret != 0) {
199                 dump_data_pw("calc seq num:", seq_num, 8);
200                 dump_data_pw("wire seq num:", sig->data+8, 8);
201                 return NT_STATUS_ACCESS_DENIED;
202         }
203
204         return NT_STATUS_OK;
205 }
206
207 NTSTATUS netsec_outgoing_packet(struct schannel_state *state,
208                                 TALLOC_CTX *mem_ctx,
209                                 bool do_seal,
210                                 uint8_t *data, size_t length,
211                                 DATA_BLOB *sig)
212 {
213         uint32_t min_sig_size = 0;
214         uint32_t used_sig_size = 0;
215         uint8_t header[8];
216         uint8_t checksum[32];
217         uint32_t checksum_length = sizeof(checksum_length);
218         uint8_t _confounder[8];
219         uint8_t *confounder = NULL;
220         uint32_t confounder_ofs = 0;
221         uint8_t seq_num[8];
222
223         netsec_offset_and_sizes(state,
224                                 do_seal,
225                                 &min_sig_size,
226                                 &used_sig_size,
227                                 &checksum_length,
228                                 &confounder_ofs);
229
230         RSIVAL(seq_num, 0, state->seq_num);
231         SIVAL(seq_num, 4, state->initiator?0x80:0);
232
233         if (do_seal) {
234                 confounder = _confounder;
235                 generate_random_buffer(confounder, 8);
236         } else {
237                 confounder = NULL;
238         }
239
240         netsec_do_sign(state, confounder,
241                        data, length,
242                        header, checksum);
243
244         if (do_seal) {
245                 netsec_do_seal(state, seq_num,
246                                confounder,
247                                data, length);
248         }
249
250         netsec_do_seq_num(state, checksum, checksum_length, seq_num);
251
252         (*sig) = data_blob_talloc_zero(mem_ctx, used_sig_size);
253
254         memcpy(sig->data, header, 8);
255         memcpy(sig->data+8, seq_num, 8);
256         memcpy(sig->data+16, checksum, checksum_length);
257
258         if (confounder) {
259                 memcpy(sig->data+confounder_ofs, confounder, 8);
260         }
261
262         dump_data_pw("signature:", sig->data+ 0, 8);
263         dump_data_pw("seq_num  :", sig->data+ 8, 8);
264         dump_data_pw("digest   :", sig->data+16, checksum_length);
265         dump_data_pw("confound :", sig->data+confounder_ofs, 8);
266
267         return NT_STATUS_OK;
268 }