CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutSign UpSign In
rapid7

Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place.

GitHub Repository: rapid7/metasploit-framework
Path: blob/master/external/source/vncdll/winvnc/d3des.c
Views: 11779
1
/*
2
* This is D3DES (V5.09) by Richard Outerbridge with the double and
3
* triple-length support removed for use in VNC. Also the bytebit[] array
4
* has been reversed so that the most significant bit in each byte of the
5
* key is ignored, not the least significant.
6
*
7
* These changes are
8
* Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
9
*
10
* This software is distributed in the hope that it will be useful,
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13
*/
14
15
/* D3DES (V5.09) -
16
*
17
* A portable, public domain, version of the Data Encryption Standard.
18
*
19
* Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
20
* Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
21
* code; Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
22
* Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
23
* for humouring me on.
24
*
25
* Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
26
* (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
27
*/
28
29
#include "d3des.h"
30
31
static void scrunch(unsigned char *, unsigned long *);
32
static void unscrun(unsigned long *, unsigned char *);
33
static void desfunc(unsigned long *, unsigned long *);
34
static void cookey(unsigned long *);
35
36
static unsigned long KnL[32] = { 0L };
37
static unsigned long KnR[32] = { 0L };
38
static unsigned long Kn3[32] = { 0L };
39
static unsigned char Df_Key[24] = {
40
0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef,
41
0xfe,0xdc,0xba,0x98,0x76,0x54,0x32,0x10,
42
0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 };
43
44
static unsigned short bytebit[8] = {
45
01, 02, 04, 010, 020, 040, 0100, 0200 };
46
47
static unsigned long bigbyte[24] = {
48
0x800000L, 0x400000L, 0x200000L, 0x100000L,
49
0x80000L, 0x40000L, 0x20000L, 0x10000L,
50
0x8000L, 0x4000L, 0x2000L, 0x1000L,
51
0x800L, 0x400L, 0x200L, 0x100L,
52
0x80L, 0x40L, 0x20L, 0x10L,
53
0x8L, 0x4L, 0x2L, 0x1L };
54
55
/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
56
57
static unsigned char pc1[56] = {
58
56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17,
59
9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35,
60
62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21,
61
13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3 };
62
63
static unsigned char totrot[16] = {
64
1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
65
66
static unsigned char pc2[48] = {
67
13, 16, 10, 23, 0, 4, 2, 27, 14, 5, 20, 9,
68
22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1,
69
40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
70
43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
71
72
void deskey(key, edf) /* Thanks to James Gillogly & Phil Karn! */
73
unsigned char *key;
74
int edf;
75
{
76
register int i, j, l, m, n;
77
unsigned char pc1m[56], pcr[56];
78
unsigned long kn[32];
79
80
for ( j = 0; j < 56; j++ ) {
81
l = pc1[j];
82
m = l & 07;
83
pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
84
}
85
for( i = 0; i < 16; i++ ) {
86
if( edf == DE1 ) m = (15 - i) << 1;
87
else m = i << 1;
88
n = m + 1;
89
kn[m] = kn[n] = 0L;
90
for( j = 0; j < 28; j++ ) {
91
l = j + totrot[i];
92
if( l < 28 ) pcr[j] = pc1m[l];
93
else pcr[j] = pc1m[l - 28];
94
}
95
for( j = 28; j < 56; j++ ) {
96
l = j + totrot[i];
97
if( l < 56 ) pcr[j] = pc1m[l];
98
else pcr[j] = pc1m[l - 28];
99
}
100
for( j = 0; j < 24; j++ ) {
101
if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
102
if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
103
}
104
}
105
cookey(kn);
106
return;
107
}
108
109
static void cookey(raw1)
110
register unsigned long *raw1;
111
{
112
register unsigned long *cook, *raw0;
113
unsigned long dough[32];
114
register int i;
115
116
cook = dough;
117
for( i = 0; i < 16; i++, raw1++ ) {
118
raw0 = raw1++;
119
*cook = (*raw0 & 0x00fc0000L) << 6;
120
*cook |= (*raw0 & 0x00000fc0L) << 10;
121
*cook |= (*raw1 & 0x00fc0000L) >> 10;
122
*cook++ |= (*raw1 & 0x00000fc0L) >> 6;
123
*cook = (*raw0 & 0x0003f000L) << 12;
124
*cook |= (*raw0 & 0x0000003fL) << 16;
125
*cook |= (*raw1 & 0x0003f000L) >> 4;
126
*cook++ |= (*raw1 & 0x0000003fL);
127
}
128
usekey(dough);
129
return;
130
}
131
132
void cpkey(into)
133
register unsigned long *into;
134
{
135
register unsigned long *from, *endp;
136
137
from = KnL, endp = &KnL[32];
138
while( from < endp ) *into++ = *from++;
139
return;
140
}
141
142
void usekey(from)
143
register unsigned long *from;
144
{
145
register unsigned long *to, *endp;
146
147
to = KnL, endp = &KnL[32];
148
while( to < endp ) *to++ = *from++;
149
return;
150
}
151
152
void des(inblock, outblock)
153
unsigned char *inblock, *outblock;
154
{
155
unsigned long work[2];
156
157
scrunch(inblock, work);
158
desfunc(work, KnL);
159
unscrun(work, outblock);
160
return;
161
}
162
163
static void scrunch(outof, into)
164
register unsigned char *outof;
165
register unsigned long *into;
166
{
167
*into = (*outof++ & 0xffL) << 24;
168
*into |= (*outof++ & 0xffL) << 16;
169
*into |= (*outof++ & 0xffL) << 8;
170
*into++ |= (*outof++ & 0xffL);
171
*into = (*outof++ & 0xffL) << 24;
172
*into |= (*outof++ & 0xffL) << 16;
173
*into |= (*outof++ & 0xffL) << 8;
174
*into |= (*outof & 0xffL);
175
return;
176
}
177
178
static void unscrun(outof, into)
179
register unsigned long *outof;
180
register unsigned char *into;
181
{
182
*into++ = (unsigned char)((*outof >> 24) & 0xffL);
183
*into++ = (unsigned char)((*outof >> 16) & 0xffL);
184
*into++ = (unsigned char)((*outof >> 8) & 0xffL);
185
*into++ = (unsigned char)( *outof++ & 0xffL);
186
*into++ = (unsigned char)((*outof >> 24) & 0xffL);
187
*into++ = (unsigned char)((*outof >> 16) & 0xffL);
188
*into++ = (unsigned char)((*outof >> 8) & 0xffL);
189
*into = (unsigned char)( *outof & 0xffL);
190
return;
191
}
192
193
static unsigned long SP1[64] = {
194
0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
195
0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
196
0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
197
0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
198
0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
199
0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
200
0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
201
0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
202
0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
203
0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
204
0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
205
0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
206
0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
207
0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
208
0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
209
0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
210
211
static unsigned long SP2[64] = {
212
0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
213
0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
214
0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
215
0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
216
0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
217
0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
218
0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
219
0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
220
0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
221
0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
222
0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
223
0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
224
0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
225
0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
226
0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
227
0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
228
229
static unsigned long SP3[64] = {
230
0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
231
0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
232
0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
233
0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
234
0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
235
0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
236
0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
237
0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
238
0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
239
0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
240
0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
241
0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
242
0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
243
0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
244
0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
245
0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
246
247
static unsigned long SP4[64] = {
248
0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
249
0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
250
0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
251
0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
252
0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
253
0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
254
0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
255
0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
256
0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
257
0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
258
0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
259
0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
260
0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
261
0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
262
0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
263
0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
264
265
static unsigned long SP5[64] = {
266
0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
267
0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
268
0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
269
0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
270
0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
271
0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
272
0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
273
0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
274
0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
275
0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
276
0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
277
0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
278
0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
279
0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
280
0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
281
0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
282
283
static unsigned long SP6[64] = {
284
0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
285
0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
286
0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
287
0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
288
0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
289
0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
290
0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
291
0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
292
0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
293
0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
294
0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
295
0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
296
0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
297
0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
298
0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
299
0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
300
301
static unsigned long SP7[64] = {
302
0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
303
0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
304
0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
305
0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
306
0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
307
0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
308
0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
309
0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
310
0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
311
0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
312
0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
313
0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
314
0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
315
0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
316
0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
317
0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
318
319
static unsigned long SP8[64] = {
320
0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
321
0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
322
0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
323
0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
324
0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
325
0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
326
0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
327
0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
328
0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
329
0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
330
0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
331
0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
332
0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
333
0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
334
0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
335
0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
336
337
static void desfunc(block, keys)
338
register unsigned long *block, *keys;
339
{
340
register unsigned long fval, work, right, leftt;
341
register int round;
342
343
leftt = block[0];
344
right = block[1];
345
work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
346
right ^= work;
347
leftt ^= (work << 4);
348
work = ((leftt >> 16) ^ right) & 0x0000ffffL;
349
right ^= work;
350
leftt ^= (work << 16);
351
work = ((right >> 2) ^ leftt) & 0x33333333L;
352
leftt ^= work;
353
right ^= (work << 2);
354
work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
355
leftt ^= work;
356
right ^= (work << 8);
357
right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
358
work = (leftt ^ right) & 0xaaaaaaaaL;
359
leftt ^= work;
360
right ^= work;
361
leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
362
363
for( round = 0; round < 8; round++ ) {
364
work = (right << 28) | (right >> 4);
365
work ^= *keys++;
366
fval = SP7[ work & 0x3fL];
367
fval |= SP5[(work >> 8) & 0x3fL];
368
fval |= SP3[(work >> 16) & 0x3fL];
369
fval |= SP1[(work >> 24) & 0x3fL];
370
work = right ^ *keys++;
371
fval |= SP8[ work & 0x3fL];
372
fval |= SP6[(work >> 8) & 0x3fL];
373
fval |= SP4[(work >> 16) & 0x3fL];
374
fval |= SP2[(work >> 24) & 0x3fL];
375
leftt ^= fval;
376
work = (leftt << 28) | (leftt >> 4);
377
work ^= *keys++;
378
fval = SP7[ work & 0x3fL];
379
fval |= SP5[(work >> 8) & 0x3fL];
380
fval |= SP3[(work >> 16) & 0x3fL];
381
fval |= SP1[(work >> 24) & 0x3fL];
382
work = leftt ^ *keys++;
383
fval |= SP8[ work & 0x3fL];
384
fval |= SP6[(work >> 8) & 0x3fL];
385
fval |= SP4[(work >> 16) & 0x3fL];
386
fval |= SP2[(work >> 24) & 0x3fL];
387
right ^= fval;
388
}
389
390
right = (right << 31) | (right >> 1);
391
work = (leftt ^ right) & 0xaaaaaaaaL;
392
leftt ^= work;
393
right ^= work;
394
leftt = (leftt << 31) | (leftt >> 1);
395
work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
396
right ^= work;
397
leftt ^= (work << 8);
398
work = ((leftt >> 2) ^ right) & 0x33333333L;
399
right ^= work;
400
leftt ^= (work << 2);
401
work = ((right >> 16) ^ leftt) & 0x0000ffffL;
402
leftt ^= work;
403
right ^= (work << 16);
404
work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
405
leftt ^= work;
406
right ^= (work << 4);
407
*block++ = right;
408
*block = leftt;
409
return;
410
}
411
412
/* Validation sets:
413
*
414
* Single-length key, single-length plaintext -
415
* Key : 0123 4567 89ab cdef
416
* Plain : 0123 4567 89ab cde7
417
* Cipher : c957 4425 6a5e d31d
418
*
419
* Double-length key, single-length plaintext -
420
* Key : 0123 4567 89ab cdef fedc ba98 7654 3210
421
* Plain : 0123 4567 89ab cde7
422
* Cipher : 7f1d 0a77 826b 8aff
423
*
424
* Double-length key, double-length plaintext -
425
* Key : 0123 4567 89ab cdef fedc ba98 7654 3210
426
* Plain : 0123 4567 89ab cdef 0123 4567 89ab cdff
427
* Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
428
*
429
* Triple-length key, single-length plaintext -
430
* Key : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
431
* Plain : 0123 4567 89ab cde7
432
* Cipher : de0b 7c06 ae5e 0ed5
433
*
434
* Triple-length key, double-length plaintext -
435
* Key : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
436
* Plain : 0123 4567 89ab cdef 0123 4567 89ab cdff
437
* Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
438
*
439
* d3des V5.0a rwo 9208.07 18:44 Graven Imagery
440
**********************************************************************/
441
442