CoCalc -- chcr

GitHub Repository: torvalds/linux
Path: blob/master/drivers/crypto/chelsio/chcr_algo.c
²⁹⁵²⁴ views
1
/*
2
 * This file is part of the Chelsio T6 Crypto driver for Linux.
3
 *
4
 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5
 *
6
 * This software is available to you under a choice of one of two
7
 * licenses.  You may choose to be licensed under the terms of the GNU
8
 * General Public License (GPL) Version 2, available from the file
9
 * COPYING in the main directory of this source tree, or the
10
 * OpenIB.org BSD license below:
11
 *
12
 *     Redistribution and use in source and binary forms, with or
13
 *     without modification, are permitted provided that the following
14
 *     conditions are met:
15
 *
16
 *      - Redistributions of source code must retain the above
17
 *        copyright notice, this list of conditions and the following
18
 *        disclaimer.
19
 *
20
 *      - Redistributions in binary form must reproduce the above
21
 *        copyright notice, this list of conditions and the following
22
 *        disclaimer in the documentation and/or other materials
23
 *        provided with the distribution.
24
 *
25
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32
 * SOFTWARE.
33
 *
34
 * Written and Maintained by:
35
 *	Manoj Malviya ([email protected])
36
 *	Atul Gupta ([email protected])
37
 *	Jitendra Lulla ([email protected])
38
 *	Yeshaswi M R Gowda ([email protected])
39
 *	Harsh Jain ([email protected])
40
 */
41

42
#define pr_fmt(fmt) "chcr:" fmt
43

44
#include <linux/kernel.h>
45
#include <linux/module.h>
46
#include <linux/crypto.h>
47
#include <linux/skbuff.h>
48
#include <linux/rtnetlink.h>
49
#include <linux/highmem.h>
50
#include <linux/scatterlist.h>
51

52
#include <crypto/aes.h>
53
#include <crypto/algapi.h>
54
#include <crypto/gcm.h>
55
#include <crypto/sha1.h>
56
#include <crypto/sha2.h>
57
#include <crypto/authenc.h>
58
#include <crypto/ctr.h>
59
#include <crypto/gf128mul.h>
60
#include <crypto/internal/aead.h>
61
#include <crypto/null.h>
62
#include <crypto/internal/skcipher.h>
63
#include <crypto/aead.h>
64
#include <crypto/scatterwalk.h>
65
#include <crypto/internal/hash.h>
66

67
#include "t4fw_api.h"
68
#include "t4_msg.h"
69
#include "chcr_core.h"
70
#include "chcr_algo.h"
71
#include "chcr_crypto.h"
72

73
#define IV AES_BLOCK_SIZE
74

75
static unsigned int sgl_ent_len[] = {
76
	0, 0, 16, 24, 40, 48, 64, 72, 88,
77
	96, 112, 120, 136, 144, 160, 168, 184,
78
	192, 208, 216, 232, 240, 256, 264, 280,
79
	288, 304, 312, 328, 336, 352, 360, 376
80
};
81

82
static unsigned int dsgl_ent_len[] = {
83
	0, 32, 32, 48, 48, 64, 64, 80, 80,
84
	112, 112, 128, 128, 144, 144, 160, 160,
85
	192, 192, 208, 208, 224, 224, 240, 240,
86
	272, 272, 288, 288, 304, 304, 320, 320
87
};
88

89
static u32 round_constant[11] = {
90
	0x01000000, 0x02000000, 0x04000000, 0x08000000,
91
	0x10000000, 0x20000000, 0x40000000, 0x80000000,
92
	0x1B000000, 0x36000000, 0x6C000000
93
};
94

95
static int chcr_handle_cipher_resp(struct skcipher_request *req,
96
				   unsigned char *input, int err);
97

98
static inline  struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
99
{
100
	return &ctx->crypto_ctx->aeadctx;
101
}
102

103
static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
104
{
105
	return &ctx->crypto_ctx->ablkctx;
106
}
107

108
static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
109
{
110
	return &ctx->crypto_ctx->hmacctx;
111
}
112

113
static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
114
{
115
	return gctx->ctx->gcm;
116
}
117

118
static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
119
{
120
	return gctx->ctx->authenc;
121
}
122

123
static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
124
{
125
	return container_of(ctx->dev, struct uld_ctx, dev);
126
}
127

128
static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
129
{
130
	memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
131
}
132

133
static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
134
			 unsigned int entlen,
135
			 unsigned int skip)
136
{
137
	int nents = 0;
138
	unsigned int less;
139
	unsigned int skip_len = 0;
140

141
	while (sg && skip) {
142
		if (sg_dma_len(sg) <= skip) {
143
			skip -= sg_dma_len(sg);
144
			skip_len = 0;
145
			sg = sg_next(sg);
146
		} else {
147
			skip_len = skip;
148
			skip = 0;
149
		}
150
	}
151

152
	while (sg && reqlen) {
153
		less = min(reqlen, sg_dma_len(sg) - skip_len);
154
		nents += DIV_ROUND_UP(less, entlen);
155
		reqlen -= less;
156
		skip_len = 0;
157
		sg = sg_next(sg);
158
	}
159
	return nents;
160
}
161

162
static inline int get_aead_subtype(struct crypto_aead *aead)
163
{
164
	struct aead_alg *alg = crypto_aead_alg(aead);
165
	struct chcr_alg_template *chcr_crypto_alg =
166
		container_of(alg, struct chcr_alg_template, alg.aead);
167
	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
168
}
169

170
void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
171
{
172
	u8 temp[SHA512_DIGEST_SIZE];
173
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
174
	int authsize = crypto_aead_authsize(tfm);
175
	struct cpl_fw6_pld *fw6_pld;
176
	int cmp = 0;
177

178
	fw6_pld = (struct cpl_fw6_pld *)input;
179
	if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
180
	    (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
181
		cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
182
	} else {
183

184
		sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
185
				authsize, req->assoclen +
186
				req->cryptlen - authsize);
187
		cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
188
	}
189
	if (cmp)
190
		*err = -EBADMSG;
191
	else
192
		*err = 0;
193
}
194

195
static int chcr_inc_wrcount(struct chcr_dev *dev)
196
{
197
	if (dev->state == CHCR_DETACH)
198
		return 1;
199
	atomic_inc(&dev->inflight);
200
	return 0;
201
}
202

203
static inline void chcr_dec_wrcount(struct chcr_dev *dev)
204
{
205
	atomic_dec(&dev->inflight);
206
}
207

208
static inline int chcr_handle_aead_resp(struct aead_request *req,
209
					 unsigned char *input,
210
					 int err)
211
{
212
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
213
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
214
	struct chcr_dev *dev = a_ctx(tfm)->dev;
215

216
	chcr_aead_common_exit(req);
217
	if (reqctx->verify == VERIFY_SW) {
218
		chcr_verify_tag(req, input, &err);
219
		reqctx->verify = VERIFY_HW;
220
	}
221
	chcr_dec_wrcount(dev);
222
	aead_request_complete(req, err);
223

224
	return err;
225
}
226

227
static void get_aes_decrypt_key(unsigned char *dec_key,
228
				       const unsigned char *key,
229
				       unsigned int keylength)
230
{
231
	u32 temp;
232
	u32 w_ring[MAX_NK];
233
	int i, j, k;
234
	u8  nr, nk;
235

236
	switch (keylength) {
237
	case AES_KEYLENGTH_128BIT:
238
		nk = KEYLENGTH_4BYTES;
239
		nr = NUMBER_OF_ROUNDS_10;
240
		break;
241
	case AES_KEYLENGTH_192BIT:
242
		nk = KEYLENGTH_6BYTES;
243
		nr = NUMBER_OF_ROUNDS_12;
244
		break;
245
	case AES_KEYLENGTH_256BIT:
246
		nk = KEYLENGTH_8BYTES;
247
		nr = NUMBER_OF_ROUNDS_14;
248
		break;
249
	default:
250
		return;
251
	}
252
	for (i = 0; i < nk; i++)
253
		w_ring[i] = get_unaligned_be32(&key[i * 4]);
254

255
	i = 0;
256
	temp = w_ring[nk - 1];
257
	while (i + nk < (nr + 1) * 4) {
258
		if (!(i % nk)) {
259
			/* RotWord(temp) */
260
			temp = (temp << 8) | (temp >> 24);
261
			temp = aes_ks_subword(temp);
262
			temp ^= round_constant[i / nk];
263
		} else if (nk == 8 && (i % 4 == 0)) {
264
			temp = aes_ks_subword(temp);
265
		}
266
		w_ring[i % nk] ^= temp;
267
		temp = w_ring[i % nk];
268
		i++;
269
	}
270
	i--;
271
	for (k = 0, j = i % nk; k < nk; k++) {
272
		put_unaligned_be32(w_ring[j], &dec_key[k * 4]);
273
		j--;
274
		if (j < 0)
275
			j += nk;
276
	}
277
}
278

279
static int chcr_prepare_hmac_key(const u8 *raw_key, unsigned int raw_key_len,
280
				 int digestsize, void *istate, void *ostate)
281
{
282
	__be32 *istate32 = istate, *ostate32 = ostate;
283
	__be64 *istate64 = istate, *ostate64 = ostate;
284
	union {
285
		struct hmac_sha1_key sha1;
286
		struct hmac_sha224_key sha224;
287
		struct hmac_sha256_key sha256;
288
		struct hmac_sha384_key sha384;
289
		struct hmac_sha512_key sha512;
290
	} k;
291

292
	switch (digestsize) {
293
	case SHA1_DIGEST_SIZE:
294
		hmac_sha1_preparekey(&k.sha1, raw_key, raw_key_len);
295
		for (int i = 0; i < ARRAY_SIZE(k.sha1.istate.h); i++) {
296
			istate32[i] = cpu_to_be32(k.sha1.istate.h[i]);
297
			ostate32[i] = cpu_to_be32(k.sha1.ostate.h[i]);
298
		}
299
		break;
300
	case SHA224_DIGEST_SIZE:
301
		hmac_sha224_preparekey(&k.sha224, raw_key, raw_key_len);
302
		for (int i = 0; i < ARRAY_SIZE(k.sha224.key.istate.h); i++) {
303
			istate32[i] = cpu_to_be32(k.sha224.key.istate.h[i]);
304
			ostate32[i] = cpu_to_be32(k.sha224.key.ostate.h[i]);
305
		}
306
		break;
307
	case SHA256_DIGEST_SIZE:
308
		hmac_sha256_preparekey(&k.sha256, raw_key, raw_key_len);
309
		for (int i = 0; i < ARRAY_SIZE(k.sha256.key.istate.h); i++) {
310
			istate32[i] = cpu_to_be32(k.sha256.key.istate.h[i]);
311
			ostate32[i] = cpu_to_be32(k.sha256.key.ostate.h[i]);
312
		}
313
		break;
314
	case SHA384_DIGEST_SIZE:
315
		hmac_sha384_preparekey(&k.sha384, raw_key, raw_key_len);
316
		for (int i = 0; i < ARRAY_SIZE(k.sha384.key.istate.h); i++) {
317
			istate64[i] = cpu_to_be64(k.sha384.key.istate.h[i]);
318
			ostate64[i] = cpu_to_be64(k.sha384.key.ostate.h[i]);
319
		}
320
		break;
321
	case SHA512_DIGEST_SIZE:
322
		hmac_sha512_preparekey(&k.sha512, raw_key, raw_key_len);
323
		for (int i = 0; i < ARRAY_SIZE(k.sha512.key.istate.h); i++) {
324
			istate64[i] = cpu_to_be64(k.sha512.key.istate.h[i]);
325
			ostate64[i] = cpu_to_be64(k.sha512.key.ostate.h[i]);
326
		}
327
		break;
328
	default:
329
		return -EINVAL;
330
	}
331
	memzero_explicit(&k, sizeof(k));
332
	return 0;
333
}
334

335
static inline int is_hmac(struct crypto_tfm *tfm)
336
{
337
	struct crypto_alg *alg = tfm->__crt_alg;
338
	struct chcr_alg_template *chcr_crypto_alg =
339
		container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
340
			     alg.hash);
341
	if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
342
		return 1;
343
	return 0;
344
}
345

346
static inline void dsgl_walk_init(struct dsgl_walk *walk,
347
				   struct cpl_rx_phys_dsgl *dsgl)
348
{
349
	walk->dsgl = dsgl;
350
	walk->nents = 0;
351
	walk->to = (struct phys_sge_pairs *)(dsgl + 1);
352
}
353

354
static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid,
355
				 int pci_chan_id)
356
{
357
	struct cpl_rx_phys_dsgl *phys_cpl;
358

359
	phys_cpl = walk->dsgl;
360

361
	phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
362
				    | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
363
	phys_cpl->pcirlxorder_to_noofsgentr =
364
		htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
365
		      CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
366
		      CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
367
		      CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
368
		      CPL_RX_PHYS_DSGL_DCAID_V(0) |
369
		      CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
370
	phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
371
	phys_cpl->rss_hdr_int.qid = htons(qid);
372
	phys_cpl->rss_hdr_int.hash_val = 0;
373
	phys_cpl->rss_hdr_int.channel = pci_chan_id;
374
}
375

376
static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
377
					size_t size,
378
					dma_addr_t addr)
379
{
380
	int j;
381

382
	if (!size)
383
		return;
384
	j = walk->nents;
385
	walk->to->len[j % 8] = htons(size);
386
	walk->to->addr[j % 8] = cpu_to_be64(addr);
387
	j++;
388
	if ((j % 8) == 0)
389
		walk->to++;
390
	walk->nents = j;
391
}
392

393
static void  dsgl_walk_add_sg(struct dsgl_walk *walk,
394
			   struct scatterlist *sg,
395
			      unsigned int slen,
396
			      unsigned int skip)
397
{
398
	int skip_len = 0;
399
	unsigned int left_size = slen, len = 0;
400
	unsigned int j = walk->nents;
401
	int offset, ent_len;
402

403
	if (!slen)
404
		return;
405
	while (sg && skip) {
406
		if (sg_dma_len(sg) <= skip) {
407
			skip -= sg_dma_len(sg);
408
			skip_len = 0;
409
			sg = sg_next(sg);
410
		} else {
411
			skip_len = skip;
412
			skip = 0;
413
		}
414
	}
415

416
	while (left_size && sg) {
417
		len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
418
		offset = 0;
419
		while (len) {
420
			ent_len =  min_t(u32, len, CHCR_DST_SG_SIZE);
421
			walk->to->len[j % 8] = htons(ent_len);
422
			walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
423
						      offset + skip_len);
424
			offset += ent_len;
425
			len -= ent_len;
426
			j++;
427
			if ((j % 8) == 0)
428
				walk->to++;
429
		}
430
		walk->last_sg = sg;
431
		walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
432
					  skip_len) + skip_len;
433
		left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
434
		skip_len = 0;
435
		sg = sg_next(sg);
436
	}
437
	walk->nents = j;
438
}
439

440
static inline void ulptx_walk_init(struct ulptx_walk *walk,
441
				   struct ulptx_sgl *ulp)
442
{
443
	walk->sgl = ulp;
444
	walk->nents = 0;
445
	walk->pair_idx = 0;
446
	walk->pair = ulp->sge;
447
	walk->last_sg = NULL;
448
	walk->last_sg_len = 0;
449
}
450

451
static inline void ulptx_walk_end(struct ulptx_walk *walk)
452
{
453
	walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
454
			      ULPTX_NSGE_V(walk->nents));
455
}
456

457

458
static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
459
					size_t size,
460
					dma_addr_t addr)
461
{
462
	if (!size)
463
		return;
464

465
	if (walk->nents == 0) {
466
		walk->sgl->len0 = cpu_to_be32(size);
467
		walk->sgl->addr0 = cpu_to_be64(addr);
468
	} else {
469
		walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr);
470
		walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
471
		walk->pair_idx = !walk->pair_idx;
472
		if (!walk->pair_idx)
473
			walk->pair++;
474
	}
475
	walk->nents++;
476
}
477

478
static void  ulptx_walk_add_sg(struct ulptx_walk *walk,
479
					struct scatterlist *sg,
480
			       unsigned int len,
481
			       unsigned int skip)
482
{
483
	int small;
484
	int skip_len = 0;
485
	unsigned int sgmin;
486

487
	if (!len)
488
		return;
489
	while (sg && skip) {
490
		if (sg_dma_len(sg) <= skip) {
491
			skip -= sg_dma_len(sg);
492
			skip_len = 0;
493
			sg = sg_next(sg);
494
		} else {
495
			skip_len = skip;
496
			skip = 0;
497
		}
498
	}
499
	WARN(!sg, "SG should not be null here\n");
500
	if (sg && (walk->nents == 0)) {
501
		small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
502
		sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
503
		walk->sgl->len0 = cpu_to_be32(sgmin);
504
		walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
505
		walk->nents++;
506
		len -= sgmin;
507
		walk->last_sg = sg;
508
		walk->last_sg_len = sgmin + skip_len;
509
		skip_len += sgmin;
510
		if (sg_dma_len(sg) == skip_len) {
511
			sg = sg_next(sg);
512
			skip_len = 0;
513
		}
514
	}
515

516
	while (sg && len) {
517
		small = min(sg_dma_len(sg) - skip_len, len);
518
		sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
519
		walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
520
		walk->pair->addr[walk->pair_idx] =
521
			cpu_to_be64(sg_dma_address(sg) + skip_len);
522
		walk->pair_idx = !walk->pair_idx;
523
		walk->nents++;
524
		if (!walk->pair_idx)
525
			walk->pair++;
526
		len -= sgmin;
527
		skip_len += sgmin;
528
		walk->last_sg = sg;
529
		walk->last_sg_len = skip_len;
530
		if (sg_dma_len(sg) == skip_len) {
531
			sg = sg_next(sg);
532
			skip_len = 0;
533
		}
534
	}
535
}
536

537
static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm)
538
{
539
	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
540
	struct chcr_alg_template *chcr_crypto_alg =
541
		container_of(alg, struct chcr_alg_template, alg.skcipher);
542

543
	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
544
}
545

546
static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
547
{
548
	struct adapter *adap = netdev2adap(dev);
549
	struct sge_uld_txq_info *txq_info =
550
		adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
551
	struct sge_uld_txq *txq;
552
	int ret = 0;
553

554
	local_bh_disable();
555
	txq = &txq_info->uldtxq[idx];
556
	spin_lock(&txq->sendq.lock);
557
	if (txq->full)
558
		ret = -1;
559
	spin_unlock(&txq->sendq.lock);
560
	local_bh_enable();
561
	return ret;
562
}
563

564
static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
565
			       struct _key_ctx *key_ctx)
566
{
567
	if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
568
		memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
569
	} else {
570
		memcpy(key_ctx->key,
571
		       ablkctx->key + (ablkctx->enckey_len >> 1),
572
		       ablkctx->enckey_len >> 1);
573
		memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
574
		       ablkctx->rrkey, ablkctx->enckey_len >> 1);
575
	}
576
	return 0;
577
}
578

579
static int chcr_hash_ent_in_wr(struct scatterlist *src,
580
			     unsigned int minsg,
581
			     unsigned int space,
582
			     unsigned int srcskip)
583
{
584
	int srclen = 0;
585
	int srcsg = minsg;
586
	int soffset = 0, sless;
587

588
	if (sg_dma_len(src) == srcskip) {
589
		src = sg_next(src);
590
		srcskip = 0;
591
	}
592
	while (src && space > (sgl_ent_len[srcsg + 1])) {
593
		sless = min_t(unsigned int, sg_dma_len(src) - soffset -	srcskip,
594
							CHCR_SRC_SG_SIZE);
595
		srclen += sless;
596
		soffset += sless;
597
		srcsg++;
598
		if (sg_dma_len(src) == (soffset + srcskip)) {
599
			src = sg_next(src);
600
			soffset = 0;
601
			srcskip = 0;
602
		}
603
	}
604
	return srclen;
605
}
606

607
static int chcr_sg_ent_in_wr(struct scatterlist *src,
608
			     struct scatterlist *dst,
609
			     unsigned int minsg,
610
			     unsigned int space,
611
			     unsigned int srcskip,
612
			     unsigned int dstskip)
613
{
614
	int srclen = 0, dstlen = 0;
615
	int srcsg = minsg, dstsg = minsg;
616
	int offset = 0, soffset = 0, less, sless = 0;
617

618
	if (sg_dma_len(src) == srcskip) {
619
		src = sg_next(src);
620
		srcskip = 0;
621
	}
622
	if (sg_dma_len(dst) == dstskip) {
623
		dst = sg_next(dst);
624
		dstskip = 0;
625
	}
626

627
	while (src && dst &&
628
	       space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
629
		sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset,
630
				CHCR_SRC_SG_SIZE);
631
		srclen += sless;
632
		srcsg++;
633
		offset = 0;
634
		while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
635
		       space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
636
			if (srclen <= dstlen)
637
				break;
638
			less = min_t(unsigned int, sg_dma_len(dst) - offset -
639
				     dstskip, CHCR_DST_SG_SIZE);
640
			dstlen += less;
641
			offset += less;
642
			if ((offset + dstskip) == sg_dma_len(dst)) {
643
				dst = sg_next(dst);
644
				offset = 0;
645
			}
646
			dstsg++;
647
			dstskip = 0;
648
		}
649
		soffset += sless;
650
		if ((soffset + srcskip) == sg_dma_len(src)) {
651
			src = sg_next(src);
652
			srcskip = 0;
653
			soffset = 0;
654
		}
655

656
	}
657
	return min(srclen, dstlen);
658
}
659

660
static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
661
				struct skcipher_request *req,
662
				u8 *iv,
663
				unsigned short op_type)
664
{
665
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
666
	int err;
667

668
	skcipher_request_set_tfm(&reqctx->fallback_req, cipher);
669
	skcipher_request_set_callback(&reqctx->fallback_req, req->base.flags,
670
				      req->base.complete, req->base.data);
671
	skcipher_request_set_crypt(&reqctx->fallback_req, req->src, req->dst,
672
				   req->cryptlen, iv);
673

674
	err = op_type ? crypto_skcipher_decrypt(&reqctx->fallback_req) :
675
			crypto_skcipher_encrypt(&reqctx->fallback_req);
676

677
	return err;
678

679
}
680

681
static inline int get_qidxs(struct crypto_async_request *req,
682
			    unsigned int *txqidx, unsigned int *rxqidx)
683
{
684
	struct crypto_tfm *tfm = req->tfm;
685
	int ret = 0;
686

687
	switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
688
	case CRYPTO_ALG_TYPE_AEAD:
689
	{
690
		struct aead_request *aead_req =
691
			container_of(req, struct aead_request, base);
692
		struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(aead_req);
693
		*txqidx = reqctx->txqidx;
694
		*rxqidx = reqctx->rxqidx;
695
		break;
696
	}
697
	case CRYPTO_ALG_TYPE_SKCIPHER:
698
	{
699
		struct skcipher_request *sk_req =
700
			container_of(req, struct skcipher_request, base);
701
		struct chcr_skcipher_req_ctx *reqctx =
702
			skcipher_request_ctx(sk_req);
703
		*txqidx = reqctx->txqidx;
704
		*rxqidx = reqctx->rxqidx;
705
		break;
706
	}
707
	case CRYPTO_ALG_TYPE_AHASH:
708
	{
709
		struct ahash_request *ahash_req =
710
			container_of(req, struct ahash_request, base);
711
		struct chcr_ahash_req_ctx *reqctx =
712
			ahash_request_ctx(ahash_req);
713
		*txqidx = reqctx->txqidx;
714
		*rxqidx = reqctx->rxqidx;
715
		break;
716
	}
717
	default:
718
		ret = -EINVAL;
719
		/* should never get here */
720
		BUG();
721
		break;
722
	}
723
	return ret;
724
}
725

726
static inline void create_wreq(struct chcr_context *ctx,
727
			       struct chcr_wr *chcr_req,
728
			       struct crypto_async_request *req,
729
			       unsigned int imm,
730
			       int hash_sz,
731
			       unsigned int len16,
732
			       unsigned int sc_len,
733
			       unsigned int lcb)
734
{
735
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
736
	unsigned int tx_channel_id, rx_channel_id;
737
	unsigned int txqidx = 0, rxqidx = 0;
738
	unsigned int qid, fid, portno;
739

740
	get_qidxs(req, &txqidx, &rxqidx);
741
	qid = u_ctx->lldi.rxq_ids[rxqidx];
742
	fid = u_ctx->lldi.rxq_ids[0];
743
	portno = rxqidx / ctx->rxq_perchan;
744
	tx_channel_id = txqidx / ctx->txq_perchan;
745
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[portno]);
746

747

748
	chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
749
	chcr_req->wreq.pld_size_hash_size =
750
		htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
751
	chcr_req->wreq.len16_pkd =
752
		htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
753
	chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
754
	chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid,
755
							    !!lcb, txqidx);
756

757
	chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid);
758
	chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
759
				((sizeof(chcr_req->wreq)) >> 4)));
760
	chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
761
	chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
762
					   sizeof(chcr_req->key_ctx) + sc_len);
763
}
764

765
/**
766
 *	create_cipher_wr - form the WR for cipher operations
767
 *	@wrparam: Container for create_cipher_wr()'s parameters
768
 */
769
static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
770
{
771
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
772
	struct chcr_context *ctx = c_ctx(tfm);
773
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
774
	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
775
	struct sk_buff *skb = NULL;
776
	struct chcr_wr *chcr_req;
777
	struct cpl_rx_phys_dsgl *phys_cpl;
778
	struct ulptx_sgl *ulptx;
779
	struct chcr_skcipher_req_ctx *reqctx =
780
		skcipher_request_ctx(wrparam->req);
781
	unsigned int temp = 0, transhdr_len, dst_size;
782
	int error;
783
	int nents;
784
	unsigned int kctx_len;
785
	gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
786
			GFP_KERNEL : GFP_ATOMIC;
787
	struct adapter *adap = padap(ctx->dev);
788
	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
789

790
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
791
	nents = sg_nents_xlen(reqctx->dstsg,  wrparam->bytes, CHCR_DST_SG_SIZE,
792
			      reqctx->dst_ofst);
793
	dst_size = get_space_for_phys_dsgl(nents);
794
	kctx_len = roundup(ablkctx->enckey_len, 16);
795
	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
796
	nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
797
				  CHCR_SRC_SG_SIZE, reqctx->src_ofst);
798
	temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
799
				     (sgl_len(nents) * 8);
800
	transhdr_len += temp;
801
	transhdr_len = roundup(transhdr_len, 16);
802
	skb = alloc_skb(SGE_MAX_WR_LEN, flags);
803
	if (!skb) {
804
		error = -ENOMEM;
805
		goto err;
806
	}
807
	chcr_req = __skb_put_zero(skb, transhdr_len);
808
	chcr_req->sec_cpl.op_ivinsrtofst =
809
			FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
810

811
	chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
812
	chcr_req->sec_cpl.aadstart_cipherstop_hi =
813
			FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
814

815
	chcr_req->sec_cpl.cipherstop_lo_authinsert =
816
			FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
817
	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
818
							 ablkctx->ciph_mode,
819
							 0, 0, IV >> 1);
820
	chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
821
							  0, 1, dst_size);
822

823
	chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
824
	if ((reqctx->op == CHCR_DECRYPT_OP) &&
825
	    (!(get_cryptoalg_subtype(tfm) ==
826
	       CRYPTO_ALG_SUB_TYPE_CTR)) &&
827
	    (!(get_cryptoalg_subtype(tfm) ==
828
	       CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
829
		generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
830
	} else {
831
		if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
832
		    (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
833
			memcpy(chcr_req->key_ctx.key, ablkctx->key,
834
			       ablkctx->enckey_len);
835
		} else {
836
			memcpy(chcr_req->key_ctx.key, ablkctx->key +
837
			       (ablkctx->enckey_len >> 1),
838
			       ablkctx->enckey_len >> 1);
839
			memcpy(chcr_req->key_ctx.key +
840
			       (ablkctx->enckey_len >> 1),
841
			       ablkctx->key,
842
			       ablkctx->enckey_len >> 1);
843
		}
844
	}
845
	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
846
	ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
847
	chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
848
	chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
849

850
	atomic_inc(&adap->chcr_stats.cipher_rqst);
851
	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
852
		+ (reqctx->imm ? (wrparam->bytes) : 0);
853
	create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
854
		    transhdr_len, temp,
855
			ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
856
	reqctx->skb = skb;
857

858
	if (reqctx->op && (ablkctx->ciph_mode ==
859
			   CHCR_SCMD_CIPHER_MODE_AES_CBC))
860
		sg_pcopy_to_buffer(wrparam->req->src,
861
			sg_nents(wrparam->req->src), wrparam->req->iv, 16,
862
			reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE);
863

864
	return skb;
865
err:
866
	return ERR_PTR(error);
867
}
868

869
static inline int chcr_keyctx_ck_size(unsigned int keylen)
870
{
871
	int ck_size = 0;
872

873
	if (keylen == AES_KEYSIZE_128)
874
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
875
	else if (keylen == AES_KEYSIZE_192)
876
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
877
	else if (keylen == AES_KEYSIZE_256)
878
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
879
	else
880
		ck_size = 0;
881

882
	return ck_size;
883
}
884
static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher,
885
				       const u8 *key,
886
				       unsigned int keylen)
887
{
888
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
889

890
	crypto_skcipher_clear_flags(ablkctx->sw_cipher,
891
				CRYPTO_TFM_REQ_MASK);
892
	crypto_skcipher_set_flags(ablkctx->sw_cipher,
893
				cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
894
	return crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
895
}
896

897
static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher,
898
			       const u8 *key,
899
			       unsigned int keylen)
900
{
901
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
902
	unsigned int ck_size, context_size;
903
	u16 alignment = 0;
904
	int err;
905

906
	err = chcr_cipher_fallback_setkey(cipher, key, keylen);
907
	if (err)
908
		goto badkey_err;
909

910
	ck_size = chcr_keyctx_ck_size(keylen);
911
	alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
912
	memcpy(ablkctx->key, key, keylen);
913
	ablkctx->enckey_len = keylen;
914
	get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
915
	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
916
			keylen + alignment) >> 4;
917

918
	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
919
						0, 0, context_size);
920
	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
921
	return 0;
922
badkey_err:
923
	ablkctx->enckey_len = 0;
924

925
	return err;
926
}
927

928
static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher,
929
				   const u8 *key,
930
				   unsigned int keylen)
931
{
932
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
933
	unsigned int ck_size, context_size;
934
	u16 alignment = 0;
935
	int err;
936

937
	err = chcr_cipher_fallback_setkey(cipher, key, keylen);
938
	if (err)
939
		goto badkey_err;
940
	ck_size = chcr_keyctx_ck_size(keylen);
941
	alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
942
	memcpy(ablkctx->key, key, keylen);
943
	ablkctx->enckey_len = keylen;
944
	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
945
			keylen + alignment) >> 4;
946

947
	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
948
						0, 0, context_size);
949
	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
950

951
	return 0;
952
badkey_err:
953
	ablkctx->enckey_len = 0;
954

955
	return err;
956
}
957

958
static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher,
959
				   const u8 *key,
960
				   unsigned int keylen)
961
{
962
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
963
	unsigned int ck_size, context_size;
964
	u16 alignment = 0;
965
	int err;
966

967
	if (keylen < CTR_RFC3686_NONCE_SIZE)
968
		return -EINVAL;
969
	memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
970
	       CTR_RFC3686_NONCE_SIZE);
971

972
	keylen -= CTR_RFC3686_NONCE_SIZE;
973
	err = chcr_cipher_fallback_setkey(cipher, key, keylen);
974
	if (err)
975
		goto badkey_err;
976

977
	ck_size = chcr_keyctx_ck_size(keylen);
978
	alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
979
	memcpy(ablkctx->key, key, keylen);
980
	ablkctx->enckey_len = keylen;
981
	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
982
			keylen + alignment) >> 4;
983

984
	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
985
						0, 0, context_size);
986
	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
987

988
	return 0;
989
badkey_err:
990
	ablkctx->enckey_len = 0;
991

992
	return err;
993
}
994
static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
995
{
996
	unsigned int size = AES_BLOCK_SIZE;
997
	__be32 *b = (__be32 *)(dstiv + size);
998
	u32 c, prev;
999

1000
	memcpy(dstiv, srciv, AES_BLOCK_SIZE);
1001
	for (; size >= 4; size -= 4) {
1002
		prev = be32_to_cpu(*--b);
1003
		c = prev + add;
1004
		*b = cpu_to_be32(c);
1005
		if (prev < c)
1006
			break;
1007
		add = 1;
1008
	}
1009

1010
}
1011

1012
static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1013
{
1014
	__be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1015
	u64 c;
1016
	u32 temp = be32_to_cpu(*--b);
1017

1018
	temp = ~temp;
1019
	c = (u64)temp +  1; // No of block can processed without overflow
1020
	if ((bytes / AES_BLOCK_SIZE) >= c)
1021
		bytes = c * AES_BLOCK_SIZE;
1022
	return bytes;
1023
}
1024

1025
static int chcr_update_tweak(struct skcipher_request *req, u8 *iv,
1026
			     u32 isfinal)
1027
{
1028
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1029
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1030
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1031
	struct crypto_aes_ctx aes;
1032
	int ret, i;
1033
	u8 *key;
1034
	unsigned int keylen;
1035
	int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1036
	int round8 = round / 8;
1037

1038
	memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1039

1040
	keylen = ablkctx->enckey_len / 2;
1041
	key = ablkctx->key + keylen;
1042
	/* For a 192 bit key remove the padded zeroes which was
1043
	 * added in chcr_xts_setkey
1044
	 */
1045
	if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr))
1046
			== CHCR_KEYCTX_CIPHER_KEY_SIZE_192)
1047
		ret = aes_expandkey(&aes, key, keylen - 8);
1048
	else
1049
		ret = aes_expandkey(&aes, key, keylen);
1050
	if (ret)
1051
		return ret;
1052
	aes_encrypt(&aes, iv, iv);
1053
	for (i = 0; i < round8; i++)
1054
		gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1055

1056
	for (i = 0; i < (round % 8); i++)
1057
		gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1058

1059
	if (!isfinal)
1060
		aes_decrypt(&aes, iv, iv);
1061

1062
	memzero_explicit(&aes, sizeof(aes));
1063
	return 0;
1064
}
1065

1066
static int chcr_update_cipher_iv(struct skcipher_request *req,
1067
				   struct cpl_fw6_pld *fw6_pld, u8 *iv)
1068
{
1069
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1070
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1071
	int subtype = get_cryptoalg_subtype(tfm);
1072
	int ret = 0;
1073

1074
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1075
		ctr_add_iv(iv, req->iv, (reqctx->processed /
1076
			   AES_BLOCK_SIZE));
1077
	else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1078
		*(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1079
			CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1080
						AES_BLOCK_SIZE) + 1);
1081
	else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1082
		ret = chcr_update_tweak(req, iv, 0);
1083
	else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1084
		if (reqctx->op)
1085
			/*Updated before sending last WR*/
1086
			memcpy(iv, req->iv, AES_BLOCK_SIZE);
1087
		else
1088
			memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1089
	}
1090

1091
	return ret;
1092

1093
}
1094

1095
/* We need separate function for final iv because in rfc3686  Initial counter
1096
 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1097
 * for subsequent update requests
1098
 */
1099

1100
static int chcr_final_cipher_iv(struct skcipher_request *req,
1101
				   struct cpl_fw6_pld *fw6_pld, u8 *iv)
1102
{
1103
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1104
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1105
	int subtype = get_cryptoalg_subtype(tfm);
1106
	int ret = 0;
1107

1108
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1109
		ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed,
1110
						       AES_BLOCK_SIZE));
1111
	else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) {
1112
		if (!reqctx->partial_req)
1113
			memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1114
		else
1115
			ret = chcr_update_tweak(req, iv, 1);
1116
	}
1117
	else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1118
		/*Already updated for Decrypt*/
1119
		if (!reqctx->op)
1120
			memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1121

1122
	}
1123
	return ret;
1124

1125
}
1126

1127
static int chcr_handle_cipher_resp(struct skcipher_request *req,
1128
				   unsigned char *input, int err)
1129
{
1130
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1131
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1132
	struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1133
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1134
	struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1135
	struct chcr_dev *dev = c_ctx(tfm)->dev;
1136
	struct chcr_context *ctx = c_ctx(tfm);
1137
	struct adapter *adap = padap(ctx->dev);
1138
	struct cipher_wr_param wrparam;
1139
	struct sk_buff *skb;
1140
	int bytes;
1141

1142
	if (err)
1143
		goto unmap;
1144
	if (req->cryptlen == reqctx->processed) {
1145
		chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1146
				      req);
1147
		err = chcr_final_cipher_iv(req, fw6_pld, req->iv);
1148
		goto complete;
1149
	}
1150

1151
	if (!reqctx->imm) {
1152
		bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
1153
					  CIP_SPACE_LEFT(ablkctx->enckey_len),
1154
					  reqctx->src_ofst, reqctx->dst_ofst);
1155
		if ((bytes + reqctx->processed) >= req->cryptlen)
1156
			bytes  = req->cryptlen - reqctx->processed;
1157
		else
1158
			bytes = rounddown(bytes, 16);
1159
	} else {
1160
		/*CTR mode counter overflow*/
1161
		bytes  = req->cryptlen - reqctx->processed;
1162
	}
1163
	err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1164
	if (err)
1165
		goto unmap;
1166

1167
	if (unlikely(bytes == 0)) {
1168
		chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1169
				      req);
1170
		memcpy(req->iv, reqctx->init_iv, IV);
1171
		atomic_inc(&adap->chcr_stats.fallback);
1172
		err = chcr_cipher_fallback(ablkctx->sw_cipher, req, req->iv,
1173
					   reqctx->op);
1174
		goto complete;
1175
	}
1176

1177
	if (get_cryptoalg_subtype(tfm) ==
1178
	    CRYPTO_ALG_SUB_TYPE_CTR)
1179
		bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1180
	wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx];
1181
	wrparam.req = req;
1182
	wrparam.bytes = bytes;
1183
	skb = create_cipher_wr(&wrparam);
1184
	if (IS_ERR(skb)) {
1185
		pr_err("%s : Failed to form WR. No memory\n", __func__);
1186
		err = PTR_ERR(skb);
1187
		goto unmap;
1188
	}
1189
	skb->dev = u_ctx->lldi.ports[0];
1190
	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1191
	chcr_send_wr(skb);
1192
	reqctx->last_req_len = bytes;
1193
	reqctx->processed += bytes;
1194
	if (get_cryptoalg_subtype(tfm) ==
1195
		CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1196
			CRYPTO_TFM_REQ_MAY_SLEEP ) {
1197
		complete(&ctx->cbc_aes_aio_done);
1198
	}
1199
	return 0;
1200
unmap:
1201
	chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1202
complete:
1203
	if (get_cryptoalg_subtype(tfm) ==
1204
		CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1205
			CRYPTO_TFM_REQ_MAY_SLEEP ) {
1206
		complete(&ctx->cbc_aes_aio_done);
1207
	}
1208
	chcr_dec_wrcount(dev);
1209
	skcipher_request_complete(req, err);
1210
	return err;
1211
}
1212

1213
static int process_cipher(struct skcipher_request *req,
1214
				  unsigned short qid,
1215
				  struct sk_buff **skb,
1216
				  unsigned short op_type)
1217
{
1218
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1219
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1220
	unsigned int ivsize = crypto_skcipher_ivsize(tfm);
1221
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1222
	struct adapter *adap = padap(c_ctx(tfm)->dev);
1223
	struct	cipher_wr_param wrparam;
1224
	int bytes, err = -EINVAL;
1225
	int subtype;
1226

1227
	reqctx->processed = 0;
1228
	reqctx->partial_req = 0;
1229
	if (!req->iv)
1230
		goto error;
1231
	subtype = get_cryptoalg_subtype(tfm);
1232
	if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1233
	    (req->cryptlen == 0) ||
1234
	    (req->cryptlen % crypto_skcipher_blocksize(tfm))) {
1235
		if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS)
1236
			goto fallback;
1237
		else if (req->cryptlen % crypto_skcipher_blocksize(tfm) &&
1238
			 subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1239
			goto fallback;
1240
		pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1241
		       ablkctx->enckey_len, req->cryptlen, ivsize);
1242
		goto error;
1243
	}
1244

1245
	err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1246
	if (err)
1247
		goto error;
1248
	if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1249
					    AES_MIN_KEY_SIZE +
1250
					    sizeof(struct cpl_rx_phys_dsgl) +
1251
					/*Min dsgl size*/
1252
					    32))) {
1253
		/* Can be sent as Imm*/
1254
		unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1255

1256
		dnents = sg_nents_xlen(req->dst, req->cryptlen,
1257
				       CHCR_DST_SG_SIZE, 0);
1258
		phys_dsgl = get_space_for_phys_dsgl(dnents);
1259
		kctx_len = roundup(ablkctx->enckey_len, 16);
1260
		transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1261
		reqctx->imm = (transhdr_len + IV + req->cryptlen) <=
1262
			SGE_MAX_WR_LEN;
1263
		bytes = IV + req->cryptlen;
1264

1265
	} else {
1266
		reqctx->imm = 0;
1267
	}
1268

1269
	if (!reqctx->imm) {
1270
		bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
1271
					  CIP_SPACE_LEFT(ablkctx->enckey_len),
1272
					  0, 0);
1273
		if ((bytes + reqctx->processed) >= req->cryptlen)
1274
			bytes  = req->cryptlen - reqctx->processed;
1275
		else
1276
			bytes = rounddown(bytes, 16);
1277
	} else {
1278
		bytes = req->cryptlen;
1279
	}
1280
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) {
1281
		bytes = adjust_ctr_overflow(req->iv, bytes);
1282
	}
1283
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1284
		memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1285
		memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
1286
				CTR_RFC3686_IV_SIZE);
1287

1288
		/* initialize counter portion of counter block */
1289
		*(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1290
			CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1291
		memcpy(reqctx->init_iv, reqctx->iv, IV);
1292

1293
	} else {
1294

1295
		memcpy(reqctx->iv, req->iv, IV);
1296
		memcpy(reqctx->init_iv, req->iv, IV);
1297
	}
1298
	if (unlikely(bytes == 0)) {
1299
		chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1300
				      req);
1301
fallback:       atomic_inc(&adap->chcr_stats.fallback);
1302
		err = chcr_cipher_fallback(ablkctx->sw_cipher, req,
1303
					   subtype ==
1304
					   CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ?
1305
					   reqctx->iv : req->iv,
1306
					   op_type);
1307
		goto error;
1308
	}
1309
	reqctx->op = op_type;
1310
	reqctx->srcsg = req->src;
1311
	reqctx->dstsg = req->dst;
1312
	reqctx->src_ofst = 0;
1313
	reqctx->dst_ofst = 0;
1314
	wrparam.qid = qid;
1315
	wrparam.req = req;
1316
	wrparam.bytes = bytes;
1317
	*skb = create_cipher_wr(&wrparam);
1318
	if (IS_ERR(*skb)) {
1319
		err = PTR_ERR(*skb);
1320
		goto unmap;
1321
	}
1322
	reqctx->processed = bytes;
1323
	reqctx->last_req_len = bytes;
1324
	reqctx->partial_req = !!(req->cryptlen - reqctx->processed);
1325

1326
	return 0;
1327
unmap:
1328
	chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1329
error:
1330
	return err;
1331
}
1332

1333
static int chcr_aes_encrypt(struct skcipher_request *req)
1334
{
1335
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1336
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1337
	struct chcr_dev *dev = c_ctx(tfm)->dev;
1338
	struct sk_buff *skb = NULL;
1339
	int err;
1340
	struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1341
	struct chcr_context *ctx = c_ctx(tfm);
1342
	unsigned int cpu;
1343

1344
	cpu = get_cpu();
1345
	reqctx->txqidx = cpu % ctx->ntxq;
1346
	reqctx->rxqidx = cpu % ctx->nrxq;
1347
	put_cpu();
1348

1349
	err = chcr_inc_wrcount(dev);
1350
	if (err)
1351
		return -ENXIO;
1352
	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1353
						reqctx->txqidx) &&
1354
		(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1355
			err = -ENOSPC;
1356
			goto error;
1357
	}
1358

1359
	err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1360
			     &skb, CHCR_ENCRYPT_OP);
1361
	if (err || !skb)
1362
		return  err;
1363
	skb->dev = u_ctx->lldi.ports[0];
1364
	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1365
	chcr_send_wr(skb);
1366
	if (get_cryptoalg_subtype(tfm) ==
1367
		CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1368
			CRYPTO_TFM_REQ_MAY_SLEEP ) {
1369
			reqctx->partial_req = 1;
1370
			wait_for_completion(&ctx->cbc_aes_aio_done);
1371
        }
1372
	return -EINPROGRESS;
1373
error:
1374
	chcr_dec_wrcount(dev);
1375
	return err;
1376
}
1377

1378
static int chcr_aes_decrypt(struct skcipher_request *req)
1379
{
1380
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1381
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1382
	struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1383
	struct chcr_dev *dev = c_ctx(tfm)->dev;
1384
	struct sk_buff *skb = NULL;
1385
	int err;
1386
	struct chcr_context *ctx = c_ctx(tfm);
1387
	unsigned int cpu;
1388

1389
	cpu = get_cpu();
1390
	reqctx->txqidx = cpu % ctx->ntxq;
1391
	reqctx->rxqidx = cpu % ctx->nrxq;
1392
	put_cpu();
1393

1394
	err = chcr_inc_wrcount(dev);
1395
	if (err)
1396
		return -ENXIO;
1397

1398
	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1399
						reqctx->txqidx) &&
1400
		(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))))
1401
			return -ENOSPC;
1402
	err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1403
			     &skb, CHCR_DECRYPT_OP);
1404
	if (err || !skb)
1405
		return err;
1406
	skb->dev = u_ctx->lldi.ports[0];
1407
	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1408
	chcr_send_wr(skb);
1409
	return -EINPROGRESS;
1410
}
1411
static int chcr_device_init(struct chcr_context *ctx)
1412
{
1413
	struct uld_ctx *u_ctx = NULL;
1414
	int txq_perchan, ntxq;
1415
	int err = 0, rxq_perchan;
1416

1417
	if (!ctx->dev) {
1418
		u_ctx = assign_chcr_device();
1419
		if (!u_ctx) {
1420
			err = -ENXIO;
1421
			pr_err("chcr device assignment fails\n");
1422
			goto out;
1423
		}
1424
		ctx->dev = &u_ctx->dev;
1425
		ntxq = u_ctx->lldi.ntxq;
1426
		rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1427
		txq_perchan = ntxq / u_ctx->lldi.nchan;
1428
		ctx->ntxq = ntxq;
1429
		ctx->nrxq = u_ctx->lldi.nrxq;
1430
		ctx->rxq_perchan = rxq_perchan;
1431
		ctx->txq_perchan = txq_perchan;
1432
	}
1433
out:
1434
	return err;
1435
}
1436

1437
static int chcr_init_tfm(struct crypto_skcipher *tfm)
1438
{
1439
	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1440
	struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1441
	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1442

1443
	ablkctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0,
1444
				CRYPTO_ALG_NEED_FALLBACK);
1445
	if (IS_ERR(ablkctx->sw_cipher)) {
1446
		pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1447
		return PTR_ERR(ablkctx->sw_cipher);
1448
	}
1449
	init_completion(&ctx->cbc_aes_aio_done);
1450
	crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) +
1451
					 crypto_skcipher_reqsize(ablkctx->sw_cipher));
1452

1453
	return chcr_device_init(ctx);
1454
}
1455

1456
static int chcr_rfc3686_init(struct crypto_skcipher *tfm)
1457
{
1458
	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1459
	struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1460
	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1461

1462
	/*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1463
	 * cannot be used as fallback in chcr_handle_cipher_response
1464
	 */
1465
	ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1466
				CRYPTO_ALG_NEED_FALLBACK);
1467
	if (IS_ERR(ablkctx->sw_cipher)) {
1468
		pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1469
		return PTR_ERR(ablkctx->sw_cipher);
1470
	}
1471
	crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) +
1472
				    crypto_skcipher_reqsize(ablkctx->sw_cipher));
1473
	return chcr_device_init(ctx);
1474
}
1475

1476

1477
static void chcr_exit_tfm(struct crypto_skcipher *tfm)
1478
{
1479
	struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1480
	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1481

1482
	crypto_free_skcipher(ablkctx->sw_cipher);
1483
}
1484

1485
static int get_alg_config(struct algo_param *params,
1486
			  unsigned int auth_size)
1487
{
1488
	switch (auth_size) {
1489
	case SHA1_DIGEST_SIZE:
1490
		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1491
		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1492
		params->result_size = SHA1_DIGEST_SIZE;
1493
		break;
1494
	case SHA224_DIGEST_SIZE:
1495
		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1496
		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1497
		params->result_size = SHA256_DIGEST_SIZE;
1498
		break;
1499
	case SHA256_DIGEST_SIZE:
1500
		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1501
		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1502
		params->result_size = SHA256_DIGEST_SIZE;
1503
		break;
1504
	case SHA384_DIGEST_SIZE:
1505
		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1506
		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1507
		params->result_size = SHA512_DIGEST_SIZE;
1508
		break;
1509
	case SHA512_DIGEST_SIZE:
1510
		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1511
		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1512
		params->result_size = SHA512_DIGEST_SIZE;
1513
		break;
1514
	default:
1515
		pr_err("ERROR, unsupported digest size\n");
1516
		return -EINVAL;
1517
	}
1518
	return 0;
1519
}
1520

1521
/**
1522
 *	create_hash_wr - Create hash work request
1523
 *	@req: Cipher req base
1524
 *	@param: Container for create_hash_wr()'s parameters
1525
 */
1526
static struct sk_buff *create_hash_wr(struct ahash_request *req,
1527
				      struct hash_wr_param *param)
1528
{
1529
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1530
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1531
	struct chcr_context *ctx = h_ctx(tfm);
1532
	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1533
	struct sk_buff *skb = NULL;
1534
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
1535
	struct chcr_wr *chcr_req;
1536
	struct ulptx_sgl *ulptx;
1537
	unsigned int nents = 0, transhdr_len;
1538
	unsigned int temp = 0;
1539
	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1540
		GFP_ATOMIC;
1541
	struct adapter *adap = padap(h_ctx(tfm)->dev);
1542
	int error = 0;
1543
	unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan;
1544

1545
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
1546
	transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
1547
	req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
1548
				param->sg_len) <= SGE_MAX_WR_LEN;
1549
	nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len,
1550
		      CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst);
1551
	nents += param->bfr_len ? 1 : 0;
1552
	transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
1553
				param->sg_len, 16) : (sgl_len(nents) * 8);
1554
	transhdr_len = roundup(transhdr_len, 16);
1555

1556
	skb = alloc_skb(transhdr_len, flags);
1557
	if (!skb)
1558
		return ERR_PTR(-ENOMEM);
1559
	chcr_req = __skb_put_zero(skb, transhdr_len);
1560

1561
	chcr_req->sec_cpl.op_ivinsrtofst =
1562
		FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0);
1563

1564
	chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1565

1566
	chcr_req->sec_cpl.aadstart_cipherstop_hi =
1567
		FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1568
	chcr_req->sec_cpl.cipherstop_lo_authinsert =
1569
		FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1570
	chcr_req->sec_cpl.seqno_numivs =
1571
		FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1572
					 param->opad_needed, 0);
1573

1574
	chcr_req->sec_cpl.ivgen_hdrlen =
1575
		FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1576

1577
	memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1578
	       param->alg_prm.result_size);
1579

1580
	if (param->opad_needed)
1581
		memcpy(chcr_req->key_ctx.key +
1582
		       ((param->alg_prm.result_size <= 32) ? 32 :
1583
			CHCR_HASH_MAX_DIGEST_SIZE),
1584
		       hmacctx->opad, param->alg_prm.result_size);
1585

1586
	chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1587
					    param->alg_prm.mk_size, 0,
1588
					    param->opad_needed,
1589
					    ((param->kctx_len +
1590
					     sizeof(chcr_req->key_ctx)) >> 4));
1591
	chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1592
	ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
1593
				     DUMMY_BYTES);
1594
	if (param->bfr_len != 0) {
1595
		req_ctx->hctx_wr.dma_addr =
1596
			dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
1597
				       param->bfr_len, DMA_TO_DEVICE);
1598
		if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
1599
				       req_ctx->hctx_wr. dma_addr)) {
1600
			error = -ENOMEM;
1601
			goto err;
1602
		}
1603
		req_ctx->hctx_wr.dma_len = param->bfr_len;
1604
	} else {
1605
		req_ctx->hctx_wr.dma_addr = 0;
1606
	}
1607
	chcr_add_hash_src_ent(req, ulptx, param);
1608
	/* Request upto max wr size */
1609
	temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
1610
				(param->sg_len + param->bfr_len) : 0);
1611
	atomic_inc(&adap->chcr_stats.digest_rqst);
1612
	create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm,
1613
		    param->hash_size, transhdr_len,
1614
		    temp,  0);
1615
	req_ctx->hctx_wr.skb = skb;
1616
	return skb;
1617
err:
1618
	kfree_skb(skb);
1619
	return  ERR_PTR(error);
1620
}
1621

1622
static int chcr_ahash_update(struct ahash_request *req)
1623
{
1624
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1625
	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1626
	struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1627
	struct chcr_context *ctx = h_ctx(rtfm);
1628
	struct chcr_dev *dev = h_ctx(rtfm)->dev;
1629
	struct sk_buff *skb;
1630
	u8 remainder = 0, bs;
1631
	unsigned int nbytes = req->nbytes;
1632
	struct hash_wr_param params;
1633
	int error;
1634
	unsigned int cpu;
1635

1636
	cpu = get_cpu();
1637
	req_ctx->txqidx = cpu % ctx->ntxq;
1638
	req_ctx->rxqidx = cpu % ctx->nrxq;
1639
	put_cpu();
1640

1641
	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1642

1643
	if (nbytes + req_ctx->reqlen >= bs) {
1644
		remainder = (nbytes + req_ctx->reqlen) % bs;
1645
		nbytes = nbytes + req_ctx->reqlen - remainder;
1646
	} else {
1647
		sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1648
				   + req_ctx->reqlen, nbytes, 0);
1649
		req_ctx->reqlen += nbytes;
1650
		return 0;
1651
	}
1652
	error = chcr_inc_wrcount(dev);
1653
	if (error)
1654
		return -ENXIO;
1655
	/* Detach state for CHCR means lldi or padap is freed. Increasing
1656
	 * inflight count for dev guarantees that lldi and padap is valid
1657
	 */
1658
	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1659
						req_ctx->txqidx) &&
1660
		(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1661
			error = -ENOSPC;
1662
			goto err;
1663
	}
1664

1665
	chcr_init_hctx_per_wr(req_ctx);
1666
	error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1667
	if (error) {
1668
		error = -ENOMEM;
1669
		goto err;
1670
	}
1671
	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1672
	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1673
	params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1674
				     HASH_SPACE_LEFT(params.kctx_len), 0);
1675
	if (params.sg_len > req->nbytes)
1676
		params.sg_len = req->nbytes;
1677
	params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
1678
			req_ctx->reqlen;
1679
	params.opad_needed = 0;
1680
	params.more = 1;
1681
	params.last = 0;
1682
	params.bfr_len = req_ctx->reqlen;
1683
	params.scmd1 = 0;
1684
	req_ctx->hctx_wr.srcsg = req->src;
1685

1686
	params.hash_size = params.alg_prm.result_size;
1687
	req_ctx->data_len += params.sg_len + params.bfr_len;
1688
	skb = create_hash_wr(req, &params);
1689
	if (IS_ERR(skb)) {
1690
		error = PTR_ERR(skb);
1691
		goto unmap;
1692
	}
1693

1694
	req_ctx->hctx_wr.processed += params.sg_len;
1695
	if (remainder) {
1696
		/* Swap buffers */
1697
		swap(req_ctx->reqbfr, req_ctx->skbfr);
1698
		sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1699
				   req_ctx->reqbfr, remainder, req->nbytes -
1700
				   remainder);
1701
	}
1702
	req_ctx->reqlen = remainder;
1703
	skb->dev = u_ctx->lldi.ports[0];
1704
	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1705
	chcr_send_wr(skb);
1706
	return -EINPROGRESS;
1707
unmap:
1708
	chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1709
err:
1710
	chcr_dec_wrcount(dev);
1711
	return error;
1712
}
1713

1714
static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1715
{
1716
	memset(bfr_ptr, 0, bs);
1717
	*bfr_ptr = 0x80;
1718
	if (bs == 64)
1719
		*(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1  << 3);
1720
	else
1721
		*(__be64 *)(bfr_ptr + 120) =  cpu_to_be64(scmd1  << 3);
1722
}
1723

1724
static int chcr_ahash_final(struct ahash_request *req)
1725
{
1726
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1727
	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1728
	struct chcr_dev *dev = h_ctx(rtfm)->dev;
1729
	struct hash_wr_param params;
1730
	struct sk_buff *skb;
1731
	struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1732
	struct chcr_context *ctx = h_ctx(rtfm);
1733
	u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1734
	int error;
1735
	unsigned int cpu;
1736

1737
	cpu = get_cpu();
1738
	req_ctx->txqidx = cpu % ctx->ntxq;
1739
	req_ctx->rxqidx = cpu % ctx->nrxq;
1740
	put_cpu();
1741

1742
	error = chcr_inc_wrcount(dev);
1743
	if (error)
1744
		return -ENXIO;
1745

1746
	chcr_init_hctx_per_wr(req_ctx);
1747
	if (is_hmac(crypto_ahash_tfm(rtfm)))
1748
		params.opad_needed = 1;
1749
	else
1750
		params.opad_needed = 0;
1751
	params.sg_len = 0;
1752
	req_ctx->hctx_wr.isfinal = 1;
1753
	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1754
	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1755
	if (is_hmac(crypto_ahash_tfm(rtfm))) {
1756
		params.opad_needed = 1;
1757
		params.kctx_len *= 2;
1758
	} else {
1759
		params.opad_needed = 0;
1760
	}
1761

1762
	req_ctx->hctx_wr.result = 1;
1763
	params.bfr_len = req_ctx->reqlen;
1764
	req_ctx->data_len += params.bfr_len + params.sg_len;
1765
	req_ctx->hctx_wr.srcsg = req->src;
1766
	if (req_ctx->reqlen == 0) {
1767
		create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1768
		params.last = 0;
1769
		params.more = 1;
1770
		params.scmd1 = 0;
1771
		params.bfr_len = bs;
1772

1773
	} else {
1774
		params.scmd1 = req_ctx->data_len;
1775
		params.last = 1;
1776
		params.more = 0;
1777
	}
1778
	params.hash_size = crypto_ahash_digestsize(rtfm);
1779
	skb = create_hash_wr(req, &params);
1780
	if (IS_ERR(skb)) {
1781
		error = PTR_ERR(skb);
1782
		goto err;
1783
	}
1784
	req_ctx->reqlen = 0;
1785
	skb->dev = u_ctx->lldi.ports[0];
1786
	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1787
	chcr_send_wr(skb);
1788
	return -EINPROGRESS;
1789
err:
1790
	chcr_dec_wrcount(dev);
1791
	return error;
1792
}
1793

1794
static int chcr_ahash_finup(struct ahash_request *req)
1795
{
1796
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1797
	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1798
	struct chcr_dev *dev = h_ctx(rtfm)->dev;
1799
	struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1800
	struct chcr_context *ctx = h_ctx(rtfm);
1801
	struct sk_buff *skb;
1802
	struct hash_wr_param params;
1803
	u8  bs;
1804
	int error;
1805
	unsigned int cpu;
1806

1807
	cpu = get_cpu();
1808
	req_ctx->txqidx = cpu % ctx->ntxq;
1809
	req_ctx->rxqidx = cpu % ctx->nrxq;
1810
	put_cpu();
1811

1812
	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1813
	error = chcr_inc_wrcount(dev);
1814
	if (error)
1815
		return -ENXIO;
1816

1817
	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1818
						req_ctx->txqidx) &&
1819
		(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1820
			error = -ENOSPC;
1821
			goto err;
1822
	}
1823
	chcr_init_hctx_per_wr(req_ctx);
1824
	error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1825
	if (error) {
1826
		error = -ENOMEM;
1827
		goto err;
1828
	}
1829

1830
	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1831
	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1832
	if (is_hmac(crypto_ahash_tfm(rtfm))) {
1833
		params.kctx_len *= 2;
1834
		params.opad_needed = 1;
1835
	} else {
1836
		params.opad_needed = 0;
1837
	}
1838

1839
	params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1840
				    HASH_SPACE_LEFT(params.kctx_len), 0);
1841
	if (params.sg_len < req->nbytes) {
1842
		if (is_hmac(crypto_ahash_tfm(rtfm))) {
1843
			params.kctx_len /= 2;
1844
			params.opad_needed = 0;
1845
		}
1846
		params.last = 0;
1847
		params.more = 1;
1848
		params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
1849
					- req_ctx->reqlen;
1850
		params.hash_size = params.alg_prm.result_size;
1851
		params.scmd1 = 0;
1852
	} else {
1853
		params.last = 1;
1854
		params.more = 0;
1855
		params.sg_len = req->nbytes;
1856
		params.hash_size = crypto_ahash_digestsize(rtfm);
1857
		params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
1858
				params.sg_len;
1859
	}
1860
	params.bfr_len = req_ctx->reqlen;
1861
	req_ctx->data_len += params.bfr_len + params.sg_len;
1862
	req_ctx->hctx_wr.result = 1;
1863
	req_ctx->hctx_wr.srcsg = req->src;
1864
	if ((req_ctx->reqlen + req->nbytes) == 0) {
1865
		create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1866
		params.last = 0;
1867
		params.more = 1;
1868
		params.scmd1 = 0;
1869
		params.bfr_len = bs;
1870
	}
1871
	skb = create_hash_wr(req, &params);
1872
	if (IS_ERR(skb)) {
1873
		error = PTR_ERR(skb);
1874
		goto unmap;
1875
	}
1876
	req_ctx->reqlen = 0;
1877
	req_ctx->hctx_wr.processed += params.sg_len;
1878
	skb->dev = u_ctx->lldi.ports[0];
1879
	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1880
	chcr_send_wr(skb);
1881
	return -EINPROGRESS;
1882
unmap:
1883
	chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1884
err:
1885
	chcr_dec_wrcount(dev);
1886
	return error;
1887
}
1888

1889
static int chcr_hmac_init(struct ahash_request *areq);
1890
static int chcr_sha_init(struct ahash_request *areq);
1891

1892
static int chcr_ahash_digest(struct ahash_request *req)
1893
{
1894
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1895
	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1896
	struct chcr_dev *dev = h_ctx(rtfm)->dev;
1897
	struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1898
	struct chcr_context *ctx = h_ctx(rtfm);
1899
	struct sk_buff *skb;
1900
	struct hash_wr_param params;
1901
	u8  bs;
1902
	int error;
1903
	unsigned int cpu;
1904

1905
	cpu = get_cpu();
1906
	req_ctx->txqidx = cpu % ctx->ntxq;
1907
	req_ctx->rxqidx = cpu % ctx->nrxq;
1908
	put_cpu();
1909

1910
	if (is_hmac(crypto_ahash_tfm(rtfm)))
1911
		chcr_hmac_init(req);
1912
	else
1913
		chcr_sha_init(req);
1914

1915
	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1916
	error = chcr_inc_wrcount(dev);
1917
	if (error)
1918
		return -ENXIO;
1919

1920
	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1921
						req_ctx->txqidx) &&
1922
		(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1923
			error = -ENOSPC;
1924
			goto err;
1925
	}
1926

1927
	chcr_init_hctx_per_wr(req_ctx);
1928
	error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1929
	if (error) {
1930
		error = -ENOMEM;
1931
		goto err;
1932
	}
1933

1934
	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1935
	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1936
	if (is_hmac(crypto_ahash_tfm(rtfm))) {
1937
		params.kctx_len *= 2;
1938
		params.opad_needed = 1;
1939
	} else {
1940
		params.opad_needed = 0;
1941
	}
1942
	params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1943
				HASH_SPACE_LEFT(params.kctx_len), 0);
1944
	if (params.sg_len < req->nbytes) {
1945
		if (is_hmac(crypto_ahash_tfm(rtfm))) {
1946
			params.kctx_len /= 2;
1947
			params.opad_needed = 0;
1948
		}
1949
		params.last = 0;
1950
		params.more = 1;
1951
		params.scmd1 = 0;
1952
		params.sg_len = rounddown(params.sg_len, bs);
1953
		params.hash_size = params.alg_prm.result_size;
1954
	} else {
1955
		params.sg_len = req->nbytes;
1956
		params.hash_size = crypto_ahash_digestsize(rtfm);
1957
		params.last = 1;
1958
		params.more = 0;
1959
		params.scmd1 = req->nbytes + req_ctx->data_len;
1960

1961
	}
1962
	params.bfr_len = 0;
1963
	req_ctx->hctx_wr.result = 1;
1964
	req_ctx->hctx_wr.srcsg = req->src;
1965
	req_ctx->data_len += params.bfr_len + params.sg_len;
1966

1967
	if (req->nbytes == 0) {
1968
		create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1969
		params.more = 1;
1970
		params.bfr_len = bs;
1971
	}
1972

1973
	skb = create_hash_wr(req, &params);
1974
	if (IS_ERR(skb)) {
1975
		error = PTR_ERR(skb);
1976
		goto unmap;
1977
	}
1978
	req_ctx->hctx_wr.processed += params.sg_len;
1979
	skb->dev = u_ctx->lldi.ports[0];
1980
	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1981
	chcr_send_wr(skb);
1982
	return -EINPROGRESS;
1983
unmap:
1984
	chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1985
err:
1986
	chcr_dec_wrcount(dev);
1987
	return error;
1988
}
1989

1990
static int chcr_ahash_continue(struct ahash_request *req)
1991
{
1992
	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
1993
	struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
1994
	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1995
	struct chcr_context *ctx = h_ctx(rtfm);
1996
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
1997
	struct sk_buff *skb;
1998
	struct hash_wr_param params;
1999
	u8  bs;
2000
	int error;
2001
	unsigned int cpu;
2002

2003
	cpu = get_cpu();
2004
	reqctx->txqidx = cpu % ctx->ntxq;
2005
	reqctx->rxqidx = cpu % ctx->nrxq;
2006
	put_cpu();
2007

2008
	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2009
	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
2010
	params.kctx_len = roundup(params.alg_prm.result_size, 16);
2011
	if (is_hmac(crypto_ahash_tfm(rtfm))) {
2012
		params.kctx_len *= 2;
2013
		params.opad_needed = 1;
2014
	} else {
2015
		params.opad_needed = 0;
2016
	}
2017
	params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
2018
					    HASH_SPACE_LEFT(params.kctx_len),
2019
					    hctx_wr->src_ofst);
2020
	if ((params.sg_len + hctx_wr->processed) > req->nbytes)
2021
		params.sg_len = req->nbytes - hctx_wr->processed;
2022
	if (!hctx_wr->result ||
2023
	    ((params.sg_len + hctx_wr->processed) < req->nbytes)) {
2024
		if (is_hmac(crypto_ahash_tfm(rtfm))) {
2025
			params.kctx_len /= 2;
2026
			params.opad_needed = 0;
2027
		}
2028
		params.last = 0;
2029
		params.more = 1;
2030
		params.sg_len = rounddown(params.sg_len, bs);
2031
		params.hash_size = params.alg_prm.result_size;
2032
		params.scmd1 = 0;
2033
	} else {
2034
		params.last = 1;
2035
		params.more = 0;
2036
		params.hash_size = crypto_ahash_digestsize(rtfm);
2037
		params.scmd1 = reqctx->data_len + params.sg_len;
2038
	}
2039
	params.bfr_len = 0;
2040
	reqctx->data_len += params.sg_len;
2041
	skb = create_hash_wr(req, &params);
2042
	if (IS_ERR(skb)) {
2043
		error = PTR_ERR(skb);
2044
		goto err;
2045
	}
2046
	hctx_wr->processed += params.sg_len;
2047
	skb->dev = u_ctx->lldi.ports[0];
2048
	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
2049
	chcr_send_wr(skb);
2050
	return 0;
2051
err:
2052
	return error;
2053
}
2054

2055
static inline void chcr_handle_ahash_resp(struct ahash_request *req,
2056
					  unsigned char *input,
2057
					  int err)
2058
{
2059
	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2060
	struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2061
	int digestsize, updated_digestsize;
2062
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2063
	struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
2064
	struct chcr_dev *dev = h_ctx(tfm)->dev;
2065

2066
	if (input == NULL)
2067
		goto out;
2068
	digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
2069
	updated_digestsize = digestsize;
2070
	if (digestsize == SHA224_DIGEST_SIZE)
2071
		updated_digestsize = SHA256_DIGEST_SIZE;
2072
	else if (digestsize == SHA384_DIGEST_SIZE)
2073
		updated_digestsize = SHA512_DIGEST_SIZE;
2074

2075
	if (hctx_wr->dma_addr) {
2076
		dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
2077
				 hctx_wr->dma_len, DMA_TO_DEVICE);
2078
		hctx_wr->dma_addr = 0;
2079
	}
2080
	if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
2081
				 req->nbytes)) {
2082
		if (hctx_wr->result == 1) {
2083
			hctx_wr->result = 0;
2084
			memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
2085
			       digestsize);
2086
		} else {
2087
			memcpy(reqctx->partial_hash,
2088
			       input + sizeof(struct cpl_fw6_pld),
2089
			       updated_digestsize);
2090

2091
		}
2092
		goto unmap;
2093
	}
2094
	memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
2095
	       updated_digestsize);
2096

2097
	err = chcr_ahash_continue(req);
2098
	if (err)
2099
		goto unmap;
2100
	return;
2101
unmap:
2102
	if (hctx_wr->is_sg_map)
2103
		chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
2104

2105

2106
out:
2107
	chcr_dec_wrcount(dev);
2108
	ahash_request_complete(req, err);
2109
}
2110

2111
/*
2112
 *	chcr_handle_resp - Unmap the DMA buffers associated with the request
2113
 *	@req: crypto request
2114
 */
2115
int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
2116
			 int err)
2117
{
2118
	struct crypto_tfm *tfm = req->tfm;
2119
	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2120
	struct adapter *adap = padap(ctx->dev);
2121

2122
	switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
2123
	case CRYPTO_ALG_TYPE_AEAD:
2124
		err = chcr_handle_aead_resp(aead_request_cast(req), input, err);
2125
		break;
2126

2127
	case CRYPTO_ALG_TYPE_SKCIPHER:
2128
		 chcr_handle_cipher_resp(skcipher_request_cast(req),
2129
					       input, err);
2130
		break;
2131
	case CRYPTO_ALG_TYPE_AHASH:
2132
		chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
2133
		}
2134
	atomic_inc(&adap->chcr_stats.complete);
2135
	return err;
2136
}
2137
static int chcr_ahash_export(struct ahash_request *areq, void *out)
2138
{
2139
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2140
	struct chcr_ahash_req_ctx *state = out;
2141

2142
	state->reqlen = req_ctx->reqlen;
2143
	state->data_len = req_ctx->data_len;
2144
	memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
2145
	memcpy(state->partial_hash, req_ctx->partial_hash,
2146
	       CHCR_HASH_MAX_DIGEST_SIZE);
2147
	chcr_init_hctx_per_wr(state);
2148
	return 0;
2149
}
2150

2151
static int chcr_ahash_import(struct ahash_request *areq, const void *in)
2152
{
2153
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2154
	struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
2155

2156
	req_ctx->reqlen = state->reqlen;
2157
	req_ctx->data_len = state->data_len;
2158
	req_ctx->reqbfr = req_ctx->bfr1;
2159
	req_ctx->skbfr = req_ctx->bfr2;
2160
	memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
2161
	memcpy(req_ctx->partial_hash, state->partial_hash,
2162
	       CHCR_HASH_MAX_DIGEST_SIZE);
2163
	chcr_init_hctx_per_wr(req_ctx);
2164
	return 0;
2165
}
2166

2167
static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2168
			     unsigned int keylen)
2169
{
2170
	struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
2171

2172
	/* use the key to calculate the ipad and opad. ipad will sent with the
2173
	 * first request's data. opad will be sent with the final hash result
2174
	 * ipad in hmacctx->ipad and opad in hmacctx->opad location
2175
	 */
2176
	return chcr_prepare_hmac_key(key, keylen, crypto_ahash_digestsize(tfm),
2177
				     hmacctx->ipad, hmacctx->opad);
2178
}
2179

2180
static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
2181
			       unsigned int key_len)
2182
{
2183
	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
2184
	unsigned short context_size = 0;
2185
	int err;
2186

2187
	err = chcr_cipher_fallback_setkey(cipher, key, key_len);
2188
	if (err)
2189
		goto badkey_err;
2190

2191
	memcpy(ablkctx->key, key, key_len);
2192
	ablkctx->enckey_len = key_len;
2193
	get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
2194
	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
2195
	/* Both keys for xts must be aligned to 16 byte boundary
2196
	 * by padding with zeros. So for 24 byte keys padding 8 zeroes.
2197
	 */
2198
	if (key_len == 48) {
2199
		context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len
2200
				+ 16) >> 4;
2201
		memmove(ablkctx->key + 32, ablkctx->key + 24, 24);
2202
		memset(ablkctx->key + 24, 0, 8);
2203
		memset(ablkctx->key + 56, 0, 8);
2204
		ablkctx->enckey_len = 64;
2205
		ablkctx->key_ctx_hdr =
2206
			FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192,
2207
					 CHCR_KEYCTX_NO_KEY, 1,
2208
					 0, context_size);
2209
	} else {
2210
		ablkctx->key_ctx_hdr =
2211
		FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
2212
				 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
2213
				 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
2214
				 CHCR_KEYCTX_NO_KEY, 1,
2215
				 0, context_size);
2216
	}
2217
	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
2218
	return 0;
2219
badkey_err:
2220
	ablkctx->enckey_len = 0;
2221

2222
	return err;
2223
}
2224

2225
static int chcr_sha_init(struct ahash_request *areq)
2226
{
2227
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2228
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2229
	int digestsize =  crypto_ahash_digestsize(tfm);
2230

2231
	req_ctx->data_len = 0;
2232
	req_ctx->reqlen = 0;
2233
	req_ctx->reqbfr = req_ctx->bfr1;
2234
	req_ctx->skbfr = req_ctx->bfr2;
2235
	copy_hash_init_values(req_ctx->partial_hash, digestsize);
2236

2237
	return 0;
2238
}
2239

2240
static int chcr_sha_cra_init(struct crypto_tfm *tfm)
2241
{
2242
	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2243
				 sizeof(struct chcr_ahash_req_ctx));
2244
	return chcr_device_init(crypto_tfm_ctx(tfm));
2245
}
2246

2247
static int chcr_hmac_init(struct ahash_request *areq)
2248
{
2249
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2250
	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
2251
	struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
2252
	unsigned int digestsize = crypto_ahash_digestsize(rtfm);
2253
	unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2254

2255
	chcr_sha_init(areq);
2256
	req_ctx->data_len = bs;
2257
	if (is_hmac(crypto_ahash_tfm(rtfm))) {
2258
		if (digestsize == SHA224_DIGEST_SIZE)
2259
			memcpy(req_ctx->partial_hash, hmacctx->ipad,
2260
			       SHA256_DIGEST_SIZE);
2261
		else if (digestsize == SHA384_DIGEST_SIZE)
2262
			memcpy(req_ctx->partial_hash, hmacctx->ipad,
2263
			       SHA512_DIGEST_SIZE);
2264
		else
2265
			memcpy(req_ctx->partial_hash, hmacctx->ipad,
2266
			       digestsize);
2267
	}
2268
	return 0;
2269
}
2270

2271
static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
2272
{
2273
	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2274
				 sizeof(struct chcr_ahash_req_ctx));
2275
	return chcr_device_init(crypto_tfm_ctx(tfm));
2276
}
2277

2278
inline void chcr_aead_common_exit(struct aead_request *req)
2279
{
2280
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2281
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2282
	struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
2283

2284
	chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
2285
}
2286

2287
static int chcr_aead_common_init(struct aead_request *req)
2288
{
2289
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2290
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2291
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2292
	unsigned int authsize = crypto_aead_authsize(tfm);
2293
	int error = -EINVAL;
2294

2295
	/* validate key size */
2296
	if (aeadctx->enckey_len == 0)
2297
		goto err;
2298
	if (reqctx->op && req->cryptlen < authsize)
2299
		goto err;
2300
	if (reqctx->b0_len)
2301
		reqctx->scratch_pad = reqctx->iv + IV;
2302
	else
2303
		reqctx->scratch_pad = NULL;
2304

2305
	error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2306
				  reqctx->op);
2307
	if (error) {
2308
		error = -ENOMEM;
2309
		goto err;
2310
	}
2311

2312
	return 0;
2313
err:
2314
	return error;
2315
}
2316

2317
static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
2318
				   int aadmax, int wrlen,
2319
				   unsigned short op_type)
2320
{
2321
	unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2322

2323
	if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2324
	    dst_nents > MAX_DSGL_ENT ||
2325
	    (req->assoclen > aadmax) ||
2326
	    (wrlen > SGE_MAX_WR_LEN))
2327
		return 1;
2328
	return 0;
2329
}
2330

2331
static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2332
{
2333
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2334
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2335
	struct aead_request *subreq = aead_request_ctx_dma(req);
2336

2337
	aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2338
	aead_request_set_callback(subreq, req->base.flags,
2339
				  req->base.complete, req->base.data);
2340
	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2341
				 req->iv);
2342
	aead_request_set_ad(subreq, req->assoclen);
2343
	return op_type ? crypto_aead_decrypt(subreq) :
2344
		crypto_aead_encrypt(subreq);
2345
}
2346

2347
static struct sk_buff *create_authenc_wr(struct aead_request *req,
2348
					 unsigned short qid,
2349
					 int size)
2350
{
2351
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2352
	struct chcr_context *ctx = a_ctx(tfm);
2353
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2354
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2355
	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2356
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2357
	struct sk_buff *skb = NULL;
2358
	struct chcr_wr *chcr_req;
2359
	struct cpl_rx_phys_dsgl *phys_cpl;
2360
	struct ulptx_sgl *ulptx;
2361
	unsigned int transhdr_len;
2362
	unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
2363
	unsigned int   kctx_len = 0, dnents, snents;
2364
	unsigned int  authsize = crypto_aead_authsize(tfm);
2365
	int error = -EINVAL;
2366
	u8 *ivptr;
2367
	int null = 0;
2368
	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2369
		GFP_ATOMIC;
2370
	struct adapter *adap = padap(ctx->dev);
2371
	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2372

2373
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2374
	if (req->cryptlen == 0)
2375
		return NULL;
2376

2377
	reqctx->b0_len = 0;
2378
	error = chcr_aead_common_init(req);
2379
	if (error)
2380
		return ERR_PTR(error);
2381

2382
	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
2383
		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2384
		null = 1;
2385
	}
2386
	dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
2387
		(reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0);
2388
	dnents += MIN_AUTH_SG; // For IV
2389
	snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
2390
			       CHCR_SRC_SG_SIZE, 0);
2391
	dst_size = get_space_for_phys_dsgl(dnents);
2392
	kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4)
2393
		- sizeof(chcr_req->key_ctx);
2394
	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2395
	reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <
2396
			SGE_MAX_WR_LEN;
2397
	temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16)
2398
			: (sgl_len(snents) * 8);
2399
	transhdr_len += temp;
2400
	transhdr_len = roundup(transhdr_len, 16);
2401

2402
	if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2403
				    transhdr_len, reqctx->op)) {
2404
		atomic_inc(&adap->chcr_stats.fallback);
2405
		chcr_aead_common_exit(req);
2406
		return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
2407
	}
2408
	skb = alloc_skb(transhdr_len, flags);
2409
	if (!skb) {
2410
		error = -ENOMEM;
2411
		goto err;
2412
	}
2413

2414
	chcr_req = __skb_put_zero(skb, transhdr_len);
2415

2416
	temp  = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2417

2418
	/*
2419
	 * Input order	is AAD,IV and Payload. where IV should be included as
2420
	 * the part of authdata. All other fields should be filled according
2421
	 * to the hardware spec
2422
	 */
2423
	chcr_req->sec_cpl.op_ivinsrtofst =
2424
				FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2425
	chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen);
2426
	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2427
					null ? 0 : 1 + IV,
2428
					null ? 0 : IV + req->assoclen,
2429
					req->assoclen + IV + 1,
2430
					(temp & 0x1F0) >> 4);
2431
	chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2432
					temp & 0xF,
2433
					null ? 0 : req->assoclen + IV + 1,
2434
					temp, temp);
2435
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2436
	    subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2437
		temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2438
	else
2439
		temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
2440
	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
2441
					(reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
2442
					temp,
2443
					actx->auth_mode, aeadctx->hmac_ctrl,
2444
					IV >> 1);
2445
	chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2446
					 0, 0, dst_size);
2447

2448
	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2449
	if (reqctx->op == CHCR_ENCRYPT_OP ||
2450
		subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2451
		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2452
		memcpy(chcr_req->key_ctx.key, aeadctx->key,
2453
		       aeadctx->enckey_len);
2454
	else
2455
		memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2456
		       aeadctx->enckey_len);
2457

2458
	memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2459
	       actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16));
2460
	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2461
	ivptr = (u8 *)(phys_cpl + 1) + dst_size;
2462
	ulptx = (struct ulptx_sgl *)(ivptr + IV);
2463
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2464
	    subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2465
		memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2466
		memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv,
2467
				CTR_RFC3686_IV_SIZE);
2468
		*(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE +
2469
			CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2470
	} else {
2471
		memcpy(ivptr, req->iv, IV);
2472
	}
2473
	chcr_add_aead_dst_ent(req, phys_cpl, qid);
2474
	chcr_add_aead_src_ent(req, ulptx);
2475
	atomic_inc(&adap->chcr_stats.cipher_rqst);
2476
	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
2477
		kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
2478
	create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2479
		   transhdr_len, temp, 0);
2480
	reqctx->skb = skb;
2481

2482
	return skb;
2483
err:
2484
	chcr_aead_common_exit(req);
2485

2486
	return ERR_PTR(error);
2487
}
2488

2489
int chcr_aead_dma_map(struct device *dev,
2490
		      struct aead_request *req,
2491
		      unsigned short op_type)
2492
{
2493
	int error;
2494
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2495
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2496
	unsigned int authsize = crypto_aead_authsize(tfm);
2497
	int src_len, dst_len;
2498

2499
	/* calculate and handle src and dst sg length separately
2500
	 * for inplace and out-of place operations
2501
	 */
2502
	if (req->src == req->dst) {
2503
		src_len = req->assoclen + req->cryptlen + (op_type ?
2504
							0 : authsize);
2505
		dst_len = src_len;
2506
	} else {
2507
		src_len = req->assoclen + req->cryptlen;
2508
		dst_len = req->assoclen + req->cryptlen + (op_type ?
2509
							-authsize : authsize);
2510
	}
2511

2512
	if (!req->cryptlen || !src_len || !dst_len)
2513
		return 0;
2514
	reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
2515
					DMA_BIDIRECTIONAL);
2516
	if (dma_mapping_error(dev, reqctx->iv_dma))
2517
		return -ENOMEM;
2518
	if (reqctx->b0_len)
2519
		reqctx->b0_dma = reqctx->iv_dma + IV;
2520
	else
2521
		reqctx->b0_dma = 0;
2522
	if (req->src == req->dst) {
2523
		error = dma_map_sg(dev, req->src,
2524
				sg_nents_for_len(req->src, src_len),
2525
					DMA_BIDIRECTIONAL);
2526
		if (!error)
2527
			goto err;
2528
	} else {
2529
		error = dma_map_sg(dev, req->src,
2530
				   sg_nents_for_len(req->src, src_len),
2531
				   DMA_TO_DEVICE);
2532
		if (!error)
2533
			goto err;
2534
		error = dma_map_sg(dev, req->dst,
2535
				   sg_nents_for_len(req->dst, dst_len),
2536
				   DMA_FROM_DEVICE);
2537
		if (!error) {
2538
			dma_unmap_sg(dev, req->src,
2539
				     sg_nents_for_len(req->src, src_len),
2540
				     DMA_TO_DEVICE);
2541
			goto err;
2542
		}
2543
	}
2544

2545
	return 0;
2546
err:
2547
	dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2548
	return -ENOMEM;
2549
}
2550

2551
void chcr_aead_dma_unmap(struct device *dev,
2552
			 struct aead_request *req,
2553
			 unsigned short op_type)
2554
{
2555
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2556
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2557
	unsigned int authsize = crypto_aead_authsize(tfm);
2558
	int src_len, dst_len;
2559

2560
	/* calculate and handle src and dst sg length separately
2561
	 * for inplace and out-of place operations
2562
	 */
2563
	if (req->src == req->dst) {
2564
		src_len = req->assoclen + req->cryptlen + (op_type ?
2565
							0 : authsize);
2566
		dst_len = src_len;
2567
	} else {
2568
		src_len = req->assoclen + req->cryptlen;
2569
		dst_len = req->assoclen + req->cryptlen + (op_type ?
2570
						-authsize : authsize);
2571
	}
2572

2573
	if (!req->cryptlen || !src_len || !dst_len)
2574
		return;
2575

2576
	dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
2577
					DMA_BIDIRECTIONAL);
2578
	if (req->src == req->dst) {
2579
		dma_unmap_sg(dev, req->src,
2580
			     sg_nents_for_len(req->src, src_len),
2581
			     DMA_BIDIRECTIONAL);
2582
	} else {
2583
		dma_unmap_sg(dev, req->src,
2584
			     sg_nents_for_len(req->src, src_len),
2585
			     DMA_TO_DEVICE);
2586
		dma_unmap_sg(dev, req->dst,
2587
			     sg_nents_for_len(req->dst, dst_len),
2588
			     DMA_FROM_DEVICE);
2589
	}
2590
}
2591

2592
void chcr_add_aead_src_ent(struct aead_request *req,
2593
			   struct ulptx_sgl *ulptx)
2594
{
2595
	struct ulptx_walk ulp_walk;
2596
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2597

2598
	if (reqctx->imm) {
2599
		u8 *buf = (u8 *)ulptx;
2600

2601
		if (reqctx->b0_len) {
2602
			memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2603
			buf += reqctx->b0_len;
2604
		}
2605
		sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2606
				   buf, req->cryptlen + req->assoclen, 0);
2607
	} else {
2608
		ulptx_walk_init(&ulp_walk, ulptx);
2609
		if (reqctx->b0_len)
2610
			ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
2611
					    reqctx->b0_dma);
2612
		ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen +
2613
				  req->assoclen,  0);
2614
		ulptx_walk_end(&ulp_walk);
2615
	}
2616
}
2617

2618
void chcr_add_aead_dst_ent(struct aead_request *req,
2619
			   struct cpl_rx_phys_dsgl *phys_cpl,
2620
			   unsigned short qid)
2621
{
2622
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2623
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2624
	struct dsgl_walk dsgl_walk;
2625
	unsigned int authsize = crypto_aead_authsize(tfm);
2626
	struct chcr_context *ctx = a_ctx(tfm);
2627
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2628
	u32 temp;
2629
	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2630

2631
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2632
	dsgl_walk_init(&dsgl_walk, phys_cpl);
2633
	dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma);
2634
	temp = req->assoclen + req->cryptlen +
2635
		(reqctx->op ? -authsize : authsize);
2636
	dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0);
2637
	dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2638
}
2639

2640
void chcr_add_cipher_src_ent(struct skcipher_request *req,
2641
			     void *ulptx,
2642
			     struct  cipher_wr_param *wrparam)
2643
{
2644
	struct ulptx_walk ulp_walk;
2645
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2646
	u8 *buf = ulptx;
2647

2648
	memcpy(buf, reqctx->iv, IV);
2649
	buf += IV;
2650
	if (reqctx->imm) {
2651
		sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2652
				   buf, wrparam->bytes, reqctx->processed);
2653
	} else {
2654
		ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf);
2655
		ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2656
				  reqctx->src_ofst);
2657
		reqctx->srcsg = ulp_walk.last_sg;
2658
		reqctx->src_ofst = ulp_walk.last_sg_len;
2659
		ulptx_walk_end(&ulp_walk);
2660
	}
2661
}
2662

2663
void chcr_add_cipher_dst_ent(struct skcipher_request *req,
2664
			     struct cpl_rx_phys_dsgl *phys_cpl,
2665
			     struct  cipher_wr_param *wrparam,
2666
			     unsigned short qid)
2667
{
2668
	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2669
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
2670
	struct chcr_context *ctx = c_ctx(tfm);
2671
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2672
	struct dsgl_walk dsgl_walk;
2673
	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2674

2675
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2676
	dsgl_walk_init(&dsgl_walk, phys_cpl);
2677
	dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2678
			 reqctx->dst_ofst);
2679
	reqctx->dstsg = dsgl_walk.last_sg;
2680
	reqctx->dst_ofst = dsgl_walk.last_sg_len;
2681
	dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2682
}
2683

2684
void chcr_add_hash_src_ent(struct ahash_request *req,
2685
			   struct ulptx_sgl *ulptx,
2686
			   struct hash_wr_param *param)
2687
{
2688
	struct ulptx_walk ulp_walk;
2689
	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2690

2691
	if (reqctx->hctx_wr.imm) {
2692
		u8 *buf = (u8 *)ulptx;
2693

2694
		if (param->bfr_len) {
2695
			memcpy(buf, reqctx->reqbfr, param->bfr_len);
2696
			buf += param->bfr_len;
2697
		}
2698

2699
		sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg,
2700
				   sg_nents(reqctx->hctx_wr.srcsg), buf,
2701
				   param->sg_len, 0);
2702
	} else {
2703
		ulptx_walk_init(&ulp_walk, ulptx);
2704
		if (param->bfr_len)
2705
			ulptx_walk_add_page(&ulp_walk, param->bfr_len,
2706
					    reqctx->hctx_wr.dma_addr);
2707
		ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg,
2708
				  param->sg_len, reqctx->hctx_wr.src_ofst);
2709
		reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
2710
		reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
2711
		ulptx_walk_end(&ulp_walk);
2712
	}
2713
}
2714

2715
int chcr_hash_dma_map(struct device *dev,
2716
		      struct ahash_request *req)
2717
{
2718
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2719
	int error = 0;
2720

2721
	if (!req->nbytes)
2722
		return 0;
2723
	error = dma_map_sg(dev, req->src, sg_nents(req->src),
2724
			   DMA_TO_DEVICE);
2725
	if (!error)
2726
		return -ENOMEM;
2727
	req_ctx->hctx_wr.is_sg_map = 1;
2728
	return 0;
2729
}
2730

2731
void chcr_hash_dma_unmap(struct device *dev,
2732
			 struct ahash_request *req)
2733
{
2734
	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2735

2736
	if (!req->nbytes)
2737
		return;
2738

2739
	dma_unmap_sg(dev, req->src, sg_nents(req->src),
2740
			   DMA_TO_DEVICE);
2741
	req_ctx->hctx_wr.is_sg_map = 0;
2742

2743
}
2744

2745
int chcr_cipher_dma_map(struct device *dev,
2746
			struct skcipher_request *req)
2747
{
2748
	int error;
2749

2750
	if (req->src == req->dst) {
2751
		error = dma_map_sg(dev, req->src, sg_nents(req->src),
2752
				   DMA_BIDIRECTIONAL);
2753
		if (!error)
2754
			goto err;
2755
	} else {
2756
		error = dma_map_sg(dev, req->src, sg_nents(req->src),
2757
				   DMA_TO_DEVICE);
2758
		if (!error)
2759
			goto err;
2760
		error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2761
				   DMA_FROM_DEVICE);
2762
		if (!error) {
2763
			dma_unmap_sg(dev, req->src, sg_nents(req->src),
2764
				   DMA_TO_DEVICE);
2765
			goto err;
2766
		}
2767
	}
2768

2769
	return 0;
2770
err:
2771
	return -ENOMEM;
2772
}
2773

2774
void chcr_cipher_dma_unmap(struct device *dev,
2775
			   struct skcipher_request *req)
2776
{
2777
	if (req->src == req->dst) {
2778
		dma_unmap_sg(dev, req->src, sg_nents(req->src),
2779
				   DMA_BIDIRECTIONAL);
2780
	} else {
2781
		dma_unmap_sg(dev, req->src, sg_nents(req->src),
2782
				   DMA_TO_DEVICE);
2783
		dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2784
				   DMA_FROM_DEVICE);
2785
	}
2786
}
2787

2788
static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2789
{
2790
	__be32 data;
2791

2792
	memset(block, 0, csize);
2793
	block += csize;
2794

2795
	if (csize >= 4)
2796
		csize = 4;
2797
	else if (msglen > (unsigned int)(1 << (8 * csize)))
2798
		return -EOVERFLOW;
2799

2800
	data = cpu_to_be32(msglen);
2801
	memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2802

2803
	return 0;
2804
}
2805

2806
static int generate_b0(struct aead_request *req, u8 *ivptr,
2807
			unsigned short op_type)
2808
{
2809
	unsigned int l, lp, m;
2810
	int rc;
2811
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2812
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2813
	u8 *b0 = reqctx->scratch_pad;
2814

2815
	m = crypto_aead_authsize(aead);
2816

2817
	memcpy(b0, ivptr, 16);
2818

2819
	lp = b0[0];
2820
	l = lp + 1;
2821

2822
	/* set m, bits 3-5 */
2823
	*b0 |= (8 * ((m - 2) / 2));
2824

2825
	/* set adata, bit 6, if associated data is used */
2826
	if (req->assoclen)
2827
		*b0 |= 64;
2828
	rc = set_msg_len(b0 + 16 - l,
2829
			 (op_type == CHCR_DECRYPT_OP) ?
2830
			 req->cryptlen - m : req->cryptlen, l);
2831

2832
	return rc;
2833
}
2834

2835
static inline int crypto_ccm_check_iv(const u8 *iv)
2836
{
2837
	/* 2 <= L <= 8, so 1 <= L' <= 7. */
2838
	if (iv[0] < 1 || iv[0] > 7)
2839
		return -EINVAL;
2840

2841
	return 0;
2842
}
2843

2844
static int ccm_format_packet(struct aead_request *req,
2845
			     u8 *ivptr,
2846
			     unsigned int sub_type,
2847
			     unsigned short op_type,
2848
			     unsigned int assoclen)
2849
{
2850
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2851
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2852
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2853
	int rc = 0;
2854

2855
	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2856
		ivptr[0] = 3;
2857
		memcpy(ivptr + 1, &aeadctx->salt[0], 3);
2858
		memcpy(ivptr + 4, req->iv, 8);
2859
		memset(ivptr + 12, 0, 4);
2860
	} else {
2861
		memcpy(ivptr, req->iv, 16);
2862
	}
2863
	if (assoclen)
2864
		put_unaligned_be16(assoclen, &reqctx->scratch_pad[16]);
2865

2866
	rc = generate_b0(req, ivptr, op_type);
2867
	/* zero the ctr value */
2868
	memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1);
2869
	return rc;
2870
}
2871

2872
static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2873
				  unsigned int dst_size,
2874
				  struct aead_request *req,
2875
				  unsigned short op_type)
2876
{
2877
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2878
	struct chcr_context *ctx = a_ctx(tfm);
2879
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2880
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2881
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2882
	unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2883
	unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2884
	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2885
	unsigned int ccm_xtra;
2886
	unsigned int tag_offset = 0, auth_offset = 0;
2887
	unsigned int assoclen;
2888

2889
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2890

2891
	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2892
		assoclen = req->assoclen - 8;
2893
	else
2894
		assoclen = req->assoclen;
2895
	ccm_xtra = CCM_B0_SIZE +
2896
		((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2897

2898
	auth_offset = req->cryptlen ?
2899
		(req->assoclen + IV + 1 + ccm_xtra) : 0;
2900
	if (op_type == CHCR_DECRYPT_OP) {
2901
		if (crypto_aead_authsize(tfm) != req->cryptlen)
2902
			tag_offset = crypto_aead_authsize(tfm);
2903
		else
2904
			auth_offset = 0;
2905
	}
2906

2907
	sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2908
	sec_cpl->pldlen =
2909
		htonl(req->assoclen + IV + req->cryptlen + ccm_xtra);
2910
	/* For CCM there wil be b0 always. So AAD start will be 1 always */
2911
	sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2912
				1 + IV,	IV + assoclen + ccm_xtra,
2913
				req->assoclen + IV + 1 + ccm_xtra, 0);
2914

2915
	sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2916
					auth_offset, tag_offset,
2917
					(op_type == CHCR_ENCRYPT_OP) ? 0 :
2918
					crypto_aead_authsize(tfm));
2919
	sec_cpl->seqno_numivs =  FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2920
					(op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2921
					cipher_mode, mac_mode,
2922
					aeadctx->hmac_ctrl, IV >> 1);
2923

2924
	sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2925
					0, dst_size);
2926
}
2927

2928
static int aead_ccm_validate_input(unsigned short op_type,
2929
				   struct aead_request *req,
2930
				   struct chcr_aead_ctx *aeadctx,
2931
				   unsigned int sub_type)
2932
{
2933
	if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2934
		if (crypto_ccm_check_iv(req->iv)) {
2935
			pr_err("CCM: IV check fails\n");
2936
			return -EINVAL;
2937
		}
2938
	} else {
2939
		if (req->assoclen != 16 && req->assoclen != 20) {
2940
			pr_err("RFC4309: Invalid AAD length %d\n",
2941
			       req->assoclen);
2942
			return -EINVAL;
2943
		}
2944
	}
2945
	return 0;
2946
}
2947

2948
static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2949
					  unsigned short qid,
2950
					  int size)
2951
{
2952
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2953
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2954
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2955
	struct sk_buff *skb = NULL;
2956
	struct chcr_wr *chcr_req;
2957
	struct cpl_rx_phys_dsgl *phys_cpl;
2958
	struct ulptx_sgl *ulptx;
2959
	unsigned int transhdr_len;
2960
	unsigned int dst_size = 0, kctx_len, dnents, temp, snents;
2961
	unsigned int sub_type, assoclen = req->assoclen;
2962
	unsigned int authsize = crypto_aead_authsize(tfm);
2963
	int error = -EINVAL;
2964
	u8 *ivptr;
2965
	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2966
		GFP_ATOMIC;
2967
	struct adapter *adap = padap(a_ctx(tfm)->dev);
2968

2969
	sub_type = get_aead_subtype(tfm);
2970
	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2971
		assoclen -= 8;
2972
	reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2973
	error = chcr_aead_common_init(req);
2974
	if (error)
2975
		return ERR_PTR(error);
2976

2977
	error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type);
2978
	if (error)
2979
		goto err;
2980
	dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen
2981
			+ (reqctx->op ? -authsize : authsize),
2982
			CHCR_DST_SG_SIZE, 0);
2983
	dnents += MIN_CCM_SG; // For IV and B0
2984
	dst_size = get_space_for_phys_dsgl(dnents);
2985
	snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
2986
			       CHCR_SRC_SG_SIZE, 0);
2987
	snents += MIN_CCM_SG; //For B0
2988
	kctx_len = roundup(aeadctx->enckey_len, 16) * 2;
2989
	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2990
	reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen +
2991
		       reqctx->b0_len) <= SGE_MAX_WR_LEN;
2992
	temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen +
2993
				     reqctx->b0_len, 16) :
2994
		(sgl_len(snents) *  8);
2995
	transhdr_len += temp;
2996
	transhdr_len = roundup(transhdr_len, 16);
2997

2998
	if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
2999
				reqctx->b0_len, transhdr_len, reqctx->op)) {
3000
		atomic_inc(&adap->chcr_stats.fallback);
3001
		chcr_aead_common_exit(req);
3002
		return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
3003
	}
3004
	skb = alloc_skb(transhdr_len,  flags);
3005

3006
	if (!skb) {
3007
		error = -ENOMEM;
3008
		goto err;
3009
	}
3010

3011
	chcr_req = __skb_put_zero(skb, transhdr_len);
3012

3013
	fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op);
3014

3015
	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3016
	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3017
	memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3018
			aeadctx->key, aeadctx->enckey_len);
3019

3020
	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3021
	ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3022
	ulptx = (struct ulptx_sgl *)(ivptr + IV);
3023
	error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen);
3024
	if (error)
3025
		goto dstmap_fail;
3026
	chcr_add_aead_dst_ent(req, phys_cpl, qid);
3027
	chcr_add_aead_src_ent(req, ulptx);
3028

3029
	atomic_inc(&adap->chcr_stats.aead_rqst);
3030
	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3031
		kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen +
3032
		reqctx->b0_len) : 0);
3033
	create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
3034
		    transhdr_len, temp, 0);
3035
	reqctx->skb = skb;
3036

3037
	return skb;
3038
dstmap_fail:
3039
	kfree_skb(skb);
3040
err:
3041
	chcr_aead_common_exit(req);
3042
	return ERR_PTR(error);
3043
}
3044

3045
static struct sk_buff *create_gcm_wr(struct aead_request *req,
3046
				     unsigned short qid,
3047
				     int size)
3048
{
3049
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3050
	struct chcr_context *ctx = a_ctx(tfm);
3051
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
3052
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3053
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3054
	struct sk_buff *skb = NULL;
3055
	struct chcr_wr *chcr_req;
3056
	struct cpl_rx_phys_dsgl *phys_cpl;
3057
	struct ulptx_sgl *ulptx;
3058
	unsigned int transhdr_len, dnents = 0, snents;
3059
	unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
3060
	unsigned int authsize = crypto_aead_authsize(tfm);
3061
	int error = -EINVAL;
3062
	u8 *ivptr;
3063
	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
3064
		GFP_ATOMIC;
3065
	struct adapter *adap = padap(ctx->dev);
3066
	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
3067

3068
	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
3069
	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
3070
		assoclen = req->assoclen - 8;
3071

3072
	reqctx->b0_len = 0;
3073
	error = chcr_aead_common_init(req);
3074
	if (error)
3075
		return ERR_PTR(error);
3076
	dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
3077
				(reqctx->op ? -authsize : authsize),
3078
				CHCR_DST_SG_SIZE, 0);
3079
	snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3080
			       CHCR_SRC_SG_SIZE, 0);
3081
	dnents += MIN_GCM_SG; // For IV
3082
	dst_size = get_space_for_phys_dsgl(dnents);
3083
	kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE;
3084
	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
3085
	reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <=
3086
			SGE_MAX_WR_LEN;
3087
	temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) :
3088
		(sgl_len(snents) * 8);
3089
	transhdr_len += temp;
3090
	transhdr_len = roundup(transhdr_len, 16);
3091
	if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
3092
			    transhdr_len, reqctx->op)) {
3093

3094
		atomic_inc(&adap->chcr_stats.fallback);
3095
		chcr_aead_common_exit(req);
3096
		return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
3097
	}
3098
	skb = alloc_skb(transhdr_len, flags);
3099
	if (!skb) {
3100
		error = -ENOMEM;
3101
		goto err;
3102
	}
3103

3104
	chcr_req = __skb_put_zero(skb, transhdr_len);
3105

3106
	//Offset of tag from end
3107
	temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
3108
	chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
3109
						rx_channel_id, 2, 1);
3110
	chcr_req->sec_cpl.pldlen =
3111
		htonl(req->assoclen + IV + req->cryptlen);
3112
	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
3113
					assoclen ? 1 + IV : 0,
3114
					assoclen ? IV + assoclen : 0,
3115
					req->assoclen + IV + 1, 0);
3116
	chcr_req->sec_cpl.cipherstop_lo_authinsert =
3117
			FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1,
3118
						temp, temp);
3119
	chcr_req->sec_cpl.seqno_numivs =
3120
			FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
3121
					CHCR_ENCRYPT_OP) ? 1 : 0,
3122
					CHCR_SCMD_CIPHER_MODE_AES_GCM,
3123
					CHCR_SCMD_AUTH_MODE_GHASH,
3124
					aeadctx->hmac_ctrl, IV >> 1);
3125
	chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
3126
					0, 0, dst_size);
3127
	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3128
	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3129
	memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3130
	       GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
3131

3132
	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3133
	ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3134
	/* prepare a 16 byte iv */
3135
	/* S   A   L  T |  IV | 0x00000001 */
3136
	if (get_aead_subtype(tfm) ==
3137
	    CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
3138
		memcpy(ivptr, aeadctx->salt, 4);
3139
		memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE);
3140
	} else {
3141
		memcpy(ivptr, req->iv, GCM_AES_IV_SIZE);
3142
	}
3143
	put_unaligned_be32(0x01, &ivptr[12]);
3144
	ulptx = (struct ulptx_sgl *)(ivptr + 16);
3145

3146
	chcr_add_aead_dst_ent(req, phys_cpl, qid);
3147
	chcr_add_aead_src_ent(req, ulptx);
3148
	atomic_inc(&adap->chcr_stats.aead_rqst);
3149
	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3150
		kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
3151
	create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
3152
		    transhdr_len, temp, reqctx->verify);
3153
	reqctx->skb = skb;
3154
	return skb;
3155

3156
err:
3157
	chcr_aead_common_exit(req);
3158
	return ERR_PTR(error);
3159
}
3160

3161

3162

3163
static int chcr_aead_cra_init(struct crypto_aead *tfm)
3164
{
3165
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3166
	struct aead_alg *alg = crypto_aead_alg(tfm);
3167

3168
	aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
3169
					       CRYPTO_ALG_NEED_FALLBACK |
3170
					       CRYPTO_ALG_ASYNC);
3171
	if  (IS_ERR(aeadctx->sw_cipher))
3172
		return PTR_ERR(aeadctx->sw_cipher);
3173
	crypto_aead_set_reqsize_dma(
3174
		tfm, max(sizeof(struct chcr_aead_reqctx),
3175
			 sizeof(struct aead_request) +
3176
			 crypto_aead_reqsize(aeadctx->sw_cipher)));
3177
	return chcr_device_init(a_ctx(tfm));
3178
}
3179

3180
static void chcr_aead_cra_exit(struct crypto_aead *tfm)
3181
{
3182
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3183

3184
	crypto_free_aead(aeadctx->sw_cipher);
3185
}
3186

3187
static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
3188
					unsigned int authsize)
3189
{
3190
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3191

3192
	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
3193
	aeadctx->mayverify = VERIFY_HW;
3194
	return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3195
}
3196
static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
3197
				    unsigned int authsize)
3198
{
3199
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3200
	u32 maxauth = crypto_aead_maxauthsize(tfm);
3201

3202
	/*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3203
	 * true for sha1. authsize == 12 condition should be before
3204
	 * authsize == (maxauth >> 1)
3205
	 */
3206
	if (authsize == ICV_4) {
3207
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3208
		aeadctx->mayverify = VERIFY_HW;
3209
	} else if (authsize == ICV_6) {
3210
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3211
		aeadctx->mayverify = VERIFY_HW;
3212
	} else if (authsize == ICV_10) {
3213
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3214
		aeadctx->mayverify = VERIFY_HW;
3215
	} else if (authsize == ICV_12) {
3216
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3217
		aeadctx->mayverify = VERIFY_HW;
3218
	} else if (authsize == ICV_14) {
3219
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3220
		aeadctx->mayverify = VERIFY_HW;
3221
	} else if (authsize == (maxauth >> 1)) {
3222
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3223
		aeadctx->mayverify = VERIFY_HW;
3224
	} else if (authsize == maxauth) {
3225
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3226
		aeadctx->mayverify = VERIFY_HW;
3227
	} else {
3228
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3229
		aeadctx->mayverify = VERIFY_SW;
3230
	}
3231
	return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3232
}
3233

3234

3235
static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
3236
{
3237
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3238

3239
	switch (authsize) {
3240
	case ICV_4:
3241
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3242
		aeadctx->mayverify = VERIFY_HW;
3243
		break;
3244
	case ICV_8:
3245
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3246
		aeadctx->mayverify = VERIFY_HW;
3247
		break;
3248
	case ICV_12:
3249
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3250
		aeadctx->mayverify = VERIFY_HW;
3251
		break;
3252
	case ICV_14:
3253
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3254
		aeadctx->mayverify = VERIFY_HW;
3255
		break;
3256
	case ICV_16:
3257
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3258
		aeadctx->mayverify = VERIFY_HW;
3259
		break;
3260
	case ICV_13:
3261
	case ICV_15:
3262
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3263
		aeadctx->mayverify = VERIFY_SW;
3264
		break;
3265
	default:
3266
		return -EINVAL;
3267
	}
3268
	return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3269
}
3270

3271
static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
3272
					  unsigned int authsize)
3273
{
3274
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3275

3276
	switch (authsize) {
3277
	case ICV_8:
3278
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3279
		aeadctx->mayverify = VERIFY_HW;
3280
		break;
3281
	case ICV_12:
3282
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3283
		aeadctx->mayverify = VERIFY_HW;
3284
		break;
3285
	case ICV_16:
3286
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3287
		aeadctx->mayverify = VERIFY_HW;
3288
		break;
3289
	default:
3290
		return -EINVAL;
3291
	}
3292
	return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3293
}
3294

3295
static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3296
				unsigned int authsize)
3297
{
3298
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3299

3300
	switch (authsize) {
3301
	case ICV_4:
3302
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3303
		aeadctx->mayverify = VERIFY_HW;
3304
		break;
3305
	case ICV_6:
3306
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3307
		aeadctx->mayverify = VERIFY_HW;
3308
		break;
3309
	case ICV_8:
3310
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3311
		aeadctx->mayverify = VERIFY_HW;
3312
		break;
3313
	case ICV_10:
3314
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3315
		aeadctx->mayverify = VERIFY_HW;
3316
		break;
3317
	case ICV_12:
3318
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3319
		aeadctx->mayverify = VERIFY_HW;
3320
		break;
3321
	case ICV_14:
3322
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3323
		aeadctx->mayverify = VERIFY_HW;
3324
		break;
3325
	case ICV_16:
3326
		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3327
		aeadctx->mayverify = VERIFY_HW;
3328
		break;
3329
	default:
3330
		return -EINVAL;
3331
	}
3332
	return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3333
}
3334

3335
static int chcr_ccm_common_setkey(struct crypto_aead *aead,
3336
				const u8 *key,
3337
				unsigned int keylen)
3338
{
3339
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3340
	unsigned char ck_size, mk_size;
3341
	int key_ctx_size = 0;
3342

3343
	key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2;
3344
	if (keylen == AES_KEYSIZE_128) {
3345
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3346
		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
3347
	} else if (keylen == AES_KEYSIZE_192) {
3348
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3349
		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3350
	} else if (keylen == AES_KEYSIZE_256) {
3351
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3352
		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3353
	} else {
3354
		aeadctx->enckey_len = 0;
3355
		return	-EINVAL;
3356
	}
3357
	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3358
						key_ctx_size >> 4);
3359
	memcpy(aeadctx->key, key, keylen);
3360
	aeadctx->enckey_len = keylen;
3361

3362
	return 0;
3363
}
3364

3365
static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3366
				const u8 *key,
3367
				unsigned int keylen)
3368
{
3369
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3370
	int error;
3371

3372
	crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3373
	crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3374
			      CRYPTO_TFM_REQ_MASK);
3375
	error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3376
	if (error)
3377
		return error;
3378
	return chcr_ccm_common_setkey(aead, key, keylen);
3379
}
3380

3381
static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3382
				    unsigned int keylen)
3383
{
3384
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3385
	int error;
3386

3387
	if (keylen < 3) {
3388
		aeadctx->enckey_len = 0;
3389
		return	-EINVAL;
3390
	}
3391
	crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3392
	crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3393
			      CRYPTO_TFM_REQ_MASK);
3394
	error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3395
	if (error)
3396
		return error;
3397
	keylen -= 3;
3398
	memcpy(aeadctx->salt, key + keylen, 3);
3399
	return chcr_ccm_common_setkey(aead, key, keylen);
3400
}
3401

3402
static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3403
			   unsigned int keylen)
3404
{
3405
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3406
	struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
3407
	unsigned int ck_size;
3408
	int ret = 0, key_ctx_size = 0;
3409
	struct crypto_aes_ctx aes;
3410

3411
	aeadctx->enckey_len = 0;
3412
	crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3413
	crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3414
			      & CRYPTO_TFM_REQ_MASK);
3415
	ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3416
	if (ret)
3417
		goto out;
3418

3419
	if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3420
	    keylen > 3) {
3421
		keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
3422
		memcpy(aeadctx->salt, key + keylen, 4);
3423
	}
3424
	if (keylen == AES_KEYSIZE_128) {
3425
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3426
	} else if (keylen == AES_KEYSIZE_192) {
3427
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3428
	} else if (keylen == AES_KEYSIZE_256) {
3429
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3430
	} else {
3431
		pr_err("GCM: Invalid key length %d\n", keylen);
3432
		ret = -EINVAL;
3433
		goto out;
3434
	}
3435

3436
	memcpy(aeadctx->key, key, keylen);
3437
	aeadctx->enckey_len = keylen;
3438
	key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) +
3439
		AEAD_H_SIZE;
3440
	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
3441
						CHCR_KEYCTX_MAC_KEY_SIZE_128,
3442
						0, 0,
3443
						key_ctx_size >> 4);
3444
	/* Calculate the H = CIPH(K, 0 repeated 16 times).
3445
	 * It will go in key context
3446
	 */
3447
	ret = aes_expandkey(&aes, key, keylen);
3448
	if (ret) {
3449
		aeadctx->enckey_len = 0;
3450
		goto out;
3451
	}
3452
	memset(gctx->ghash_h, 0, AEAD_H_SIZE);
3453
	aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h);
3454
	memzero_explicit(&aes, sizeof(aes));
3455

3456
out:
3457
	return ret;
3458
}
3459

3460
static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3461
				   unsigned int keylen)
3462
{
3463
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3464
	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3465
	/* it contains auth and cipher key both*/
3466
	struct crypto_authenc_keys keys;
3467
	unsigned int subtype;
3468
	unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3469
	int err = 0, key_ctx_len = 0;
3470
	unsigned char ck_size = 0;
3471
	struct algo_param param;
3472
	int align;
3473

3474
	crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3475
	crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3476
			      & CRYPTO_TFM_REQ_MASK);
3477
	err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3478
	if (err)
3479
		goto out;
3480

3481
	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3482
		goto out;
3483

3484
	if (get_alg_config(&param, max_authsize)) {
3485
		pr_err("Unsupported digest size\n");
3486
		goto out;
3487
	}
3488
	subtype = get_aead_subtype(authenc);
3489
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3490
		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3491
		if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3492
			goto out;
3493
		memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3494
		- CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3495
		keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3496
	}
3497
	if (keys.enckeylen == AES_KEYSIZE_128) {
3498
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3499
	} else if (keys.enckeylen == AES_KEYSIZE_192) {
3500
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3501
	} else if (keys.enckeylen == AES_KEYSIZE_256) {
3502
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3503
	} else {
3504
		pr_err("Unsupported cipher key\n");
3505
		goto out;
3506
	}
3507

3508
	/* Copy only encryption key. We use authkey to generate h(ipad) and
3509
	 * h(opad) so authkey is not needed again. authkeylen size have the
3510
	 * size of the hash digest size.
3511
	 */
3512
	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3513
	aeadctx->enckey_len = keys.enckeylen;
3514
	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3515
		subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3516

3517
		get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3518
			    aeadctx->enckey_len << 3);
3519
	}
3520

3521
	align = KEYCTX_ALIGN_PAD(max_authsize);
3522
	err = chcr_prepare_hmac_key(keys.authkey, keys.authkeylen, max_authsize,
3523
				    actx->h_iopad,
3524
				    actx->h_iopad + param.result_size + align);
3525
	if (err)
3526
		goto out;
3527

3528
	key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16) +
3529
		      (param.result_size + align) * 2;
3530
	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 0, 1,
3531
						key_ctx_len >> 4);
3532
	actx->auth_mode = param.auth_mode;
3533

3534
	memzero_explicit(&keys, sizeof(keys));
3535
	return 0;
3536

3537
out:
3538
	aeadctx->enckey_len = 0;
3539
	memzero_explicit(&keys, sizeof(keys));
3540
	return -EINVAL;
3541
}
3542

3543
static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3544
					const u8 *key, unsigned int keylen)
3545
{
3546
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3547
	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3548
	struct crypto_authenc_keys keys;
3549
	int err;
3550
	/* it contains auth and cipher key both*/
3551
	unsigned int subtype;
3552
	int key_ctx_len = 0;
3553
	unsigned char ck_size = 0;
3554

3555
	crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3556
	crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3557
			      & CRYPTO_TFM_REQ_MASK);
3558
	err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3559
	if (err)
3560
		goto out;
3561

3562
	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3563
		goto out;
3564

3565
	subtype = get_aead_subtype(authenc);
3566
	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3567
	    subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3568
		if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3569
			goto out;
3570
		memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3571
			- CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3572
		keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3573
	}
3574
	if (keys.enckeylen == AES_KEYSIZE_128) {
3575
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3576
	} else if (keys.enckeylen == AES_KEYSIZE_192) {
3577
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3578
	} else if (keys.enckeylen == AES_KEYSIZE_256) {
3579
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3580
	} else {
3581
		pr_err("Unsupported cipher key %d\n", keys.enckeylen);
3582
		goto out;
3583
	}
3584
	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3585
	aeadctx->enckey_len = keys.enckeylen;
3586
	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3587
	    subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3588
		get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3589
				aeadctx->enckey_len << 3);
3590
	}
3591
	key_ctx_len =  sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16);
3592

3593
	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3594
						0, key_ctx_len >> 4);
3595
	actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3596
	memzero_explicit(&keys, sizeof(keys));
3597
	return 0;
3598
out:
3599
	aeadctx->enckey_len = 0;
3600
	memzero_explicit(&keys, sizeof(keys));
3601
	return -EINVAL;
3602
}
3603

3604
static int chcr_aead_op(struct aead_request *req,
3605
			int size,
3606
			create_wr_t create_wr_fn)
3607
{
3608
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3609
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3610
	struct chcr_context *ctx = a_ctx(tfm);
3611
	struct uld_ctx *u_ctx = ULD_CTX(ctx);
3612
	struct sk_buff *skb;
3613
	struct chcr_dev *cdev;
3614

3615
	cdev = a_ctx(tfm)->dev;
3616
	if (!cdev) {
3617
		pr_err("%s : No crypto device.\n", __func__);
3618
		return -ENXIO;
3619
	}
3620

3621
	if (chcr_inc_wrcount(cdev)) {
3622
	/* Detach state for CHCR means lldi or padap is freed.
3623
	 * We cannot increment fallback here.
3624
	 */
3625
		return chcr_aead_fallback(req, reqctx->op);
3626
	}
3627

3628
	if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3629
					reqctx->txqidx) &&
3630
		(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
3631
			chcr_dec_wrcount(cdev);
3632
			return -ENOSPC;
3633
	}
3634

3635
	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3636
	    crypto_ipsec_check_assoclen(req->assoclen) != 0) {
3637
		pr_err("RFC4106: Invalid value of assoclen %d\n",
3638
		       req->assoclen);
3639
		return -EINVAL;
3640
	}
3641

3642
	/* Form a WR from req */
3643
	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size);
3644

3645
	if (IS_ERR_OR_NULL(skb)) {
3646
		chcr_dec_wrcount(cdev);
3647
		return PTR_ERR_OR_ZERO(skb);
3648
	}
3649

3650
	skb->dev = u_ctx->lldi.ports[0];
3651
	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
3652
	chcr_send_wr(skb);
3653
	return -EINPROGRESS;
3654
}
3655

3656
static int chcr_aead_encrypt(struct aead_request *req)
3657
{
3658
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3659
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3660
	struct chcr_context *ctx = a_ctx(tfm);
3661
	unsigned int cpu;
3662

3663
	cpu = get_cpu();
3664
	reqctx->txqidx = cpu % ctx->ntxq;
3665
	reqctx->rxqidx = cpu % ctx->nrxq;
3666
	put_cpu();
3667

3668
	reqctx->verify = VERIFY_HW;
3669
	reqctx->op = CHCR_ENCRYPT_OP;
3670

3671
	switch (get_aead_subtype(tfm)) {
3672
	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3673
	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3674
	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3675
	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3676
		return chcr_aead_op(req, 0, create_authenc_wr);
3677
	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3678
	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3679
		return chcr_aead_op(req, 0, create_aead_ccm_wr);
3680
	default:
3681
		return chcr_aead_op(req, 0, create_gcm_wr);
3682
	}
3683
}
3684

3685
static int chcr_aead_decrypt(struct aead_request *req)
3686
{
3687
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3688
	struct chcr_context *ctx = a_ctx(tfm);
3689
	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3690
	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3691
	int size;
3692
	unsigned int cpu;
3693

3694
	cpu = get_cpu();
3695
	reqctx->txqidx = cpu % ctx->ntxq;
3696
	reqctx->rxqidx = cpu % ctx->nrxq;
3697
	put_cpu();
3698

3699
	if (aeadctx->mayverify == VERIFY_SW) {
3700
		size = crypto_aead_maxauthsize(tfm);
3701
		reqctx->verify = VERIFY_SW;
3702
	} else {
3703
		size = 0;
3704
		reqctx->verify = VERIFY_HW;
3705
	}
3706
	reqctx->op = CHCR_DECRYPT_OP;
3707
	switch (get_aead_subtype(tfm)) {
3708
	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3709
	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3710
	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3711
	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3712
		return chcr_aead_op(req, size, create_authenc_wr);
3713
	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3714
	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3715
		return chcr_aead_op(req, size, create_aead_ccm_wr);
3716
	default:
3717
		return chcr_aead_op(req, size, create_gcm_wr);
3718
	}
3719
}
3720

3721
static struct chcr_alg_template driver_algs[] = {
3722
	/* AES-CBC */
3723
	{
3724
		.type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3725
		.is_registered = 0,
3726
		.alg.skcipher = {
3727
			.base.cra_name		= "cbc(aes)",
3728
			.base.cra_driver_name	= "cbc-aes-chcr",
3729
			.base.cra_blocksize	= AES_BLOCK_SIZE,
3730

3731
			.init			= chcr_init_tfm,
3732
			.exit			= chcr_exit_tfm,
3733
			.min_keysize		= AES_MIN_KEY_SIZE,
3734
			.max_keysize		= AES_MAX_KEY_SIZE,
3735
			.ivsize			= AES_BLOCK_SIZE,
3736
			.setkey			= chcr_aes_cbc_setkey,
3737
			.encrypt		= chcr_aes_encrypt,
3738
			.decrypt		= chcr_aes_decrypt,
3739
			}
3740
	},
3741
	{
3742
		.type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3743
		.is_registered = 0,
3744
		.alg.skcipher = {
3745
			.base.cra_name		= "xts(aes)",
3746
			.base.cra_driver_name	= "xts-aes-chcr",
3747
			.base.cra_blocksize	= AES_BLOCK_SIZE,
3748

3749
			.init			= chcr_init_tfm,
3750
			.exit			= chcr_exit_tfm,
3751
			.min_keysize		= 2 * AES_MIN_KEY_SIZE,
3752
			.max_keysize		= 2 * AES_MAX_KEY_SIZE,
3753
			.ivsize			= AES_BLOCK_SIZE,
3754
			.setkey			= chcr_aes_xts_setkey,
3755
			.encrypt		= chcr_aes_encrypt,
3756
			.decrypt		= chcr_aes_decrypt,
3757
			}
3758
	},
3759
	{
3760
		.type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3761
		.is_registered = 0,
3762
		.alg.skcipher = {
3763
			.base.cra_name		= "ctr(aes)",
3764
			.base.cra_driver_name	= "ctr-aes-chcr",
3765
			.base.cra_blocksize	= 1,
3766

3767
			.init			= chcr_init_tfm,
3768
			.exit			= chcr_exit_tfm,
3769
			.min_keysize		= AES_MIN_KEY_SIZE,
3770
			.max_keysize		= AES_MAX_KEY_SIZE,
3771
			.ivsize			= AES_BLOCK_SIZE,
3772
			.setkey			= chcr_aes_ctr_setkey,
3773
			.encrypt		= chcr_aes_encrypt,
3774
			.decrypt		= chcr_aes_decrypt,
3775
		}
3776
	},
3777
	{
3778
		.type = CRYPTO_ALG_TYPE_SKCIPHER |
3779
			CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3780
		.is_registered = 0,
3781
		.alg.skcipher = {
3782
			.base.cra_name		= "rfc3686(ctr(aes))",
3783
			.base.cra_driver_name	= "rfc3686-ctr-aes-chcr",
3784
			.base.cra_blocksize	= 1,
3785

3786
			.init			= chcr_rfc3686_init,
3787
			.exit			= chcr_exit_tfm,
3788
			.min_keysize		= AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3789
			.max_keysize		= AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3790
			.ivsize			= CTR_RFC3686_IV_SIZE,
3791
			.setkey			= chcr_aes_rfc3686_setkey,
3792
			.encrypt		= chcr_aes_encrypt,
3793
			.decrypt		= chcr_aes_decrypt,
3794
		}
3795
	},
3796
	/* SHA */
3797
	{
3798
		.type = CRYPTO_ALG_TYPE_AHASH,
3799
		.is_registered = 0,
3800
		.alg.hash = {
3801
			.halg.digestsize = SHA1_DIGEST_SIZE,
3802
			.halg.base = {
3803
				.cra_name = "sha1",
3804
				.cra_driver_name = "sha1-chcr",
3805
				.cra_blocksize = SHA1_BLOCK_SIZE,
3806
			}
3807
		}
3808
	},
3809
	{
3810
		.type = CRYPTO_ALG_TYPE_AHASH,
3811
		.is_registered = 0,
3812
		.alg.hash = {
3813
			.halg.digestsize = SHA256_DIGEST_SIZE,
3814
			.halg.base = {
3815
				.cra_name = "sha256",
3816
				.cra_driver_name = "sha256-chcr",
3817
				.cra_blocksize = SHA256_BLOCK_SIZE,
3818
			}
3819
		}
3820
	},
3821
	{
3822
		.type = CRYPTO_ALG_TYPE_AHASH,
3823
		.is_registered = 0,
3824
		.alg.hash = {
3825
			.halg.digestsize = SHA224_DIGEST_SIZE,
3826
			.halg.base = {
3827
				.cra_name = "sha224",
3828
				.cra_driver_name = "sha224-chcr",
3829
				.cra_blocksize = SHA224_BLOCK_SIZE,
3830
			}
3831
		}
3832
	},
3833
	{
3834
		.type = CRYPTO_ALG_TYPE_AHASH,
3835
		.is_registered = 0,
3836
		.alg.hash = {
3837
			.halg.digestsize = SHA384_DIGEST_SIZE,
3838
			.halg.base = {
3839
				.cra_name = "sha384",
3840
				.cra_driver_name = "sha384-chcr",
3841
				.cra_blocksize = SHA384_BLOCK_SIZE,
3842
			}
3843
		}
3844
	},
3845
	{
3846
		.type = CRYPTO_ALG_TYPE_AHASH,
3847
		.is_registered = 0,
3848
		.alg.hash = {
3849
			.halg.digestsize = SHA512_DIGEST_SIZE,
3850
			.halg.base = {
3851
				.cra_name = "sha512",
3852
				.cra_driver_name = "sha512-chcr",
3853
				.cra_blocksize = SHA512_BLOCK_SIZE,
3854
			}
3855
		}
3856
	},
3857
	/* HMAC */
3858
	{
3859
		.type = CRYPTO_ALG_TYPE_HMAC,
3860
		.is_registered = 0,
3861
		.alg.hash = {
3862
			.halg.digestsize = SHA1_DIGEST_SIZE,
3863
			.halg.base = {
3864
				.cra_name = "hmac(sha1)",
3865
				.cra_driver_name = "hmac-sha1-chcr",
3866
				.cra_blocksize = SHA1_BLOCK_SIZE,
3867
			}
3868
		}
3869
	},
3870
	{
3871
		.type = CRYPTO_ALG_TYPE_HMAC,
3872
		.is_registered = 0,
3873
		.alg.hash = {
3874
			.halg.digestsize = SHA224_DIGEST_SIZE,
3875
			.halg.base = {
3876
				.cra_name = "hmac(sha224)",
3877
				.cra_driver_name = "hmac-sha224-chcr",
3878
				.cra_blocksize = SHA224_BLOCK_SIZE,
3879
			}
3880
		}
3881
	},
3882
	{
3883
		.type = CRYPTO_ALG_TYPE_HMAC,
3884
		.is_registered = 0,
3885
		.alg.hash = {
3886
			.halg.digestsize = SHA256_DIGEST_SIZE,
3887
			.halg.base = {
3888
				.cra_name = "hmac(sha256)",
3889
				.cra_driver_name = "hmac-sha256-chcr",
3890
				.cra_blocksize = SHA256_BLOCK_SIZE,
3891
			}
3892
		}
3893
	},
3894
	{
3895
		.type = CRYPTO_ALG_TYPE_HMAC,
3896
		.is_registered = 0,
3897
		.alg.hash = {
3898
			.halg.digestsize = SHA384_DIGEST_SIZE,
3899
			.halg.base = {
3900
				.cra_name = "hmac(sha384)",
3901
				.cra_driver_name = "hmac-sha384-chcr",
3902
				.cra_blocksize = SHA384_BLOCK_SIZE,
3903
			}
3904
		}
3905
	},
3906
	{
3907
		.type = CRYPTO_ALG_TYPE_HMAC,
3908
		.is_registered = 0,
3909
		.alg.hash = {
3910
			.halg.digestsize = SHA512_DIGEST_SIZE,
3911
			.halg.base = {
3912
				.cra_name = "hmac(sha512)",
3913
				.cra_driver_name = "hmac-sha512-chcr",
3914
				.cra_blocksize = SHA512_BLOCK_SIZE,
3915
			}
3916
		}
3917
	},
3918
	/* Add AEAD Algorithms */
3919
	{
3920
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3921
		.is_registered = 0,
3922
		.alg.aead = {
3923
			.base = {
3924
				.cra_name = "gcm(aes)",
3925
				.cra_driver_name = "gcm-aes-chcr",
3926
				.cra_blocksize	= 1,
3927
				.cra_priority = CHCR_AEAD_PRIORITY,
3928
				.cra_ctxsize =	sizeof(struct chcr_context) +
3929
						sizeof(struct chcr_aead_ctx) +
3930
						sizeof(struct chcr_gcm_ctx),
3931
			},
3932
			.ivsize = GCM_AES_IV_SIZE,
3933
			.maxauthsize = GHASH_DIGEST_SIZE,
3934
			.setkey = chcr_gcm_setkey,
3935
			.setauthsize = chcr_gcm_setauthsize,
3936
		}
3937
	},
3938
	{
3939
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3940
		.is_registered = 0,
3941
		.alg.aead = {
3942
			.base = {
3943
				.cra_name = "rfc4106(gcm(aes))",
3944
				.cra_driver_name = "rfc4106-gcm-aes-chcr",
3945
				.cra_blocksize	 = 1,
3946
				.cra_priority = CHCR_AEAD_PRIORITY + 1,
3947
				.cra_ctxsize =	sizeof(struct chcr_context) +
3948
						sizeof(struct chcr_aead_ctx) +
3949
						sizeof(struct chcr_gcm_ctx),
3950

3951
			},
3952
			.ivsize = GCM_RFC4106_IV_SIZE,
3953
			.maxauthsize	= GHASH_DIGEST_SIZE,
3954
			.setkey = chcr_gcm_setkey,
3955
			.setauthsize	= chcr_4106_4309_setauthsize,
3956
		}
3957
	},
3958
	{
3959
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3960
		.is_registered = 0,
3961
		.alg.aead = {
3962
			.base = {
3963
				.cra_name = "ccm(aes)",
3964
				.cra_driver_name = "ccm-aes-chcr",
3965
				.cra_blocksize	 = 1,
3966
				.cra_priority = CHCR_AEAD_PRIORITY,
3967
				.cra_ctxsize =	sizeof(struct chcr_context) +
3968
						sizeof(struct chcr_aead_ctx),
3969

3970
			},
3971
			.ivsize = AES_BLOCK_SIZE,
3972
			.maxauthsize	= GHASH_DIGEST_SIZE,
3973
			.setkey = chcr_aead_ccm_setkey,
3974
			.setauthsize	= chcr_ccm_setauthsize,
3975
		}
3976
	},
3977
	{
3978
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3979
		.is_registered = 0,
3980
		.alg.aead = {
3981
			.base = {
3982
				.cra_name = "rfc4309(ccm(aes))",
3983
				.cra_driver_name = "rfc4309-ccm-aes-chcr",
3984
				.cra_blocksize	 = 1,
3985
				.cra_priority = CHCR_AEAD_PRIORITY + 1,
3986
				.cra_ctxsize =	sizeof(struct chcr_context) +
3987
						sizeof(struct chcr_aead_ctx),
3988

3989
			},
3990
			.ivsize = 8,
3991
			.maxauthsize	= GHASH_DIGEST_SIZE,
3992
			.setkey = chcr_aead_rfc4309_setkey,
3993
			.setauthsize = chcr_4106_4309_setauthsize,
3994
		}
3995
	},
3996
	{
3997
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3998
		.is_registered = 0,
3999
		.alg.aead = {
4000
			.base = {
4001
				.cra_name = "authenc(hmac(sha1),cbc(aes))",
4002
				.cra_driver_name =
4003
					"authenc-hmac-sha1-cbc-aes-chcr",
4004
				.cra_blocksize	 = AES_BLOCK_SIZE,
4005
				.cra_priority = CHCR_AEAD_PRIORITY,
4006
				.cra_ctxsize =	sizeof(struct chcr_context) +
4007
						sizeof(struct chcr_aead_ctx) +
4008
						sizeof(struct chcr_authenc_ctx),
4009

4010
			},
4011
			.ivsize = AES_BLOCK_SIZE,
4012
			.maxauthsize = SHA1_DIGEST_SIZE,
4013
			.setkey = chcr_authenc_setkey,
4014
			.setauthsize = chcr_authenc_setauthsize,
4015
		}
4016
	},
4017
	{
4018
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4019
		.is_registered = 0,
4020
		.alg.aead = {
4021
			.base = {
4022

4023
				.cra_name = "authenc(hmac(sha256),cbc(aes))",
4024
				.cra_driver_name =
4025
					"authenc-hmac-sha256-cbc-aes-chcr",
4026
				.cra_blocksize	 = AES_BLOCK_SIZE,
4027
				.cra_priority = CHCR_AEAD_PRIORITY,
4028
				.cra_ctxsize =	sizeof(struct chcr_context) +
4029
						sizeof(struct chcr_aead_ctx) +
4030
						sizeof(struct chcr_authenc_ctx),
4031

4032
			},
4033
			.ivsize = AES_BLOCK_SIZE,
4034
			.maxauthsize	= SHA256_DIGEST_SIZE,
4035
			.setkey = chcr_authenc_setkey,
4036
			.setauthsize = chcr_authenc_setauthsize,
4037
		}
4038
	},
4039
	{
4040
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4041
		.is_registered = 0,
4042
		.alg.aead = {
4043
			.base = {
4044
				.cra_name = "authenc(hmac(sha224),cbc(aes))",
4045
				.cra_driver_name =
4046
					"authenc-hmac-sha224-cbc-aes-chcr",
4047
				.cra_blocksize	 = AES_BLOCK_SIZE,
4048
				.cra_priority = CHCR_AEAD_PRIORITY,
4049
				.cra_ctxsize =	sizeof(struct chcr_context) +
4050
						sizeof(struct chcr_aead_ctx) +
4051
						sizeof(struct chcr_authenc_ctx),
4052
			},
4053
			.ivsize = AES_BLOCK_SIZE,
4054
			.maxauthsize = SHA224_DIGEST_SIZE,
4055
			.setkey = chcr_authenc_setkey,
4056
			.setauthsize = chcr_authenc_setauthsize,
4057
		}
4058
	},
4059
	{
4060
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4061
		.is_registered = 0,
4062
		.alg.aead = {
4063
			.base = {
4064
				.cra_name = "authenc(hmac(sha384),cbc(aes))",
4065
				.cra_driver_name =
4066
					"authenc-hmac-sha384-cbc-aes-chcr",
4067
				.cra_blocksize	 = AES_BLOCK_SIZE,
4068
				.cra_priority = CHCR_AEAD_PRIORITY,
4069
				.cra_ctxsize =	sizeof(struct chcr_context) +
4070
						sizeof(struct chcr_aead_ctx) +
4071
						sizeof(struct chcr_authenc_ctx),
4072

4073
			},
4074
			.ivsize = AES_BLOCK_SIZE,
4075
			.maxauthsize = SHA384_DIGEST_SIZE,
4076
			.setkey = chcr_authenc_setkey,
4077
			.setauthsize = chcr_authenc_setauthsize,
4078
		}
4079
	},
4080
	{
4081
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4082
		.is_registered = 0,
4083
		.alg.aead = {
4084
			.base = {
4085
				.cra_name = "authenc(hmac(sha512),cbc(aes))",
4086
				.cra_driver_name =
4087
					"authenc-hmac-sha512-cbc-aes-chcr",
4088
				.cra_blocksize	 = AES_BLOCK_SIZE,
4089
				.cra_priority = CHCR_AEAD_PRIORITY,
4090
				.cra_ctxsize =	sizeof(struct chcr_context) +
4091
						sizeof(struct chcr_aead_ctx) +
4092
						sizeof(struct chcr_authenc_ctx),
4093

4094
			},
4095
			.ivsize = AES_BLOCK_SIZE,
4096
			.maxauthsize = SHA512_DIGEST_SIZE,
4097
			.setkey = chcr_authenc_setkey,
4098
			.setauthsize = chcr_authenc_setauthsize,
4099
		}
4100
	},
4101
	{
4102
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
4103
		.is_registered = 0,
4104
		.alg.aead = {
4105
			.base = {
4106
				.cra_name = "authenc(digest_null,cbc(aes))",
4107
				.cra_driver_name =
4108
					"authenc-digest_null-cbc-aes-chcr",
4109
				.cra_blocksize	 = AES_BLOCK_SIZE,
4110
				.cra_priority = CHCR_AEAD_PRIORITY,
4111
				.cra_ctxsize =	sizeof(struct chcr_context) +
4112
						sizeof(struct chcr_aead_ctx) +
4113
						sizeof(struct chcr_authenc_ctx),
4114

4115
			},
4116
			.ivsize  = AES_BLOCK_SIZE,
4117
			.maxauthsize = 0,
4118
			.setkey  = chcr_aead_digest_null_setkey,
4119
			.setauthsize = chcr_authenc_null_setauthsize,
4120
		}
4121
	},
4122
	{
4123
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4124
		.is_registered = 0,
4125
		.alg.aead = {
4126
			.base = {
4127
				.cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4128
				.cra_driver_name =
4129
				"authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4130
				.cra_blocksize	 = 1,
4131
				.cra_priority = CHCR_AEAD_PRIORITY,
4132
				.cra_ctxsize =	sizeof(struct chcr_context) +
4133
						sizeof(struct chcr_aead_ctx) +
4134
						sizeof(struct chcr_authenc_ctx),
4135

4136
			},
4137
			.ivsize = CTR_RFC3686_IV_SIZE,
4138
			.maxauthsize = SHA1_DIGEST_SIZE,
4139
			.setkey = chcr_authenc_setkey,
4140
			.setauthsize = chcr_authenc_setauthsize,
4141
		}
4142
	},
4143
	{
4144
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4145
		.is_registered = 0,
4146
		.alg.aead = {
4147
			.base = {
4148

4149
				.cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4150
				.cra_driver_name =
4151
				"authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4152
				.cra_blocksize	 = 1,
4153
				.cra_priority = CHCR_AEAD_PRIORITY,
4154
				.cra_ctxsize =	sizeof(struct chcr_context) +
4155
						sizeof(struct chcr_aead_ctx) +
4156
						sizeof(struct chcr_authenc_ctx),
4157

4158
			},
4159
			.ivsize = CTR_RFC3686_IV_SIZE,
4160
			.maxauthsize	= SHA256_DIGEST_SIZE,
4161
			.setkey = chcr_authenc_setkey,
4162
			.setauthsize = chcr_authenc_setauthsize,
4163
		}
4164
	},
4165
	{
4166
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4167
		.is_registered = 0,
4168
		.alg.aead = {
4169
			.base = {
4170
				.cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4171
				.cra_driver_name =
4172
				"authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4173
				.cra_blocksize	 = 1,
4174
				.cra_priority = CHCR_AEAD_PRIORITY,
4175
				.cra_ctxsize =	sizeof(struct chcr_context) +
4176
						sizeof(struct chcr_aead_ctx) +
4177
						sizeof(struct chcr_authenc_ctx),
4178
			},
4179
			.ivsize = CTR_RFC3686_IV_SIZE,
4180
			.maxauthsize = SHA224_DIGEST_SIZE,
4181
			.setkey = chcr_authenc_setkey,
4182
			.setauthsize = chcr_authenc_setauthsize,
4183
		}
4184
	},
4185
	{
4186
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4187
		.is_registered = 0,
4188
		.alg.aead = {
4189
			.base = {
4190
				.cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4191
				.cra_driver_name =
4192
				"authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4193
				.cra_blocksize	 = 1,
4194
				.cra_priority = CHCR_AEAD_PRIORITY,
4195
				.cra_ctxsize =	sizeof(struct chcr_context) +
4196
						sizeof(struct chcr_aead_ctx) +
4197
						sizeof(struct chcr_authenc_ctx),
4198

4199
			},
4200
			.ivsize = CTR_RFC3686_IV_SIZE,
4201
			.maxauthsize = SHA384_DIGEST_SIZE,
4202
			.setkey = chcr_authenc_setkey,
4203
			.setauthsize = chcr_authenc_setauthsize,
4204
		}
4205
	},
4206
	{
4207
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4208
		.is_registered = 0,
4209
		.alg.aead = {
4210
			.base = {
4211
				.cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4212
				.cra_driver_name =
4213
				"authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4214
				.cra_blocksize	 = 1,
4215
				.cra_priority = CHCR_AEAD_PRIORITY,
4216
				.cra_ctxsize =	sizeof(struct chcr_context) +
4217
						sizeof(struct chcr_aead_ctx) +
4218
						sizeof(struct chcr_authenc_ctx),
4219

4220
			},
4221
			.ivsize = CTR_RFC3686_IV_SIZE,
4222
			.maxauthsize = SHA512_DIGEST_SIZE,
4223
			.setkey = chcr_authenc_setkey,
4224
			.setauthsize = chcr_authenc_setauthsize,
4225
		}
4226
	},
4227
	{
4228
		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4229
		.is_registered = 0,
4230
		.alg.aead = {
4231
			.base = {
4232
				.cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4233
				.cra_driver_name =
4234
				"authenc-digest_null-rfc3686-ctr-aes-chcr",
4235
				.cra_blocksize	 = 1,
4236
				.cra_priority = CHCR_AEAD_PRIORITY,
4237
				.cra_ctxsize =	sizeof(struct chcr_context) +
4238
						sizeof(struct chcr_aead_ctx) +
4239
						sizeof(struct chcr_authenc_ctx),
4240

4241
			},
4242
			.ivsize  = CTR_RFC3686_IV_SIZE,
4243
			.maxauthsize = 0,
4244
			.setkey  = chcr_aead_digest_null_setkey,
4245
			.setauthsize = chcr_authenc_null_setauthsize,
4246
		}
4247
	},
4248
};
4249

4250
/*
4251
 *	chcr_unregister_alg - Deregister crypto algorithms with
4252
 *	kernel framework.
4253
 */
4254
static int chcr_unregister_alg(void)
4255
{
4256
	int i;
4257

4258
	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4259
		switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4260
		case CRYPTO_ALG_TYPE_SKCIPHER:
4261
			if (driver_algs[i].is_registered && refcount_read(
4262
			    &driver_algs[i].alg.skcipher.base.cra_refcnt)
4263
			    == 1) {
4264
				crypto_unregister_skcipher(
4265
						&driver_algs[i].alg.skcipher);
4266
				driver_algs[i].is_registered = 0;
4267
			}
4268
			break;
4269
		case CRYPTO_ALG_TYPE_AEAD:
4270
			if (driver_algs[i].is_registered && refcount_read(
4271
			    &driver_algs[i].alg.aead.base.cra_refcnt) == 1) {
4272
				crypto_unregister_aead(
4273
						&driver_algs[i].alg.aead);
4274
				driver_algs[i].is_registered = 0;
4275
			}
4276
			break;
4277
		case CRYPTO_ALG_TYPE_AHASH:
4278
			if (driver_algs[i].is_registered && refcount_read(
4279
			    &driver_algs[i].alg.hash.halg.base.cra_refcnt)
4280
			    == 1) {
4281
				crypto_unregister_ahash(
4282
						&driver_algs[i].alg.hash);
4283
				driver_algs[i].is_registered = 0;
4284
			}
4285
			break;
4286
		}
4287
	}
4288
	return 0;
4289
}
4290

4291
#define SZ_AHASH_CTX sizeof(struct chcr_context)
4292
#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4293
#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
4294

4295
/*
4296
 *	chcr_register_alg - Register crypto algorithms with kernel framework.
4297
 */
4298
static int chcr_register_alg(void)
4299
{
4300
	struct crypto_alg ai;
4301
	struct ahash_alg *a_hash;
4302
	int err = 0, i;
4303
	char *name = NULL;
4304

4305
	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4306
		if (driver_algs[i].is_registered)
4307
			continue;
4308
		switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4309
		case CRYPTO_ALG_TYPE_SKCIPHER:
4310
			driver_algs[i].alg.skcipher.base.cra_priority =
4311
				CHCR_CRA_PRIORITY;
4312
			driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE;
4313
			driver_algs[i].alg.skcipher.base.cra_flags =
4314
				CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
4315
				CRYPTO_ALG_ALLOCATES_MEMORY |
4316
				CRYPTO_ALG_NEED_FALLBACK;
4317
			driver_algs[i].alg.skcipher.base.cra_ctxsize =
4318
				sizeof(struct chcr_context) +
4319
				sizeof(struct ablk_ctx);
4320
			driver_algs[i].alg.skcipher.base.cra_alignmask = 0;
4321

4322
			err = crypto_register_skcipher(&driver_algs[i].alg.skcipher);
4323
			name = driver_algs[i].alg.skcipher.base.cra_driver_name;
4324
			break;
4325
		case CRYPTO_ALG_TYPE_AEAD:
4326
			driver_algs[i].alg.aead.base.cra_flags =
4327
				CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
4328
				CRYPTO_ALG_ALLOCATES_MEMORY;
4329
			driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4330
			driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4331
			driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4332
			driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4333
			driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4334
			err = crypto_register_aead(&driver_algs[i].alg.aead);
4335
			name = driver_algs[i].alg.aead.base.cra_driver_name;
4336
			break;
4337
		case CRYPTO_ALG_TYPE_AHASH:
4338
			a_hash = &driver_algs[i].alg.hash;
4339
			a_hash->update = chcr_ahash_update;
4340
			a_hash->final = chcr_ahash_final;
4341
			a_hash->finup = chcr_ahash_finup;
4342
			a_hash->digest = chcr_ahash_digest;
4343
			a_hash->export = chcr_ahash_export;
4344
			a_hash->import = chcr_ahash_import;
4345
			a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4346
			a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4347
			a_hash->halg.base.cra_module = THIS_MODULE;
4348
			a_hash->halg.base.cra_flags =
4349
				CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4350
			a_hash->halg.base.cra_alignmask = 0;
4351
			a_hash->halg.base.cra_exit = NULL;
4352

4353
			if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4354
				a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4355
				a_hash->init = chcr_hmac_init;
4356
				a_hash->setkey = chcr_ahash_setkey;
4357
				a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4358
			} else {
4359
				a_hash->init = chcr_sha_init;
4360
				a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4361
				a_hash->halg.base.cra_init = chcr_sha_cra_init;
4362
			}
4363
			err = crypto_register_ahash(&driver_algs[i].alg.hash);
4364
			ai = driver_algs[i].alg.hash.halg.base;
4365
			name = ai.cra_driver_name;
4366
			break;
4367
		}
4368
		if (err) {
4369
			pr_err("%s : Algorithm registration failed\n", name);
4370
			goto register_err;
4371
		} else {
4372
			driver_algs[i].is_registered = 1;
4373
		}
4374
	}
4375
	return 0;
4376

4377
register_err:
4378
	chcr_unregister_alg();
4379
	return err;
4380
}
4381

4382
/*
4383
 *	start_crypto - Register the crypto algorithms.
4384
 *	This should called once when the first device comesup. After this
4385
 *	kernel will start calling driver APIs for crypto operations.
4386
 */
4387
int start_crypto(void)
4388
{
4389
	return chcr_register_alg();
4390
}
4391

4392
/*
4393
 *	stop_crypto - Deregister all the crypto algorithms with kernel.
4394
 *	This should be called once when the last device goes down. After this
4395
 *	kernel will not call the driver API for crypto operations.
4396
 */
4397
int stop_crypto(void)
4398
{
4399
	chcr_unregister_alg();
4400
	return 0;
4401
}
4402

4403
Product

Resources

Company
