1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
 * Cryptographic API.
4
 *
5
 * Support for OMAP AES HW acceleration.
6
 *
7
 * Copyright (c) 2010 Nokia Corporation
8
 * Author: Dmitry Kasatkin <[email protected]>
9
 * Copyright (c) 2011 Texas Instruments Incorporated
10
 */
11

12
#define pr_fmt(fmt) "%20s: " fmt, __func__
13
#define prn(num) pr_debug(#num "=%d\n", num)
14
#define prx(num) pr_debug(#num "=%x\n", num)
15

16
#include <crypto/aes.h>
17
#include <crypto/gcm.h>
18
#include <crypto/internal/aead.h>
19
#include <crypto/internal/engine.h>
20
#include <crypto/internal/skcipher.h>
21
#include <linux/dma-mapping.h>
22
#include <linux/dmaengine.h>
23
#include <linux/err.h>
24
#include <linux/init.h>
25
#include <linux/interrupt.h>
26
#include <linux/io.h>
27
#include <linux/kernel.h>
28
#include <linux/module.h>
29
#include <linux/of.h>
30
#include <linux/of_address.h>
31
#include <linux/platform_device.h>
32
#include <linux/pm_runtime.h>
33
#include <linux/scatterlist.h>
34
#include <linux/string.h>
35
#include <linux/workqueue.h>
36

37
#include "omap-crypto.h"
38
#include "omap-aes.h"
39

40
/* keep registered devices data here */
41
static LIST_HEAD(dev_list);
42
static DEFINE_SPINLOCK(list_lock);
43

44
static int aes_fallback_sz = 200;
45

46
#ifdef DEBUG
47
#define omap_aes_read(dd, offset)				\
48
({								\
49
	int _read_ret;						\
50
	_read_ret = __raw_readl(dd->io_base + offset);		\
51
	pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n",	\
52
		 offset, _read_ret);				\
53
	_read_ret;						\
54
})
55
#else
56
inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
57
{
58
	return __raw_readl(dd->io_base + offset);
59
}
60
#endif
61

62
#ifdef DEBUG
63
#define omap_aes_write(dd, offset, value)				\
64
	do {								\
65
		pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n",	\
66
			 offset, value);				\
67
		__raw_writel(value, dd->io_base + offset);		\
68
	} while (0)
69
#else
70
inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
71
				  u32 value)
72
{
73
	__raw_writel(value, dd->io_base + offset);
74
}
75
#endif
76

77
static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
78
					u32 value, u32 mask)
79
{
80
	u32 val;
81

82
	val = omap_aes_read(dd, offset);
83
	val &= ~mask;
84
	val |= value;
85
	omap_aes_write(dd, offset, val);
86
}
87

88
static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
89
					u32 *value, int count)
90
{
91
	for (; count--; value++, offset += 4)
92
		omap_aes_write(dd, offset, *value);
93
}
94

95
static int omap_aes_hw_init(struct omap_aes_dev *dd)
96
{
97
	int err;
98

99
	if (!(dd->flags & FLAGS_INIT)) {
100
		dd->flags |= FLAGS_INIT;
101
		dd->err = 0;
102
	}
103

104
	err = pm_runtime_resume_and_get(dd->dev);
105
	if (err < 0) {
106
		dev_err(dd->dev, "failed to get sync: %d\n", err);
107
		return err;
108
	}
109

110
	return 0;
111
}
112

113
void omap_aes_clear_copy_flags(struct omap_aes_dev *dd)
114
{
115
	dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT);
116
	dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT);
117
	dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT);
118
}
119

120
int omap_aes_write_ctrl(struct omap_aes_dev *dd)
121
{
122
	struct omap_aes_reqctx *rctx;
123
	unsigned int key32;
124
	int i, err;
125
	u32 val;
126

127
	err = omap_aes_hw_init(dd);
128
	if (err)
129
		return err;
130

131
	key32 = dd->ctx->keylen / sizeof(u32);
132

133
	/* RESET the key as previous HASH keys should not get affected*/
134
	if (dd->flags & FLAGS_GCM)
135
		for (i = 0; i < 0x40; i = i + 4)
136
			omap_aes_write(dd, i, 0x0);
137

138
	for (i = 0; i < key32; i++) {
139
		omap_aes_write(dd, AES_REG_KEY(dd, i),
140
			       (__force u32)cpu_to_le32(dd->ctx->key[i]));
141
	}
142

143
	if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->iv)
144
		omap_aes_write_n(dd, AES_REG_IV(dd, 0), (void *)dd->req->iv, 4);
145

146
	if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) {
147
		rctx = aead_request_ctx(dd->aead_req);
148
		omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4);
149
	}
150

151
	val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
152
	if (dd->flags & FLAGS_CBC)
153
		val |= AES_REG_CTRL_CBC;
154

155
	if (dd->flags & (FLAGS_CTR | FLAGS_GCM))
156
		val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128;
157

158
	if (dd->flags & FLAGS_GCM)
159
		val |= AES_REG_CTRL_GCM;
160

161
	if (dd->flags & FLAGS_ENCRYPT)
162
		val |= AES_REG_CTRL_DIRECTION;
163

164
	omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK);
165

166
	return 0;
167
}
168

169
static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length)
170
{
171
	u32 mask, val;
172

173
	val = dd->pdata->dma_start;
174

175
	if (dd->dma_lch_out != NULL)
176
		val |= dd->pdata->dma_enable_out;
177
	if (dd->dma_lch_in != NULL)
178
		val |= dd->pdata->dma_enable_in;
179

180
	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
181
	       dd->pdata->dma_start;
182

183
	omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask);
184

185
}
186

187
static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)
188
{
189
	omap_aes_write(dd, AES_REG_LENGTH_N(0), length);
190
	omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);
191
	if (dd->flags & FLAGS_GCM)
192
		omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len);
193

194
	omap_aes_dma_trigger_omap2(dd, length);
195
}
196

197
static void omap_aes_dma_stop(struct omap_aes_dev *dd)
198
{
199
	u32 mask;
200

201
	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
202
	       dd->pdata->dma_start;
203

204
	omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);
205
}
206

207
struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx)
208
{
209
	struct omap_aes_dev *dd;
210

211
	spin_lock_bh(&list_lock);
212
	dd = list_first_entry(&dev_list, struct omap_aes_dev, list);
213
	list_move_tail(&dd->list, &dev_list);
214
	rctx->dd = dd;
215
	spin_unlock_bh(&list_lock);
216

217
	return dd;
218
}
219

220
static void omap_aes_dma_out_callback(void *data)
221
{
222
	struct omap_aes_dev *dd = data;
223

224
	/* dma_lch_out - completed */
225
	queue_work(system_bh_wq, &dd->done_task);
226
}
227

228
static int omap_aes_dma_init(struct omap_aes_dev *dd)
229
{
230
	int err;
231

232
	dd->dma_lch_out = NULL;
233
	dd->dma_lch_in = NULL;
234

235
	dd->dma_lch_in = dma_request_chan(dd->dev, "rx");
236
	if (IS_ERR(dd->dma_lch_in)) {
237
		dev_err(dd->dev, "Unable to request in DMA channel\n");
238
		return PTR_ERR(dd->dma_lch_in);
239
	}
240

241
	dd->dma_lch_out = dma_request_chan(dd->dev, "tx");
242
	if (IS_ERR(dd->dma_lch_out)) {
243
		dev_err(dd->dev, "Unable to request out DMA channel\n");
244
		err = PTR_ERR(dd->dma_lch_out);
245
		goto err_dma_out;
246
	}
247

248
	return 0;
249

250
err_dma_out:
251
	dma_release_channel(dd->dma_lch_in);
252

253
	return err;
254
}
255

256
static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
257
{
258
	if (dd->pio_only)
259
		return;
260

261
	dma_release_channel(dd->dma_lch_out);
262
	dma_release_channel(dd->dma_lch_in);
263
}
264

265
static int omap_aes_crypt_dma(struct omap_aes_dev *dd,
266
			      struct scatterlist *in_sg,
267
			      struct scatterlist *out_sg,
268
			      int in_sg_len, int out_sg_len)
269
{
270
	struct dma_async_tx_descriptor *tx_in, *tx_out = NULL, *cb_desc;
271
	struct dma_slave_config cfg;
272
	int ret;
273

274
	if (dd->pio_only) {
275
		dd->in_sg_offset = 0;
276
		if (out_sg_len)
277
			dd->out_sg_offset = 0;
278

279
		/* Enable DATAIN interrupt and let it take
280
		   care of the rest */
281
		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
282
		return 0;
283
	}
284

285
	dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
286

287
	memset(&cfg, 0, sizeof(cfg));
288

289
	cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
290
	cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
291
	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
292
	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
293
	cfg.src_maxburst = DST_MAXBURST;
294
	cfg.dst_maxburst = DST_MAXBURST;
295

296
	/* IN */
297
	ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
298
	if (ret) {
299
		dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
300
			ret);
301
		return ret;
302
	}
303

304
	tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
305
					DMA_MEM_TO_DEV,
306
					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
307
	if (!tx_in) {
308
		dev_err(dd->dev, "IN prep_slave_sg() failed\n");
309
		return -EINVAL;
310
	}
311

312
	/* No callback necessary */
313
	tx_in->callback_param = dd;
314
	tx_in->callback = NULL;
315

316
	/* OUT */
317
	if (out_sg_len) {
318
		ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
319
		if (ret) {
320
			dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
321
				ret);
322
			return ret;
323
		}
324

325
		tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg,
326
						 out_sg_len,
327
						 DMA_DEV_TO_MEM,
328
						 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
329
		if (!tx_out) {
330
			dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
331
			return -EINVAL;
332
		}
333

334
		cb_desc = tx_out;
335
	} else {
336
		cb_desc = tx_in;
337
	}
338

339
	if (dd->flags & FLAGS_GCM)
340
		cb_desc->callback = omap_aes_gcm_dma_out_callback;
341
	else
342
		cb_desc->callback = omap_aes_dma_out_callback;
343
	cb_desc->callback_param = dd;
344

345

346
	dmaengine_submit(tx_in);
347
	if (tx_out)
348
		dmaengine_submit(tx_out);
349

350
	dma_async_issue_pending(dd->dma_lch_in);
351
	if (out_sg_len)
352
		dma_async_issue_pending(dd->dma_lch_out);
353

354
	/* start DMA */
355
	dd->pdata->trigger(dd, dd->total);
356

357
	return 0;
358
}
359

360
int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
361
{
362
	int err;
363

364
	pr_debug("total: %zu\n", dd->total);
365

366
	if (!dd->pio_only) {
367
		err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
368
				 DMA_TO_DEVICE);
369
		if (!err) {
370
			dev_err(dd->dev, "dma_map_sg() error\n");
371
			return -EINVAL;
372
		}
373

374
		if (dd->out_sg_len) {
375
			err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
376
					 DMA_FROM_DEVICE);
377
			if (!err) {
378
				dev_err(dd->dev, "dma_map_sg() error\n");
379
				return -EINVAL;
380
			}
381
		}
382
	}
383

384
	err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len,
385
				 dd->out_sg_len);
386
	if (err && !dd->pio_only) {
387
		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
388
		if (dd->out_sg_len)
389
			dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
390
				     DMA_FROM_DEVICE);
391
	}
392

393
	return err;
394
}
395

396
static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
397
{
398
	struct skcipher_request *req = dd->req;
399

400
	pr_debug("err: %d\n", err);
401

402
	crypto_finalize_skcipher_request(dd->engine, req, err);
403

404
	pm_runtime_put_autosuspend(dd->dev);
405
}
406

407
int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
408
{
409
	pr_debug("total: %zu\n", dd->total);
410

411
	omap_aes_dma_stop(dd);
412

413

414
	return 0;
415
}
416

417
static int omap_aes_handle_queue(struct omap_aes_dev *dd,
418
				 struct skcipher_request *req)
419
{
420
	if (req)
421
		return crypto_transfer_skcipher_request_to_engine(dd->engine, req);
422

423
	return 0;
424
}
425

426
static int omap_aes_prepare_req(struct skcipher_request *req,
427
				struct omap_aes_dev *dd)
428
{
429
	struct omap_aes_ctx *ctx = crypto_skcipher_ctx(
430
			crypto_skcipher_reqtfm(req));
431
	struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
432
	int ret;
433
	u16 flags;
434

435
	/* assign new request to device */
436
	dd->req = req;
437
	dd->total = req->cryptlen;
438
	dd->total_save = req->cryptlen;
439
	dd->in_sg = req->src;
440
	dd->out_sg = req->dst;
441
	dd->orig_out = req->dst;
442

443
	flags = OMAP_CRYPTO_COPY_DATA;
444
	if (req->src == req->dst)
445
		flags |= OMAP_CRYPTO_FORCE_COPY;
446

447
	ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE,
448
				   dd->in_sgl, flags,
449
				   FLAGS_IN_DATA_ST_SHIFT, &dd->flags);
450
	if (ret)
451
		return ret;
452

453
	ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE,
454
				   &dd->out_sgl, 0,
455
				   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
456
	if (ret)
457
		return ret;
458

459
	dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
460
	if (dd->in_sg_len < 0)
461
		return dd->in_sg_len;
462

463
	dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);
464
	if (dd->out_sg_len < 0)
465
		return dd->out_sg_len;
466

467
	rctx->mode &= FLAGS_MODE_MASK;
468
	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
469

470
	dd->ctx = ctx;
471
	rctx->dd = dd;
472

473
	return omap_aes_write_ctrl(dd);
474
}
475

476
static int omap_aes_crypt_req(struct crypto_engine *engine,
477
			      void *areq)
478
{
479
	struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
480
	struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
481
	struct omap_aes_dev *dd = rctx->dd;
482

483
	if (!dd)
484
		return -ENODEV;
485

486
	return omap_aes_prepare_req(req, dd) ?:
487
	       omap_aes_crypt_dma_start(dd);
488
}
489

490
static void omap_aes_copy_ivout(struct omap_aes_dev *dd, u8 *ivbuf)
491
{
492
	int i;
493

494
	for (i = 0; i < 4; i++)
495
		((u32 *)ivbuf)[i] = omap_aes_read(dd, AES_REG_IV(dd, i));
496
}
497

498
static void omap_aes_done_task(struct work_struct *t)
499
{
500
	struct omap_aes_dev *dd = from_work(dd, t, done_task);
501

502
	pr_debug("enter done_task\n");
503

504
	if (!dd->pio_only) {
505
		dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
506
				       DMA_FROM_DEVICE);
507
		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
508
		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
509
			     DMA_FROM_DEVICE);
510
		omap_aes_crypt_dma_stop(dd);
511
	}
512

513
	omap_crypto_cleanup(dd->in_sg, NULL, 0, dd->total_save,
514
			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);
515

516
	omap_crypto_cleanup(dd->out_sg, dd->orig_out, 0, dd->total_save,
517
			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
518

519
	/* Update IV output */
520
	if (dd->flags & (FLAGS_CBC | FLAGS_CTR))
521
		omap_aes_copy_ivout(dd, dd->req->iv);
522

523
	omap_aes_finish_req(dd, 0);
524

525
	pr_debug("exit\n");
526
}
527

528
static int omap_aes_crypt(struct skcipher_request *req, unsigned long mode)
529
{
530
	struct omap_aes_ctx *ctx = crypto_skcipher_ctx(
531
			crypto_skcipher_reqtfm(req));
532
	struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
533
	struct omap_aes_dev *dd;
534
	int ret;
535

536
	if ((req->cryptlen % AES_BLOCK_SIZE) && !(mode & FLAGS_CTR))
537
		return -EINVAL;
538

539
	pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen,
540
		  !!(mode & FLAGS_ENCRYPT),
541
		  !!(mode & FLAGS_CBC));
542

543
	if (req->cryptlen < aes_fallback_sz) {
544
		skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
545
		skcipher_request_set_callback(&rctx->fallback_req,
546
					      req->base.flags,
547
					      req->base.complete,
548
					      req->base.data);
549
		skcipher_request_set_crypt(&rctx->fallback_req, req->src,
550
					   req->dst, req->cryptlen, req->iv);
551

552
		if (mode & FLAGS_ENCRYPT)
553
			ret = crypto_skcipher_encrypt(&rctx->fallback_req);
554
		else
555
			ret = crypto_skcipher_decrypt(&rctx->fallback_req);
556
		return ret;
557
	}
558
	dd = omap_aes_find_dev(rctx);
559
	if (!dd)
560
		return -ENODEV;
561

562
	rctx->mode = mode;
563

564
	return omap_aes_handle_queue(dd, req);
565
}
566

567
/* ********************** ALG API ************************************ */
568

569
static int omap_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
570
			   unsigned int keylen)
571
{
572
	struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
573
	int ret;
574

575
	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
576
		   keylen != AES_KEYSIZE_256)
577
		return -EINVAL;
578

579
	pr_debug("enter, keylen: %d\n", keylen);
580

581
	memcpy(ctx->key, key, keylen);
582
	ctx->keylen = keylen;
583

584
	crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
585
	crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
586
						 CRYPTO_TFM_REQ_MASK);
587

588
	ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
589
	if (!ret)
590
		return 0;
591

592
	return 0;
593
}
594

595
static int omap_aes_ecb_encrypt(struct skcipher_request *req)
596
{
597
	return omap_aes_crypt(req, FLAGS_ENCRYPT);
598
}
599

600
static int omap_aes_ecb_decrypt(struct skcipher_request *req)
601
{
602
	return omap_aes_crypt(req, 0);
603
}
604

605
static int omap_aes_cbc_encrypt(struct skcipher_request *req)
606
{
607
	return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
608
}
609

610
static int omap_aes_cbc_decrypt(struct skcipher_request *req)
611
{
612
	return omap_aes_crypt(req, FLAGS_CBC);
613
}
614

615
static int omap_aes_ctr_encrypt(struct skcipher_request *req)
616
{
617
	return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR);
618
}
619

620
static int omap_aes_ctr_decrypt(struct skcipher_request *req)
621
{
622
	return omap_aes_crypt(req, FLAGS_CTR);
623
}
624

625
static int omap_aes_init_tfm(struct crypto_skcipher *tfm)
626
{
627
	const char *name = crypto_tfm_alg_name(&tfm->base);
628
	struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
629
	struct crypto_skcipher *blk;
630

631
	blk = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
632
	if (IS_ERR(blk))
633
		return PTR_ERR(blk);
634

635
	ctx->fallback = blk;
636

637
	crypto_skcipher_set_reqsize(tfm, sizeof(struct omap_aes_reqctx) +
638
					 crypto_skcipher_reqsize(blk));
639

640
	return 0;
641
}
642

643
static void omap_aes_exit_tfm(struct crypto_skcipher *tfm)
644
{
645
	struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
646

647
	if (ctx->fallback)
648
		crypto_free_skcipher(ctx->fallback);
649

650
	ctx->fallback = NULL;
651
}
652

653
/* ********************** ALGS ************************************ */
654

655
static struct skcipher_engine_alg algs_ecb_cbc[] = {
656
{
657
	.base = {
658
		.base.cra_name		= "ecb(aes)",
659
		.base.cra_driver_name	= "ecb-aes-omap",
660
		.base.cra_priority	= 300,
661
		.base.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
662
					  CRYPTO_ALG_ASYNC |
663
					  CRYPTO_ALG_NEED_FALLBACK,
664
		.base.cra_blocksize	= AES_BLOCK_SIZE,
665
		.base.cra_ctxsize	= sizeof(struct omap_aes_ctx),
666
		.base.cra_module	= THIS_MODULE,
667

668
		.min_keysize		= AES_MIN_KEY_SIZE,
669
		.max_keysize		= AES_MAX_KEY_SIZE,
670
		.setkey			= omap_aes_setkey,
671
		.encrypt		= omap_aes_ecb_encrypt,
672
		.decrypt		= omap_aes_ecb_decrypt,
673
		.init			= omap_aes_init_tfm,
674
		.exit			= omap_aes_exit_tfm,
675
	},
676
	.op.do_one_request = omap_aes_crypt_req,
677
},
678
{
679
	.base = {
680
		.base.cra_name		= "cbc(aes)",
681
		.base.cra_driver_name	= "cbc-aes-omap",
682
		.base.cra_priority	= 300,
683
		.base.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
684
					  CRYPTO_ALG_ASYNC |
685
					  CRYPTO_ALG_NEED_FALLBACK,
686
		.base.cra_blocksize	= AES_BLOCK_SIZE,
687
		.base.cra_ctxsize	= sizeof(struct omap_aes_ctx),
688
		.base.cra_module	= THIS_MODULE,
689

690
		.min_keysize		= AES_MIN_KEY_SIZE,
691
		.max_keysize		= AES_MAX_KEY_SIZE,
692
		.ivsize			= AES_BLOCK_SIZE,
693
		.setkey			= omap_aes_setkey,
694
		.encrypt		= omap_aes_cbc_encrypt,
695
		.decrypt		= omap_aes_cbc_decrypt,
696
		.init			= omap_aes_init_tfm,
697
		.exit			= omap_aes_exit_tfm,
698
	},
699
	.op.do_one_request = omap_aes_crypt_req,
700
}
701
};
702

703
static struct skcipher_engine_alg algs_ctr[] = {
704
{
705
	.base = {
706
		.base.cra_name		= "ctr(aes)",
707
		.base.cra_driver_name	= "ctr-aes-omap",
708
		.base.cra_priority	= 300,
709
		.base.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
710
					  CRYPTO_ALG_ASYNC |
711
					  CRYPTO_ALG_NEED_FALLBACK,
712
		.base.cra_blocksize	= 1,
713
		.base.cra_ctxsize	= sizeof(struct omap_aes_ctx),
714
		.base.cra_module	= THIS_MODULE,
715

716
		.min_keysize		= AES_MIN_KEY_SIZE,
717
		.max_keysize		= AES_MAX_KEY_SIZE,
718
		.ivsize			= AES_BLOCK_SIZE,
719
		.setkey			= omap_aes_setkey,
720
		.encrypt		= omap_aes_ctr_encrypt,
721
		.decrypt		= omap_aes_ctr_decrypt,
722
		.init			= omap_aes_init_tfm,
723
		.exit			= omap_aes_exit_tfm,
724
	},
725
	.op.do_one_request = omap_aes_crypt_req,
726
}
727
};
728

729
static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = {
730
	{
731
		.algs_list	= algs_ecb_cbc,
732
		.size		= ARRAY_SIZE(algs_ecb_cbc),
733
	},
734
};
735

736
static struct aead_engine_alg algs_aead_gcm[] = {
737
{
738
	.base = {
739
		.base = {
740
			.cra_name		= "gcm(aes)",
741
			.cra_driver_name	= "gcm-aes-omap",
742
			.cra_priority		= 300,
743
			.cra_flags		= CRYPTO_ALG_ASYNC |
744
						  CRYPTO_ALG_KERN_DRIVER_ONLY,
745
			.cra_blocksize		= 1,
746
			.cra_ctxsize		= sizeof(struct omap_aes_gcm_ctx),
747
			.cra_alignmask		= 0xf,
748
			.cra_module		= THIS_MODULE,
749
		},
750
		.init		= omap_aes_gcm_cra_init,
751
		.ivsize		= GCM_AES_IV_SIZE,
752
		.maxauthsize	= AES_BLOCK_SIZE,
753
		.setkey		= omap_aes_gcm_setkey,
754
		.setauthsize	= omap_aes_gcm_setauthsize,
755
		.encrypt	= omap_aes_gcm_encrypt,
756
		.decrypt	= omap_aes_gcm_decrypt,
757
	},
758
	.op.do_one_request = omap_aes_gcm_crypt_req,
759
},
760
{
761
	.base = {
762
		.base = {
763
			.cra_name		= "rfc4106(gcm(aes))",
764
			.cra_driver_name	= "rfc4106-gcm-aes-omap",
765
			.cra_priority		= 300,
766
			.cra_flags		= CRYPTO_ALG_ASYNC |
767
						  CRYPTO_ALG_KERN_DRIVER_ONLY,
768
			.cra_blocksize		= 1,
769
			.cra_ctxsize		= sizeof(struct omap_aes_gcm_ctx),
770
			.cra_alignmask		= 0xf,
771
			.cra_module		= THIS_MODULE,
772
		},
773
		.init		= omap_aes_gcm_cra_init,
774
		.maxauthsize	= AES_BLOCK_SIZE,
775
		.ivsize		= GCM_RFC4106_IV_SIZE,
776
		.setkey		= omap_aes_4106gcm_setkey,
777
		.setauthsize	= omap_aes_4106gcm_setauthsize,
778
		.encrypt	= omap_aes_4106gcm_encrypt,
779
		.decrypt	= omap_aes_4106gcm_decrypt,
780
	},
781
	.op.do_one_request = omap_aes_gcm_crypt_req,
782
},
783
};
784

785
static struct omap_aes_aead_algs omap_aes_aead_info = {
786
	.algs_list	=	algs_aead_gcm,
787
	.size		=	ARRAY_SIZE(algs_aead_gcm),
788
};
789

790
static const struct omap_aes_pdata omap_aes_pdata_omap2 = {
791
	.algs_info	= omap_aes_algs_info_ecb_cbc,
792
	.algs_info_size	= ARRAY_SIZE(omap_aes_algs_info_ecb_cbc),
793
	.trigger	= omap_aes_dma_trigger_omap2,
794
	.key_ofs	= 0x1c,
795
	.iv_ofs		= 0x20,
796
	.ctrl_ofs	= 0x30,
797
	.data_ofs	= 0x34,
798
	.rev_ofs	= 0x44,
799
	.mask_ofs	= 0x48,
800
	.dma_enable_in	= BIT(2),
801
	.dma_enable_out	= BIT(3),
802
	.dma_start	= BIT(5),
803
	.major_mask	= 0xf0,
804
	.major_shift	= 4,
805
	.minor_mask	= 0x0f,
806
	.minor_shift	= 0,
807
};
808

809
#ifdef CONFIG_OF
810
static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr[] = {
811
	{
812
		.algs_list	= algs_ecb_cbc,
813
		.size		= ARRAY_SIZE(algs_ecb_cbc),
814
	},
815
	{
816
		.algs_list	= algs_ctr,
817
		.size		= ARRAY_SIZE(algs_ctr),
818
	},
819
};
820

821
static const struct omap_aes_pdata omap_aes_pdata_omap3 = {
822
	.algs_info	= omap_aes_algs_info_ecb_cbc_ctr,
823
	.algs_info_size	= ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
824
	.trigger	= omap_aes_dma_trigger_omap2,
825
	.key_ofs	= 0x1c,
826
	.iv_ofs		= 0x20,
827
	.ctrl_ofs	= 0x30,
828
	.data_ofs	= 0x34,
829
	.rev_ofs	= 0x44,
830
	.mask_ofs	= 0x48,
831
	.dma_enable_in	= BIT(2),
832
	.dma_enable_out	= BIT(3),
833
	.dma_start	= BIT(5),
834
	.major_mask	= 0xf0,
835
	.major_shift	= 4,
836
	.minor_mask	= 0x0f,
837
	.minor_shift	= 0,
838
};
839

840
static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
841
	.algs_info	= omap_aes_algs_info_ecb_cbc_ctr,
842
	.algs_info_size	= ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
843
	.aead_algs_info	= &omap_aes_aead_info,
844
	.trigger	= omap_aes_dma_trigger_omap4,
845
	.key_ofs	= 0x3c,
846
	.iv_ofs		= 0x40,
847
	.ctrl_ofs	= 0x50,
848
	.data_ofs	= 0x60,
849
	.rev_ofs	= 0x80,
850
	.mask_ofs	= 0x84,
851
	.irq_status_ofs = 0x8c,
852
	.irq_enable_ofs = 0x90,
853
	.dma_enable_in	= BIT(5),
854
	.dma_enable_out	= BIT(6),
855
	.major_mask	= 0x0700,
856
	.major_shift	= 8,
857
	.minor_mask	= 0x003f,
858
	.minor_shift	= 0,
859
};
860

861
static irqreturn_t omap_aes_irq(int irq, void *dev_id)
862
{
863
	struct omap_aes_dev *dd = dev_id;
864
	u32 status, i;
865
	u32 *src, *dst;
866

867
	status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));
868
	if (status & AES_REG_IRQ_DATA_IN) {
869
		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
870

871
		BUG_ON(!dd->in_sg);
872

873
		BUG_ON(dd->in_sg_offset > dd->in_sg->length);
874

875
		src = sg_virt(dd->in_sg) + dd->in_sg_offset;
876

877
		for (i = 0; i < AES_BLOCK_WORDS; i++) {
878
			omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);
879
			dd->in_sg_offset += 4;
880
			if (dd->in_sg_offset == dd->in_sg->length) {
881
				dd->in_sg = sg_next(dd->in_sg);
882
				if (dd->in_sg) {
883
					dd->in_sg_offset = 0;
884
					src = sg_virt(dd->in_sg);
885
				}
886
			} else {
887
				src++;
888
			}
889
		}
890

891
		/* Clear IRQ status */
892
		status &= ~AES_REG_IRQ_DATA_IN;
893
		omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
894

895
		/* Enable DATA_OUT interrupt */
896
		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);
897

898
	} else if (status & AES_REG_IRQ_DATA_OUT) {
899
		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
900

901
		BUG_ON(!dd->out_sg);
902

903
		BUG_ON(dd->out_sg_offset > dd->out_sg->length);
904

905
		dst = sg_virt(dd->out_sg) + dd->out_sg_offset;
906

907
		for (i = 0; i < AES_BLOCK_WORDS; i++) {
908
			*dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
909
			dd->out_sg_offset += 4;
910
			if (dd->out_sg_offset == dd->out_sg->length) {
911
				dd->out_sg = sg_next(dd->out_sg);
912
				if (dd->out_sg) {
913
					dd->out_sg_offset = 0;
914
					dst = sg_virt(dd->out_sg);
915
				}
916
			} else {
917
				dst++;
918
			}
919
		}
920

921
		dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total);
922

923
		/* Clear IRQ status */
924
		status &= ~AES_REG_IRQ_DATA_OUT;
925
		omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
926

927
		if (!dd->total)
928
			/* All bytes read! */
929
			queue_work(system_bh_wq, &dd->done_task);
930
		else
931
			/* Enable DATA_IN interrupt for next block */
932
			omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
933
	}
934

935
	return IRQ_HANDLED;
936
}
937

938
static const struct of_device_id omap_aes_of_match[] = {
939
	{
940
		.compatible	= "ti,omap2-aes",
941
		.data		= &omap_aes_pdata_omap2,
942
	},
943
	{
944
		.compatible	= "ti,omap3-aes",
945
		.data		= &omap_aes_pdata_omap3,
946
	},
947
	{
948
		.compatible	= "ti,omap4-aes",
949
		.data		= &omap_aes_pdata_omap4,
950
	},
951
	{},
952
};
953
MODULE_DEVICE_TABLE(of, omap_aes_of_match);
954

955
static int omap_aes_get_res_of(struct omap_aes_dev *dd,
956
		struct device *dev, struct resource *res)
957
{
958
	struct device_node *node = dev->of_node;
959
	int err = 0;
960

961
	dd->pdata = of_device_get_match_data(dev);
962
	if (!dd->pdata) {
963
		dev_err(dev, "no compatible OF match\n");
964
		err = -EINVAL;
965
		goto err;
966
	}
967

968
	err = of_address_to_resource(node, 0, res);
969
	if (err < 0) {
970
		dev_err(dev, "can't translate OF node address\n");
971
		err = -EINVAL;
972
		goto err;
973
	}
974

975
err:
976
	return err;
977
}
978
#else
979
static const struct of_device_id omap_aes_of_match[] = {
980
	{},
981
};
982

983
static int omap_aes_get_res_of(struct omap_aes_dev *dd,
984
		struct device *dev, struct resource *res)
985
{
986
	return -EINVAL;
987
}
988
#endif
989

990
static int omap_aes_get_res_pdev(struct omap_aes_dev *dd,
991
		struct platform_device *pdev, struct resource *res)
992
{
993
	struct device *dev = &pdev->dev;
994
	struct resource *r;
995
	int err = 0;
996

997
	/* Get the base address */
998
	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
999
	if (!r) {
1000
		dev_err(dev, "no MEM resource info\n");
1001
		err = -ENODEV;
1002
		goto err;
1003
	}
1004
	memcpy(res, r, sizeof(*res));
1005

1006
	/* Only OMAP2/3 can be non-DT */
1007
	dd->pdata = &omap_aes_pdata_omap2;
1008

1009
err:
1010
	return err;
1011
}
1012

1013
static ssize_t fallback_show(struct device *dev, struct device_attribute *attr,
1014
			     char *buf)
1015
{
1016
	return sprintf(buf, "%d\n", aes_fallback_sz);
1017
}
1018

1019
static ssize_t fallback_store(struct device *dev, struct device_attribute *attr,
1020
			      const char *buf, size_t size)
1021
{
1022
	ssize_t status;
1023
	long value;
1024

1025
	status = kstrtol(buf, 0, &value);
1026
	if (status)
1027
		return status;
1028

1029
	/* HW accelerator only works with buffers > 9 */
1030
	if (value < 9) {
1031
		dev_err(dev, "minimum fallback size 9\n");
1032
		return -EINVAL;
1033
	}
1034

1035
	aes_fallback_sz = value;
1036

1037
	return size;
1038
}
1039

1040
static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr,
1041
			      char *buf)
1042
{
1043
	struct omap_aes_dev *dd = dev_get_drvdata(dev);
1044

1045
	return sprintf(buf, "%d\n", dd->engine->queue.max_qlen);
1046
}
1047

1048
static ssize_t queue_len_store(struct device *dev,
1049
			       struct device_attribute *attr, const char *buf,
1050
			       size_t size)
1051
{
1052
	struct omap_aes_dev *dd;
1053
	ssize_t status;
1054
	long value;
1055
	unsigned long flags;
1056

1057
	status = kstrtol(buf, 0, &value);
1058
	if (status)
1059
		return status;
1060

1061
	if (value < 1)
1062
		return -EINVAL;
1063

1064
	/*
1065
	 * Changing the queue size in fly is safe, if size becomes smaller
1066
	 * than current size, it will just not accept new entries until
1067
	 * it has shrank enough.
1068
	 */
1069
	spin_lock_bh(&list_lock);
1070
	list_for_each_entry(dd, &dev_list, list) {
1071
		spin_lock_irqsave(&dd->lock, flags);
1072
		dd->engine->queue.max_qlen = value;
1073
		dd->aead_queue.base.max_qlen = value;
1074
		spin_unlock_irqrestore(&dd->lock, flags);
1075
	}
1076
	spin_unlock_bh(&list_lock);
1077

1078
	return size;
1079
}
1080

1081
static DEVICE_ATTR_RW(queue_len);
1082
static DEVICE_ATTR_RW(fallback);
1083

1084
static struct attribute *omap_aes_attrs[] = {
1085
	&dev_attr_queue_len.attr,
1086
	&dev_attr_fallback.attr,
1087
	NULL,
1088
};
1089
ATTRIBUTE_GROUPS(omap_aes);
1090

1091
static int omap_aes_probe(struct platform_device *pdev)
1092
{
1093
	struct device *dev = &pdev->dev;
1094
	struct omap_aes_dev *dd;
1095
	struct skcipher_engine_alg *algp;
1096
	struct aead_engine_alg *aalg;
1097
	struct resource res;
1098
	int err = -ENOMEM, i, j, irq = -1;
1099
	u32 reg;
1100

1101
	dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);
1102
	if (dd == NULL) {
1103
		dev_err(dev, "unable to alloc data struct.\n");
1104
		goto err_data;
1105
	}
1106
	dd->dev = dev;
1107
	platform_set_drvdata(pdev, dd);
1108

1109
	aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH);
1110

1111
	err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :
1112
			       omap_aes_get_res_pdev(dd, pdev, &res);
1113
	if (err)
1114
		goto err_res;
1115

1116
	dd->io_base = devm_ioremap_resource(dev, &res);
1117
	if (IS_ERR(dd->io_base)) {
1118
		err = PTR_ERR(dd->io_base);
1119
		goto err_res;
1120
	}
1121
	dd->phys_base = res.start;
1122

1123
	pm_runtime_use_autosuspend(dev);
1124
	pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
1125

1126
	pm_runtime_enable(dev);
1127
	err = pm_runtime_resume_and_get(dev);
1128
	if (err < 0) {
1129
		dev_err(dev, "%s: failed to get_sync(%d)\n",
1130
			__func__, err);
1131
		goto err_pm_disable;
1132
	}
1133

1134
	omap_aes_dma_stop(dd);
1135

1136
	reg = omap_aes_read(dd, AES_REG_REV(dd));
1137

1138
	pm_runtime_put_sync(dev);
1139

1140
	dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
1141
		 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
1142
		 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1143

1144
	INIT_WORK(&dd->done_task, omap_aes_done_task);
1145

1146
	err = omap_aes_dma_init(dd);
1147
	if (err == -EPROBE_DEFER) {
1148
		goto err_irq;
1149
	} else if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {
1150
		dd->pio_only = 1;
1151

1152
		irq = platform_get_irq(pdev, 0);
1153
		if (irq < 0) {
1154
			err = irq;
1155
			goto err_irq;
1156
		}
1157

1158
		err = devm_request_irq(dev, irq, omap_aes_irq, 0,
1159
				dev_name(dev), dd);
1160
		if (err) {
1161
			dev_err(dev, "Unable to grab omap-aes IRQ\n");
1162
			goto err_irq;
1163
		}
1164
	}
1165

1166
	spin_lock_init(&dd->lock);
1167

1168
	INIT_LIST_HEAD(&dd->list);
1169
	spin_lock_bh(&list_lock);
1170
	list_add_tail(&dd->list, &dev_list);
1171
	spin_unlock_bh(&list_lock);
1172

1173
	/* Initialize crypto engine */
1174
	dd->engine = crypto_engine_alloc_init(dev, 1);
1175
	if (!dd->engine) {
1176
		err = -ENOMEM;
1177
		goto err_engine;
1178
	}
1179

1180
	err = crypto_engine_start(dd->engine);
1181
	if (err)
1182
		goto err_engine;
1183

1184
	for (i = 0; i < dd->pdata->algs_info_size; i++) {
1185
		if (!dd->pdata->algs_info[i].registered) {
1186
			for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1187
				algp = &dd->pdata->algs_info[i].algs_list[j];
1188

1189
				pr_debug("reg alg: %s\n", algp->base.base.cra_name);
1190

1191
				err = crypto_engine_register_skcipher(algp);
1192
				if (err)
1193
					goto err_algs;
1194

1195
				dd->pdata->algs_info[i].registered++;
1196
			}
1197
		}
1198
	}
1199

1200
	if (dd->pdata->aead_algs_info &&
1201
	    !dd->pdata->aead_algs_info->registered) {
1202
		for (i = 0; i < dd->pdata->aead_algs_info->size; i++) {
1203
			aalg = &dd->pdata->aead_algs_info->algs_list[i];
1204

1205
			pr_debug("reg alg: %s\n", aalg->base.base.cra_name);
1206

1207
			err = crypto_engine_register_aead(aalg);
1208
			if (err)
1209
				goto err_aead_algs;
1210

1211
			dd->pdata->aead_algs_info->registered++;
1212
		}
1213
	}
1214

1215
	return 0;
1216
err_aead_algs:
1217
	for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
1218
		aalg = &dd->pdata->aead_algs_info->algs_list[i];
1219
		crypto_engine_unregister_aead(aalg);
1220
	}
1221
err_algs:
1222
	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1223
		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1224
			crypto_engine_unregister_skcipher(
1225
					&dd->pdata->algs_info[i].algs_list[j]);
1226

1227
err_engine:
1228
	if (dd->engine)
1229
		crypto_engine_exit(dd->engine);
1230

1231
	omap_aes_dma_cleanup(dd);
1232
err_irq:
1233
	cancel_work_sync(&dd->done_task);
1234
err_pm_disable:
1235
	pm_runtime_disable(dev);
1236
err_res:
1237
	dd = NULL;
1238
err_data:
1239
	dev_err(dev, "initialization failed.\n");
1240
	return err;
1241
}
1242

1243
static void omap_aes_remove(struct platform_device *pdev)
1244
{
1245
	struct omap_aes_dev *dd = platform_get_drvdata(pdev);
1246
	struct aead_engine_alg *aalg;
1247
	int i, j;
1248

1249
	spin_lock_bh(&list_lock);
1250
	list_del(&dd->list);
1251
	spin_unlock_bh(&list_lock);
1252

1253
	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1254
		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
1255
			crypto_engine_unregister_skcipher(
1256
					&dd->pdata->algs_info[i].algs_list[j]);
1257
			dd->pdata->algs_info[i].registered--;
1258
		}
1259

1260
	for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
1261
		aalg = &dd->pdata->aead_algs_info->algs_list[i];
1262
		crypto_engine_unregister_aead(aalg);
1263
		dd->pdata->aead_algs_info->registered--;
1264
	}
1265

1266
	crypto_engine_exit(dd->engine);
1267

1268
	cancel_work_sync(&dd->done_task);
1269
	omap_aes_dma_cleanup(dd);
1270
	pm_runtime_disable(dd->dev);
1271
}
1272

1273
#ifdef CONFIG_PM_SLEEP
1274
static int omap_aes_suspend(struct device *dev)
1275
{
1276
	pm_runtime_put_sync(dev);
1277
	return 0;
1278
}
1279

1280
static int omap_aes_resume(struct device *dev)
1281
{
1282
	pm_runtime_get_sync(dev);
1283
	return 0;
1284
}
1285
#endif
1286

1287
static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume);
1288

1289
static struct platform_driver omap_aes_driver = {
1290
	.probe	= omap_aes_probe,
1291
	.remove = omap_aes_remove,
1292
	.driver	= {
1293
		.name	= "omap-aes",
1294
		.pm	= &omap_aes_pm_ops,
1295
		.of_match_table	= omap_aes_of_match,
1296
		.dev_groups = omap_aes_groups,
1297
	},
1298
};
1299

1300
module_platform_driver(omap_aes_driver);
1301

1302
MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
1303
MODULE_LICENSE("GPL v2");
1304
MODULE_AUTHOR("Dmitry Kasatkin");
1305

1306

1307