1
/* SPDX-License-Identifier: GPL-2.0 */
2
/*
3
 * sun8i-ce.h - hardware cryptographic offloader for
4
 * Allwinner H3/A64/H5/H2+/H6 SoC
5
 *
6
 * Copyright (C) 2016-2019 Corentin LABBE <[email protected]>
7
 */
8
#include <crypto/aes.h>
9
#include <crypto/des.h>
10
#include <crypto/engine.h>
11
#include <crypto/skcipher.h>
12
#include <linux/atomic.h>
13
#include <linux/debugfs.h>
14
#include <linux/crypto.h>
15
#include <linux/hw_random.h>
16
#include <crypto/internal/hash.h>
17
#include <crypto/md5.h>
18
#include <crypto/rng.h>
19
#include <crypto/sha1.h>
20
#include <crypto/sha2.h>
21

22
/* CE Registers */
23
#define CE_TDQ	0x00
24
#define CE_CTR	0x04
25
#define CE_ICR	0x08
26
#define CE_ISR	0x0C
27
#define CE_TLR	0x10
28
#define CE_TSR	0x14
29
#define CE_ESR	0x18
30
#define CE_CSSGR	0x1C
31
#define CE_CDSGR	0x20
32
#define CE_CSAR	0x24
33
#define CE_CDAR	0x28
34
#define CE_TPR	0x2C
35

36
/* Used in struct ce_task */
37
/* ce_task common */
38
#define CE_ENCRYPTION		0
39
#define CE_DECRYPTION		BIT(8)
40

41
#define CE_COMM_INT		BIT(31)
42

43
/* ce_task symmetric */
44
#define CE_AES_128BITS 0
45
#define CE_AES_192BITS 1
46
#define CE_AES_256BITS 2
47

48
#define CE_OP_ECB	0
49
#define CE_OP_CBC	(1 << 8)
50

51
#define CE_ALG_AES		0
52
#define CE_ALG_DES		1
53
#define CE_ALG_3DES		2
54
#define CE_ALG_MD5              16
55
#define CE_ALG_SHA1             17
56
#define CE_ALG_SHA224           18
57
#define CE_ALG_SHA256           19
58
#define CE_ALG_SHA384           20
59
#define CE_ALG_SHA512           21
60
#define CE_ALG_TRNG		48
61
#define CE_ALG_PRNG		49
62
#define CE_ALG_TRNG_V2		0x1c
63
#define CE_ALG_PRNG_V2		0x1d
64

65
/* Used in ce_variant */
66
#define CE_ID_NOTSUPP		0xFF
67

68
#define CE_ID_CIPHER_AES	0
69
#define CE_ID_CIPHER_DES	1
70
#define CE_ID_CIPHER_DES3	2
71
#define CE_ID_CIPHER_MAX	3
72

73
#define CE_ID_HASH_MD5		0
74
#define CE_ID_HASH_SHA1		1
75
#define CE_ID_HASH_SHA224	2
76
#define CE_ID_HASH_SHA256	3
77
#define CE_ID_HASH_SHA384	4
78
#define CE_ID_HASH_SHA512	5
79
#define CE_ID_HASH_MAX		6
80

81
#define CE_ID_OP_ECB	0
82
#define CE_ID_OP_CBC	1
83
#define CE_ID_OP_MAX	2
84

85
/* Used in CE registers */
86
#define CE_ERR_ALGO_NOTSUP	BIT(0)
87
#define CE_ERR_DATALEN		BIT(1)
88
#define CE_ERR_KEYSRAM		BIT(2)
89
#define CE_ERR_ADDR_INVALID	BIT(5)
90
#define CE_ERR_KEYLADDER	BIT(6)
91

92
#define ESR_H3	0
93
#define ESR_A64	1
94
#define ESR_R40	2
95
#define ESR_H5	3
96
#define ESR_H6	4
97
#define ESR_D1	5
98

99
#define PRNG_DATA_SIZE (160 / 8)
100
#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
101
#define PRNG_LD BIT(17)
102

103
#define CE_DIE_ID_SHIFT	16
104
#define CE_DIE_ID_MASK	0x07
105

106
#define MAX_SG 8
107

108
#define CE_MAX_CLOCKS 4
109
#define CE_DMA_TIMEOUT_MS	3000
110

111
#define MAXFLOW 4
112

113
#define CE_MAX_HASH_DIGEST_SIZE		SHA512_DIGEST_SIZE
114
#define CE_MAX_HASH_BLOCK_SIZE		SHA512_BLOCK_SIZE
115

116
/*
117
 * struct ce_clock - Describe clocks used by sun8i-ce
118
 * @name:	Name of clock needed by this variant
119
 * @freq:	Frequency to set for each clock
120
 * @max_freq:	Maximum frequency for each clock (generally given by datasheet)
121
 */
122
struct ce_clock {
123
	const char *name;
124
	unsigned long freq;
125
	unsigned long max_freq;
126
};
127

128
/*
129
 * struct ce_variant - Describe CE capability for each variant hardware
130
 * @alg_cipher:	list of supported ciphers. for each CE_ID_ this will give the
131
 *              coresponding CE_ALG_XXX value
132
 * @alg_hash:	list of supported hashes. for each CE_ID_ this will give the
133
 *              corresponding CE_ALG_XXX value
134
 * @op_mode:	list of supported block modes
135
 * @cipher_t_dlen_in_bytes:	Does the request size for cipher is in
136
 *				bytes or words
137
 * @hash_t_dlen_in_bytes:	Does the request size for hash is in
138
 *				bits or words
139
 * @prng_t_dlen_in_bytes:	Does the request size for PRNG is in
140
 *				bytes or words
141
 * @trng_t_dlen_in_bytes:	Does the request size for TRNG is in
142
 *				bytes or words
143
 * @ce_clks:	list of clocks needed by this variant
144
 * @esr:	The type of error register
145
 * @prng:	The CE_ALG_XXX value for the PRNG
146
 * @trng:	The CE_ALG_XXX value for the TRNG
147
 */
148
struct ce_variant {
149
	char alg_cipher[CE_ID_CIPHER_MAX];
150
	char alg_hash[CE_ID_HASH_MAX];
151
	u32 op_mode[CE_ID_OP_MAX];
152
	bool cipher_t_dlen_in_bytes;
153
	bool hash_t_dlen_in_bits;
154
	bool prng_t_dlen_in_bytes;
155
	bool trng_t_dlen_in_bytes;
156
	bool needs_word_addresses;
157
	struct ce_clock ce_clks[CE_MAX_CLOCKS];
158
	int esr;
159
	unsigned char prng;
160
	unsigned char trng;
161
};
162

163
struct sginfo {
164
	__le32 addr;
165
	__le32 len;
166
} __packed;
167

168
/*
169
 * struct ce_task - CE Task descriptor
170
 * The structure of this descriptor could be found in the datasheet
171
 */
172
struct ce_task {
173
	__le32 t_id;
174
	__le32 t_common_ctl;
175
	__le32 t_sym_ctl;
176
	__le32 t_asym_ctl;
177
	__le32 t_key;
178
	__le32 t_iv;
179
	__le32 t_ctr;
180
	__le32 t_dlen;
181
	struct sginfo t_src[MAX_SG];
182
	struct sginfo t_dst[MAX_SG];
183
	__le32 next;
184
	__le32 reserved[3];
185
} __packed __aligned(8);
186

187
/*
188
 * struct sun8i_ce_flow - Information used by each flow
189
 * @engine:	ptr to the crypto_engine for this flow
190
 * @complete:	completion for the current task on this flow
191
 * @status:	set to 1 by interrupt if task is done
192
 * @t_phy:	Physical address of task
193
 * @tl:		pointer to the current ce_task for this flow
194
 * @stat_req:	number of request done by this flow
195
 */
196
struct sun8i_ce_flow {
197
	struct crypto_engine *engine;
198
	struct completion complete;
199
	int status;
200
	dma_addr_t t_phy;
201
	struct ce_task *tl;
202
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
203
	unsigned long stat_req;
204
#endif
205
};
206

207
/*
208
 * struct sun8i_ce_dev - main container for all this driver information
209
 * @base:	base address of CE
210
 * @ceclks:	clocks used by CE
211
 * @reset:	pointer to reset controller
212
 * @dev:	the platform device
213
 * @mlock:	Control access to device registers
214
 * @rnglock:	Control access to the RNG (dedicated channel 3)
215
 * @chanlist:	array of all flow
216
 * @flow:	flow to use in next request
217
 * @variant:	pointer to variant specific data
218
 * @dbgfs_dir:	Debugfs dentry for statistic directory
219
 * @dbgfs_stats: Debugfs dentry for statistic counters
220
 */
221
struct sun8i_ce_dev {
222
	void __iomem *base;
223
	struct clk *ceclks[CE_MAX_CLOCKS];
224
	struct reset_control *reset;
225
	struct device *dev;
226
	struct mutex mlock;
227
	struct mutex rnglock;
228
	struct sun8i_ce_flow *chanlist;
229
	atomic_t flow;
230
	const struct ce_variant *variant;
231
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
232
	struct dentry *dbgfs_dir;
233
	struct dentry *dbgfs_stats;
234
#endif
235
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
236
	struct hwrng trng;
237
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
238
	unsigned long hwrng_stat_req;
239
	unsigned long hwrng_stat_bytes;
240
#endif
241
#endif
242
};
243

244
static inline u32 desc_addr_val(struct sun8i_ce_dev *dev, dma_addr_t addr)
245
{
246
	if (dev->variant->needs_word_addresses)
247
		return addr / 4;
248

249
	return addr;
250
}
251

252
static inline __le32 desc_addr_val_le32(struct sun8i_ce_dev *dev,
253
					dma_addr_t addr)
254
{
255
	return cpu_to_le32(desc_addr_val(dev, addr));
256
}
257

258
/*
259
 * struct sun8i_cipher_req_ctx - context for a skcipher request
260
 * @op_dir:		direction (encrypt vs decrypt) for this request
261
 * @flow:		the flow to use for this request
262
 * @nr_sgs:		The number of source SG (as given by dma_map_sg())
263
 * @nr_sgd:		The number of destination SG (as given by dma_map_sg())
264
 * @addr_iv:		The IV addr returned by dma_map_single, need to unmap later
265
 * @addr_key:		The key addr returned by dma_map_single, need to unmap later
266
 * @bounce_iv:		Current IV buffer
267
 * @backup_iv:		Next IV buffer
268
 * @fallback_req:	request struct for invoking the fallback skcipher TFM
269
 */
270
struct sun8i_cipher_req_ctx {
271
	u32 op_dir;
272
	int flow;
273
	int nr_sgs;
274
	int nr_sgd;
275
	dma_addr_t addr_iv;
276
	dma_addr_t addr_key;
277
	u8 bounce_iv[AES_BLOCK_SIZE] __aligned(sizeof(u32));
278
	u8 backup_iv[AES_BLOCK_SIZE];
279
	struct skcipher_request fallback_req;   // keep at the end
280
};
281

282
/*
283
 * struct sun8i_cipher_tfm_ctx - context for a skcipher TFM
284
 * @key:		pointer to key data
285
 * @keylen:		len of the key
286
 * @ce:			pointer to the private data of driver handling this TFM
287
 * @fallback_tfm:	pointer to the fallback TFM
288
 */
289
struct sun8i_cipher_tfm_ctx {
290
	u32 *key;
291
	u32 keylen;
292
	struct sun8i_ce_dev *ce;
293
	struct crypto_skcipher *fallback_tfm;
294
};
295

296
/*
297
 * struct sun8i_ce_hash_tfm_ctx - context for an ahash TFM
298
 * @ce:			pointer to the private data of driver handling this TFM
299
 * @fallback_tfm:	pointer to the fallback TFM
300
 */
301
struct sun8i_ce_hash_tfm_ctx {
302
	struct sun8i_ce_dev *ce;
303
	struct crypto_ahash *fallback_tfm;
304
};
305

306
/*
307
 * struct sun8i_ce_hash_reqctx - context for an ahash request
308
 * @fallback_req:	pre-allocated fallback request
309
 * @flow:	the flow to use for this request
310
 * @nr_sgs: number of entries in the source scatterlist
311
 * @result_len: result length in bytes
312
 * @pad_len: padding length in bytes
313
 * @addr_res: DMA address of the result buffer, returned by dma_map_single()
314
 * @addr_pad: DMA address of the padding buffer, returned by dma_map_single()
315
 * @result: per-request result buffer
316
 * @pad: per-request padding buffer (up to 2 blocks)
317
 */
318
struct sun8i_ce_hash_reqctx {
319
	int flow;
320
	int nr_sgs;
321
	size_t result_len;
322
	size_t pad_len;
323
	dma_addr_t addr_res;
324
	dma_addr_t addr_pad;
325
	u8 result[CE_MAX_HASH_DIGEST_SIZE] __aligned(CRYPTO_DMA_ALIGN);
326
	u8 pad[2 * CE_MAX_HASH_BLOCK_SIZE];
327
	struct ahash_request fallback_req; // keep at the end
328
};
329

330
/*
331
 * struct sun8i_ce_prng_ctx - context for PRNG TFM
332
 * @seed:	The seed to use
333
 * @slen:	The size of the seed
334
 */
335
struct sun8i_ce_rng_tfm_ctx {
336
	void *seed;
337
	unsigned int slen;
338
};
339

340
/*
341
 * struct sun8i_ce_alg_template - crypto_alg template
342
 * @type:		the CRYPTO_ALG_TYPE for this template
343
 * @ce_algo_id:		the CE_ID for this template
344
 * @ce_blockmode:	the type of block operation CE_ID
345
 * @ce:			pointer to the sun8i_ce_dev structure associated with
346
 *			this template
347
 * @alg:		one of sub struct must be used
348
 * @stat_req:		number of request done on this template
349
 * @stat_fb:		number of request which has fallbacked
350
 * @stat_bytes:		total data size done by this template
351
 */
352
struct sun8i_ce_alg_template {
353
	u32 type;
354
	u32 ce_algo_id;
355
	u32 ce_blockmode;
356
	struct sun8i_ce_dev *ce;
357
	union {
358
		struct skcipher_engine_alg skcipher;
359
		struct ahash_engine_alg hash;
360
		struct rng_alg rng;
361
	} alg;
362
	unsigned long stat_req;
363
	unsigned long stat_fb;
364
	unsigned long stat_bytes;
365
	unsigned long stat_fb_maxsg;
366
	unsigned long stat_fb_leniv;
367
	unsigned long stat_fb_len0;
368
	unsigned long stat_fb_mod16;
369
	unsigned long stat_fb_srcali;
370
	unsigned long stat_fb_srclen;
371
	unsigned long stat_fb_dstali;
372
	unsigned long stat_fb_dstlen;
373
	char fbname[CRYPTO_MAX_ALG_NAME];
374
};
375

376
int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
377
			unsigned int keylen);
378
int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
379
			 unsigned int keylen);
380
int sun8i_ce_cipher_init(struct crypto_tfm *tfm);
381
void sun8i_ce_cipher_exit(struct crypto_tfm *tfm);
382
int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq);
383
int sun8i_ce_skdecrypt(struct skcipher_request *areq);
384
int sun8i_ce_skencrypt(struct skcipher_request *areq);
385

386
int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce);
387

388
int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name);
389

390
int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm);
391
void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm);
392
int sun8i_ce_hash_init(struct ahash_request *areq);
393
int sun8i_ce_hash_export(struct ahash_request *areq, void *out);
394
int sun8i_ce_hash_import(struct ahash_request *areq, const void *in);
395
int sun8i_ce_hash_final(struct ahash_request *areq);
396
int sun8i_ce_hash_update(struct ahash_request *areq);
397
int sun8i_ce_hash_finup(struct ahash_request *areq);
398
int sun8i_ce_hash_digest(struct ahash_request *areq);
399
int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq);
400

401
int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src,
402
			   unsigned int slen, u8 *dst, unsigned int dlen);
403
int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
404
void sun8i_ce_prng_exit(struct crypto_tfm *tfm);
405
int sun8i_ce_prng_init(struct crypto_tfm *tfm);
406

407
int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce);
408
void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce);
409

410