1
// SPDX-License-Identifier: GPL-2.0
2
/* Copyright (c) 2019 HiSilicon Limited. */
3
#include <linux/acpi.h>
4
#include <linux/bitops.h>
5
#include <linux/debugfs.h>
6
#include <linux/init.h>
7
#include <linux/io.h>
8
#include <linux/kernel.h>
9
#include <linux/module.h>
10
#include <linux/pci.h>
11
#include <linux/pm_runtime.h>
12
#include <linux/seq_file.h>
13
#include <linux/topology.h>
14
#include <linux/uacce.h>
15
#include "zip.h"
16

17
#define CAP_FILE_PERMISSION		0444
18
#define PCI_DEVICE_ID_HUAWEI_ZIP_PF	0xa250
19

20
#define HZIP_QUEUE_NUM_V1		4096
21

22
#define HZIP_CLOCK_GATE_CTRL		0x301004
23
#define HZIP_DECOMP_CHECK_ENABLE	BIT(16)
24
#define HZIP_FSM_MAX_CNT		0x301008
25

26
#define HZIP_PORT_ARCA_CHE_0		0x301040
27
#define HZIP_PORT_ARCA_CHE_1		0x301044
28
#define HZIP_PORT_AWCA_CHE_0		0x301060
29
#define HZIP_PORT_AWCA_CHE_1		0x301064
30
#define HZIP_CACHE_ALL_EN		0xffffffff
31

32
#define HZIP_BD_RUSER_32_63		0x301110
33
#define HZIP_SGL_RUSER_32_63		0x30111c
34
#define HZIP_DATA_RUSER_32_63		0x301128
35
#define HZIP_DATA_WUSER_32_63		0x301134
36
#define HZIP_BD_WUSER_32_63		0x301140
37

38
#define HZIP_QM_IDEL_STATUS		0x3040e4
39

40
#define HZIP_CORE_DFX_BASE		0x301000
41
#define HZIP_CORE_DFX_DECOMP_BASE	0x304000
42
#define HZIP_CORE_DFX_COMP_0		0x302000
43
#define HZIP_CORE_DFX_COMP_1		0x303000
44
#define HZIP_CORE_DFX_DECOMP_0		0x304000
45
#define HZIP_CORE_DFX_DECOMP_1		0x305000
46
#define HZIP_CORE_DFX_DECOMP_2		0x306000
47
#define HZIP_CORE_DFX_DECOMP_3		0x307000
48
#define HZIP_CORE_DFX_DECOMP_4		0x308000
49
#define HZIP_CORE_DFX_DECOMP_5		0x309000
50
#define HZIP_CORE_REGS_BASE_LEN		0xB0
51
#define HZIP_CORE_REGS_DFX_LEN		0x28
52
#define HZIP_CORE_ADDR_INTRVL		0x1000
53

54
#define HZIP_CORE_INT_SOURCE		0x3010A0
55
#define HZIP_CORE_INT_MASK_REG		0x3010A4
56
#define HZIP_CORE_INT_SET		0x3010A8
57
#define HZIP_CORE_INT_STATUS		0x3010AC
58
#define HZIP_CORE_INT_STATUS_M_ECC	BIT(1)
59
#define HZIP_CORE_SRAM_ECC_ERR_INFO	0x301148
60
#define HZIP_CORE_INT_RAS_CE_ENB	0x301160
61
#define HZIP_CORE_INT_RAS_NFE_ENB	0x301164
62
#define HZIP_CORE_INT_RAS_FE_ENB        0x301168
63
#define HZIP_CORE_INT_RAS_FE_ENB_MASK	0x0
64
#define HZIP_OOO_SHUTDOWN_SEL		0x30120C
65
#define HZIP_SRAM_ECC_ERR_NUM_SHIFT	16
66
#define HZIP_SRAM_ECC_ERR_ADDR_SHIFT	24
67
#define HZIP_CORE_INT_MASK_ALL		GENMASK(12, 0)
68
#define HZIP_AXI_ERROR_MASK		(BIT(2) | BIT(3))
69
#define HZIP_SQE_SIZE			128
70
#define HZIP_PF_DEF_Q_NUM		64
71
#define HZIP_PF_DEF_Q_BASE		0
72
#define HZIP_CTX_Q_NUM_DEF		2
73

74
#define HZIP_SOFT_CTRL_CNT_CLR_CE	0x301000
75
#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT	BIT(0)
76
#define HZIP_SOFT_CTRL_ZIP_CONTROL	0x30100C
77
#define HZIP_AXI_SHUTDOWN_ENABLE	BIT(14)
78
#define HZIP_WR_PORT			BIT(11)
79

80
#define HZIP_ALG_ZLIB_BIT		GENMASK(1, 0)
81
#define HZIP_ALG_GZIP_BIT		GENMASK(3, 2)
82
#define HZIP_ALG_DEFLATE_BIT		GENMASK(5, 4)
83
#define HZIP_ALG_LZ77_BIT		GENMASK(7, 6)
84
#define HZIP_ALG_LZ4_BIT		GENMASK(9, 8)
85

86
#define HZIP_BUF_SIZE			22
87
#define HZIP_SQE_MASK_OFFSET		64
88
#define HZIP_SQE_MASK_LEN		48
89

90
#define HZIP_CNT_CLR_CE_EN		BIT(0)
91
#define HZIP_RO_CNT_CLR_CE_EN		BIT(2)
92
#define HZIP_RD_CNT_CLR_CE_EN		(HZIP_CNT_CLR_CE_EN | \
93
					 HZIP_RO_CNT_CLR_CE_EN)
94

95
#define HZIP_PREFETCH_CFG		0x3011B0
96
#define HZIP_SVA_TRANS			0x3011C4
97
#define HZIP_PREFETCH_ENABLE		(~(BIT(26) | BIT(17) | BIT(0)))
98
#define HZIP_SVA_PREFETCH_DISABLE	BIT(26)
99
#define HZIP_SVA_DISABLE_READY		(BIT(26) | BIT(30))
100
#define HZIP_SVA_PREFETCH_NUM		GENMASK(18, 16)
101
#define HZIP_SVA_STALL_NUM		GENMASK(15, 0)
102
#define HZIP_SHAPER_RATE_COMPRESS	750
103
#define HZIP_SHAPER_RATE_DECOMPRESS	140
104
#define HZIP_DELAY_1_US			1
105
#define HZIP_POLL_TIMEOUT_US		1000
106
#define HZIP_WAIT_SVA_READY		500000
107
#define HZIP_READ_SVA_STATUS_TIMES	3
108
#define HZIP_WAIT_US_MIN		10
109
#define HZIP_WAIT_US_MAX		20
110

111
/* clock gating */
112
#define HZIP_PEH_CFG_AUTO_GATE		0x3011A8
113
#define HZIP_PEH_CFG_AUTO_GATE_EN	BIT(0)
114
#define HZIP_CORE_GATED_EN		GENMASK(15, 8)
115
#define HZIP_CORE_GATED_OOO_EN		BIT(29)
116
#define HZIP_CLOCK_GATED_EN		(HZIP_CORE_GATED_EN | \
117
					 HZIP_CORE_GATED_OOO_EN)
118

119
/* zip comp high performance */
120
#define HZIP_HIGH_PERF_OFFSET		0x301208
121

122
#define HZIP_LIT_LEN_EN_OFFSET		0x301204
123
#define HZIP_LIT_LEN_EN_EN		BIT(4)
124

125
enum {
126
	HZIP_HIGH_COMP_RATE,
127
	HZIP_HIGH_COMP_PERF,
128
};
129

130
static const char hisi_zip_name[] = "hisi_zip";
131
static struct dentry *hzip_debugfs_root;
132

133
struct hisi_zip_hw_error {
134
	u32 int_msk;
135
	const char *msg;
136
};
137

138
struct zip_dfx_item {
139
	const char *name;
140
	u32 offset;
141
};
142

143
static const struct qm_dev_alg zip_dev_algs[] = { {
144
		.alg_msk = HZIP_ALG_ZLIB_BIT,
145
		.alg = "zlib\n",
146
	}, {
147
		.alg_msk = HZIP_ALG_GZIP_BIT,
148
		.alg = "gzip\n",
149
	}, {
150
		.alg_msk = HZIP_ALG_DEFLATE_BIT,
151
		.alg = "deflate\n",
152
	}, {
153
		.alg_msk = HZIP_ALG_LZ77_BIT,
154
		.alg = "lz77_zstd\n",
155
	}, {
156
		.alg_msk = HZIP_ALG_LZ77_BIT,
157
		.alg = "lz77_only\n",
158
	}, {
159
		.alg_msk = HZIP_ALG_LZ4_BIT,
160
		.alg = "lz4\n",
161
	},
162
};
163

164
static struct hisi_qm_list zip_devices = {
165
	.register_to_crypto	= hisi_zip_register_to_crypto,
166
	.unregister_from_crypto	= hisi_zip_unregister_from_crypto,
167
};
168

169
static struct zip_dfx_item zip_dfx_files[] = {
170
	{"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)},
171
	{"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)},
172
	{"send_busy_cnt", offsetof(struct hisi_zip_dfx, send_busy_cnt)},
173
	{"err_bd_cnt", offsetof(struct hisi_zip_dfx, err_bd_cnt)},
174
};
175

176
static const struct hisi_zip_hw_error zip_hw_error[] = {
177
	{ .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" },
178
	{ .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" },
179
	{ .int_msk = BIT(2), .msg = "zip_axi_rresp_err" },
180
	{ .int_msk = BIT(3), .msg = "zip_axi_bresp_err" },
181
	{ .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" },
182
	{ .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" },
183
	{ .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" },
184
	{ .int_msk = BIT(7), .msg = "zip_pre_in_data_err" },
185
	{ .int_msk = BIT(8), .msg = "zip_com_inf_err" },
186
	{ .int_msk = BIT(9), .msg = "zip_enc_inf_err" },
187
	{ .int_msk = BIT(10), .msg = "zip_pre_out_err" },
188
	{ .int_msk = BIT(11), .msg = "zip_axi_poison_err" },
189
	{ .int_msk = BIT(12), .msg = "zip_sva_err" },
190
	{ /* sentinel */ }
191
};
192

193
enum ctrl_debug_file_index {
194
	HZIP_CLEAR_ENABLE,
195
	HZIP_DEBUG_FILE_NUM,
196
};
197

198
static const char * const ctrl_debug_file_name[] = {
199
	[HZIP_CLEAR_ENABLE] = "clear_enable",
200
};
201

202
struct ctrl_debug_file {
203
	enum ctrl_debug_file_index index;
204
	spinlock_t lock;
205
	struct hisi_zip_ctrl *ctrl;
206
};
207

208
/*
209
 * One ZIP controller has one PF and multiple VFs, some global configurations
210
 * which PF has need this structure.
211
 *
212
 * Just relevant for PF.
213
 */
214
struct hisi_zip_ctrl {
215
	struct hisi_zip *hisi_zip;
216
	struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];
217
};
218

219
enum zip_cap_type {
220
	ZIP_QM_NFE_MASK_CAP = 0x0,
221
	ZIP_QM_RESET_MASK_CAP,
222
	ZIP_QM_OOO_SHUTDOWN_MASK_CAP,
223
	ZIP_QM_CE_MASK_CAP,
224
	ZIP_NFE_MASK_CAP,
225
	ZIP_RESET_MASK_CAP,
226
	ZIP_OOO_SHUTDOWN_MASK_CAP,
227
	ZIP_CE_MASK_CAP,
228
	ZIP_CLUSTER_NUM_CAP,
229
	ZIP_CORE_TYPE_NUM_CAP,
230
	ZIP_CORE_NUM_CAP,
231
	ZIP_CLUSTER_COMP_NUM_CAP,
232
	ZIP_CLUSTER_DECOMP_NUM_CAP,
233
	ZIP_DECOMP_ENABLE_BITMAP,
234
	ZIP_COMP_ENABLE_BITMAP,
235
	ZIP_DRV_ALG_BITMAP,
236
	ZIP_DEV_ALG_BITMAP,
237
	ZIP_CORE1_ALG_BITMAP,
238
	ZIP_CORE2_ALG_BITMAP,
239
	ZIP_CORE3_ALG_BITMAP,
240
	ZIP_CORE4_ALG_BITMAP,
241
	ZIP_CORE5_ALG_BITMAP,
242
	ZIP_CAP_MAX
243
};
244

245
static struct hisi_qm_cap_info zip_basic_cap_info[] = {
246
	{ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77},
247
	{ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77},
248
	{ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77},
249
	{ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},
250
	{ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE},
251
	{ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE},
252
	{ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE},
253
	{ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1},
254
	{ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1},
255
	{ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2},
256
	{ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5},
257
	{ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2},
258
	{ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3},
259
	{ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C},
260
	{ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3},
261
	{ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x0, 0x30},
262
	{ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0x3F},
263
	{ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},
264
	{ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},
265
	{ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
266
	{ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
267
	{ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
268
	{ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0}
269
};
270

271
static const struct hisi_qm_cap_query_info zip_cap_query_info[] = {
272
	{QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE             ", 0x3124, 0x0, 0x1C57, 0x7C77},
273
	{QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET            ", 0x3128, 0x0, 0xC57, 0x6C77},
274
	{QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE              ", 0x312C, 0x0, 0x8, 0x8},
275
	{ZIP_RAS_NFE_TYPE, "ZIP_RAS_NFE_TYPE            ", 0x3130, 0x0, 0x7FE, 0x1FFE},
276
	{ZIP_RAS_NFE_RESET, "ZIP_RAS_NFE_RESET           ", 0x3134, 0x0, 0x7FE, 0x7FE},
277
	{ZIP_RAS_CE_TYPE, "ZIP_RAS_CE_TYPE             ", 0x3138, 0x0, 0x1, 0x1},
278
	{ZIP_CORE_INFO, "ZIP_CORE_INFO               ", 0x313C, 0x12080206, 0x12080206, 0x12050203},
279
	{ZIP_CORE_EN, "ZIP_CORE_EN                 ", 0x3140, 0xFC0003, 0xFC0003, 0x1C0003},
280
	{ZIP_DRV_ALG_BITMAP_TB, "ZIP_DRV_ALG_BITMAP          ", 0x3144, 0x0, 0x0, 0x30},
281
	{ZIP_ALG_BITMAP, "ZIP_ALG_BITMAP              ", 0x3148, 0xF, 0xF, 0x3F},
282
	{ZIP_CORE1_BITMAP, "ZIP_CORE1_BITMAP            ", 0x314C, 0x5, 0x5, 0xD5},
283
	{ZIP_CORE2_BITMAP, "ZIP_CORE2_BITMAP            ", 0x3150, 0x5, 0x5, 0xD5},
284
	{ZIP_CORE3_BITMAP, "ZIP_CORE3_BITMAP            ", 0x3154, 0xA, 0xA, 0x2A},
285
	{ZIP_CORE4_BITMAP, "ZIP_CORE4_BITMAP            ", 0x3158, 0xA, 0xA, 0x2A},
286
	{ZIP_CORE5_BITMAP, "ZIP_CORE5_BITMAP            ", 0x315C, 0xA, 0xA, 0x2A},
287
};
288

289
static const struct debugfs_reg32 hzip_dfx_regs[] = {
290
	{"HZIP_GET_BD_NUM                ",  0x00},
291
	{"HZIP_GET_RIGHT_BD              ",  0x04},
292
	{"HZIP_GET_ERROR_BD              ",  0x08},
293
	{"HZIP_DONE_BD_NUM               ",  0x0c},
294
	{"HZIP_WORK_CYCLE                ",  0x10},
295
	{"HZIP_IDLE_CYCLE                ",  0x18},
296
	{"HZIP_MAX_DELAY                 ",  0x20},
297
	{"HZIP_MIN_DELAY                 ",  0x24},
298
	{"HZIP_AVG_DELAY                 ",  0x28},
299
	{"HZIP_MEM_VISIBLE_DATA          ",  0x30},
300
	{"HZIP_MEM_VISIBLE_ADDR          ",  0x34},
301
	{"HZIP_CONSUMED_BYTE             ",  0x38},
302
	{"HZIP_PRODUCED_BYTE             ",  0x40},
303
	{"HZIP_COMP_INF                  ",  0x70},
304
	{"HZIP_PRE_OUT                   ",  0x78},
305
	{"HZIP_BD_RD                     ",  0x7c},
306
	{"HZIP_BD_WR                     ",  0x80},
307
	{"HZIP_GET_BD_AXI_ERR_NUM        ",  0x84},
308
	{"HZIP_GET_BD_PARSE_ERR_NUM      ",  0x88},
309
	{"HZIP_ADD_BD_AXI_ERR_NUM        ",  0x8c},
310
	{"HZIP_DECOMP_STF_RELOAD_CURR_ST ",  0x94},
311
	{"HZIP_DECOMP_LZ77_CURR_ST       ",  0x9c},
312
};
313

314
static const struct debugfs_reg32 hzip_com_dfx_regs[] = {
315
	{"HZIP_CLOCK_GATE_CTRL           ",  0x301004},
316
	{"HZIP_CORE_INT_RAS_CE_ENB       ",  0x301160},
317
	{"HZIP_CORE_INT_RAS_NFE_ENB      ",  0x301164},
318
	{"HZIP_CORE_INT_RAS_FE_ENB       ",  0x301168},
319
	{"HZIP_UNCOM_ERR_RAS_CTRL        ",  0x30116C},
320
};
321

322
static const struct debugfs_reg32 hzip_dump_dfx_regs[] = {
323
	{"HZIP_GET_BD_NUM                ",  0x00},
324
	{"HZIP_GET_RIGHT_BD              ",  0x04},
325
	{"HZIP_GET_ERROR_BD              ",  0x08},
326
	{"HZIP_DONE_BD_NUM               ",  0x0c},
327
	{"HZIP_MAX_DELAY                 ",  0x20},
328
};
329

330
/* define the ZIP's dfx regs region and region length */
331
static struct dfx_diff_registers hzip_diff_regs[] = {
332
	{
333
		.reg_offset = HZIP_CORE_DFX_BASE,
334
		.reg_len = HZIP_CORE_REGS_BASE_LEN,
335
	}, {
336
		.reg_offset = HZIP_CORE_DFX_COMP_0,
337
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
338
	}, {
339
		.reg_offset = HZIP_CORE_DFX_COMP_1,
340
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
341
	}, {
342
		.reg_offset = HZIP_CORE_DFX_DECOMP_0,
343
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
344
	}, {
345
		.reg_offset = HZIP_CORE_DFX_DECOMP_1,
346
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
347
	}, {
348
		.reg_offset = HZIP_CORE_DFX_DECOMP_2,
349
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
350
	}, {
351
		.reg_offset = HZIP_CORE_DFX_DECOMP_3,
352
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
353
	}, {
354
		.reg_offset = HZIP_CORE_DFX_DECOMP_4,
355
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
356
	}, {
357
		.reg_offset = HZIP_CORE_DFX_DECOMP_5,
358
		.reg_len = HZIP_CORE_REGS_DFX_LEN,
359
	},
360
};
361

362
static int hzip_diff_regs_show(struct seq_file *s, void *unused)
363
{
364
	struct hisi_qm *qm = s->private;
365

366
	hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,
367
					ARRAY_SIZE(hzip_diff_regs));
368

369
	return 0;
370
}
371
DEFINE_SHOW_ATTRIBUTE(hzip_diff_regs);
372

373
static int perf_mode_set(const char *val, const struct kernel_param *kp)
374
{
375
	int ret;
376
	u32 n;
377

378
	if (!val)
379
		return -EINVAL;
380

381
	ret = kstrtou32(val, 10, &n);
382
	if (ret != 0 || (n != HZIP_HIGH_COMP_PERF &&
383
			 n != HZIP_HIGH_COMP_RATE))
384
		return -EINVAL;
385

386
	return param_set_int(val, kp);
387
}
388

389
static const struct kernel_param_ops zip_com_perf_ops = {
390
	.set = perf_mode_set,
391
	.get = param_get_int,
392
};
393

394
/*
395
 * perf_mode = 0 means enable high compression rate mode,
396
 * perf_mode = 1 means enable high compression performance mode.
397
 * These two modes only apply to the compression direction.
398
 */
399
static u32 perf_mode = HZIP_HIGH_COMP_RATE;
400
module_param_cb(perf_mode, &zip_com_perf_ops, &perf_mode, 0444);
401
MODULE_PARM_DESC(perf_mode, "ZIP high perf mode 0(default), 1(enable)");
402

403
static const struct kernel_param_ops zip_uacce_mode_ops = {
404
	.set = uacce_mode_set,
405
	.get = param_get_int,
406
};
407

408
/*
409
 * uacce_mode = 0 means zip only register to crypto,
410
 * uacce_mode = 1 means zip both register to crypto and uacce.
411
 */
412
static u32 uacce_mode = UACCE_MODE_NOUACCE;
413
module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444);
414
MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
415

416
static bool pf_q_num_flag;
417
static int pf_q_num_set(const char *val, const struct kernel_param *kp)
418
{
419
	pf_q_num_flag = true;
420

421
	return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF);
422
}
423

424
static const struct kernel_param_ops pf_q_num_ops = {
425
	.set = pf_q_num_set,
426
	.get = param_get_int,
427
};
428

429
static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;
430
module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);
431
MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
432

433
static const struct kernel_param_ops vfs_num_ops = {
434
	.set = vfs_num_set,
435
	.get = param_get_int,
436
};
437

438
static u32 vfs_num;
439
module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
440
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
441

442
static const struct pci_device_id hisi_zip_dev_ids[] = {
443
	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_PF) },
444
	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
445
	{ 0, }
446
};
447
MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids);
448

449
int zip_create_qps(struct hisi_qp **qps, int qp_num, int node)
450
{
451
	if (node == NUMA_NO_NODE)
452
		node = cpu_to_node(raw_smp_processor_id());
453

454
	return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps);
455
}
456

457
bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg)
458
{
459
	u32 cap_val;
460

461
	cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_TB].cap_val;
462
	if ((alg & cap_val) == alg)
463
		return true;
464

465
	return false;
466
}
467

468
static void hisi_zip_literal_set(struct hisi_qm *qm)
469
{
470
	u32 val;
471

472
	if (qm->ver < QM_HW_V3)
473
		return;
474

475
	val = readl_relaxed(qm->io_base + HZIP_LIT_LEN_EN_OFFSET);
476
	val &= ~HZIP_LIT_LEN_EN_EN;
477

478
	/* enable literal length in stream mode compression */
479
	writel(val, qm->io_base + HZIP_LIT_LEN_EN_OFFSET);
480
}
481

482
static void hisi_zip_set_high_perf(struct hisi_qm *qm)
483
{
484
	u32 val;
485

486
	val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET);
487
	if (perf_mode == HZIP_HIGH_COMP_PERF)
488
		val |= HZIP_HIGH_COMP_PERF;
489
	else
490
		val &= ~HZIP_HIGH_COMP_PERF;
491

492
	/* Set perf mode */
493
	writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET);
494
}
495

496
static int hisi_zip_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask)
497
{
498
	u32 val, try_times = 0;
499
	u8 count = 0;
500

501
	/*
502
	 * Read the register value every 10-20us. If the value is 0 for three
503
	 * consecutive times, the SVA module is ready.
504
	 */
505
	do {
506
		val = readl(qm->io_base + offset);
507
		if (val & mask)
508
			count = 0;
509
		else if (++count == HZIP_READ_SVA_STATUS_TIMES)
510
			break;
511

512
		usleep_range(HZIP_WAIT_US_MIN, HZIP_WAIT_US_MAX);
513
	} while (++try_times < HZIP_WAIT_SVA_READY);
514

515
	if (try_times == HZIP_WAIT_SVA_READY) {
516
		pci_err(qm->pdev, "failed to wait sva prefetch ready\n");
517
		return -ETIMEDOUT;
518
	}
519

520
	return 0;
521
}
522

523
static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm)
524
{
525
	u32 val;
526
	int ret;
527

528
	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
529
		return;
530

531
	val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
532
	val |= HZIP_SVA_PREFETCH_DISABLE;
533
	writel(val, qm->io_base + HZIP_PREFETCH_CFG);
534

535
	ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS,
536
					 val, !(val & HZIP_SVA_DISABLE_READY),
537
					 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
538
	if (ret)
539
		pci_err(qm->pdev, "failed to close sva prefetch\n");
540

541
	(void)hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_STALL_NUM);
542
}
543

544
static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm)
545
{
546
	u32 val;
547
	int ret;
548

549
	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
550
		return;
551

552
	/* Enable prefetch */
553
	val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
554
	val &= HZIP_PREFETCH_ENABLE;
555
	writel(val, qm->io_base + HZIP_PREFETCH_CFG);
556

557
	ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG,
558
					 val, !(val & HZIP_SVA_PREFETCH_DISABLE),
559
					 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
560
	if (ret) {
561
		pci_err(qm->pdev, "failed to open sva prefetch\n");
562
		hisi_zip_close_sva_prefetch(qm);
563
		return;
564
	}
565

566
	ret = hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_PREFETCH_NUM);
567
	if (ret)
568
		hisi_zip_close_sva_prefetch(qm);
569
}
570

571
static void hisi_zip_enable_clock_gate(struct hisi_qm *qm)
572
{
573
	u32 val;
574

575
	if (qm->ver < QM_HW_V3)
576
		return;
577

578
	val = readl(qm->io_base + HZIP_CLOCK_GATE_CTRL);
579
	val |= HZIP_CLOCK_GATED_EN;
580
	writel(val, qm->io_base + HZIP_CLOCK_GATE_CTRL);
581

582
	val = readl(qm->io_base + HZIP_PEH_CFG_AUTO_GATE);
583
	val |= HZIP_PEH_CFG_AUTO_GATE_EN;
584
	writel(val, qm->io_base + HZIP_PEH_CFG_AUTO_GATE);
585
}
586

587
static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
588
{
589
	void __iomem *base = qm->io_base;
590
	u32 dcomp_bm, comp_bm;
591
	u32 zip_core_en;
592
	int ret;
593

594
	/* qm user domain */
595
	writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1);
596
	writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE);
597
	writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1);
598
	writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE);
599
	writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE);
600

601
	/* qm cache */
602
	writel(AXI_M_CFG, base + QM_AXI_M_CFG);
603
	writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE);
604

605
	/* disable FLR triggered by BME(bus master enable) */
606
	writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG);
607
	writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE);
608

609
	/* cache */
610
	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);
611
	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);
612
	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);
613
	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);
614

615
	/* user domain configurations */
616
	writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
617
	writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
618

619
	if (qm->use_sva && qm->ver == QM_HW_V2) {
620
		writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
621
		writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
622
		writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);
623
	} else {
624
		writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
625
		writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
626
		writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
627
	}
628
	hisi_zip_open_sva_prefetch(qm);
629

630
	/* let's open all compression/decompression cores */
631

632
	zip_core_en = qm->cap_tables.dev_cap_table[ZIP_CORE_EN].cap_val;
633
	dcomp_bm = (zip_core_en >> zip_basic_cap_info[ZIP_DECOMP_ENABLE_BITMAP].shift) &
634
			zip_basic_cap_info[ZIP_DECOMP_ENABLE_BITMAP].mask;
635
	comp_bm = (zip_core_en >> zip_basic_cap_info[ZIP_COMP_ENABLE_BITMAP].shift) &
636
			zip_basic_cap_info[ZIP_COMP_ENABLE_BITMAP].mask;
637
	writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL);
638

639
	/* enable sqc,cqc writeback */
640
	writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
641
	       CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
642
	       FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);
643

644
	hisi_zip_set_high_perf(qm);
645
	hisi_zip_literal_set(qm);
646
	hisi_zip_enable_clock_gate(qm);
647

648
	ret = hisi_dae_set_user_domain(qm);
649
	if (ret)
650
		goto close_sva_prefetch;
651

652
	return 0;
653

654
close_sva_prefetch:
655
	hisi_zip_close_sva_prefetch(qm);
656
	return ret;
657
}
658

659
static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
660
{
661
	u32 val1, val2;
662

663
	val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
664
	if (enable) {
665
		val1 |= HZIP_AXI_SHUTDOWN_ENABLE;
666
		val2 = qm->err_info.dev_err.shutdown_mask;
667
	} else {
668
		val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE;
669
		val2 = 0x0;
670
	}
671

672
	if (qm->ver > QM_HW_V2)
673
		writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL);
674

675
	writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
676
}
677

678
static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
679
{
680
	struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
681
	u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
682

683
	if (qm->ver == QM_HW_V1) {
684
		writel(HZIP_CORE_INT_MASK_ALL,
685
		       qm->io_base + HZIP_CORE_INT_MASK_REG);
686
		dev_info(&qm->pdev->dev, "Does not support hw error handle\n");
687
		return;
688
	}
689

690
	/* clear ZIP hw error source if having */
691
	writel(err_mask, qm->io_base + HZIP_CORE_INT_SOURCE);
692

693
	/* configure error type */
694
	writel(dev_err->ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB);
695
	writel(dev_err->fe, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB);
696
	writel(dev_err->nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
697

698
	hisi_zip_master_ooo_ctrl(qm, true);
699

700
	/* enable ZIP hw error interrupts */
701
	writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG);
702

703
	hisi_dae_hw_error_enable(qm);
704
}
705

706
static void hisi_zip_hw_error_disable(struct hisi_qm *qm)
707
{
708
	struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
709
	u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
710

711
	/* disable ZIP hw error interrupts */
712
	writel(err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG);
713

714
	hisi_zip_master_ooo_ctrl(qm, false);
715

716
	hisi_dae_hw_error_disable(qm);
717
}
718

719
static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
720
{
721
	struct hisi_zip *hisi_zip = file->ctrl->hisi_zip;
722

723
	return &hisi_zip->qm;
724
}
725

726
static u32 clear_enable_read(struct hisi_qm *qm)
727
{
728
	return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
729
		     HZIP_SOFT_CTRL_CNT_CLR_CE_BIT;
730
}
731

732
static int clear_enable_write(struct hisi_qm *qm, u32 val)
733
{
734
	u32 tmp;
735

736
	if (val != 1 && val != 0)
737
		return -EINVAL;
738

739
	tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
740
	       ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val;
741
	writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
742

743
	return  0;
744
}
745

746
static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf,
747
					size_t count, loff_t *pos)
748
{
749
	struct ctrl_debug_file *file = filp->private_data;
750
	struct hisi_qm *qm = file_to_qm(file);
751
	char tbuf[HZIP_BUF_SIZE];
752
	u32 val;
753
	int ret;
754

755
	ret = hisi_qm_get_dfx_access(qm);
756
	if (ret)
757
		return ret;
758

759
	spin_lock_irq(&file->lock);
760
	switch (file->index) {
761
	case HZIP_CLEAR_ENABLE:
762
		val = clear_enable_read(qm);
763
		break;
764
	default:
765
		goto err_input;
766
	}
767
	spin_unlock_irq(&file->lock);
768

769
	hisi_qm_put_dfx_access(qm);
770
	ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val);
771
	return simple_read_from_buffer(buf, count, pos, tbuf, ret);
772

773
err_input:
774
	spin_unlock_irq(&file->lock);
775
	hisi_qm_put_dfx_access(qm);
776
	return -EINVAL;
777
}
778

779
static ssize_t hisi_zip_ctrl_debug_write(struct file *filp,
780
					 const char __user *buf,
781
					 size_t count, loff_t *pos)
782
{
783
	struct ctrl_debug_file *file = filp->private_data;
784
	struct hisi_qm *qm = file_to_qm(file);
785
	char tbuf[HZIP_BUF_SIZE];
786
	unsigned long val;
787
	int len, ret;
788

789
	if (*pos != 0)
790
		return 0;
791

792
	if (count >= HZIP_BUF_SIZE)
793
		return -ENOSPC;
794

795
	len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count);
796
	if (len < 0)
797
		return len;
798

799
	tbuf[len] = '\0';
800
	ret = kstrtoul(tbuf, 0, &val);
801
	if (ret)
802
		return ret;
803

804
	ret = hisi_qm_get_dfx_access(qm);
805
	if (ret)
806
		return ret;
807

808
	spin_lock_irq(&file->lock);
809
	switch (file->index) {
810
	case HZIP_CLEAR_ENABLE:
811
		ret = clear_enable_write(qm, val);
812
		if (ret)
813
			goto err_input;
814
		break;
815
	default:
816
		ret = -EINVAL;
817
		goto err_input;
818
	}
819

820
	ret = count;
821

822
err_input:
823
	spin_unlock_irq(&file->lock);
824
	hisi_qm_put_dfx_access(qm);
825
	return ret;
826
}
827

828
static const struct file_operations ctrl_debug_fops = {
829
	.owner = THIS_MODULE,
830
	.open = simple_open,
831
	.read = hisi_zip_ctrl_debug_read,
832
	.write = hisi_zip_ctrl_debug_write,
833
};
834

835
static int zip_debugfs_atomic64_set(void *data, u64 val)
836
{
837
	if (val)
838
		return -EINVAL;
839

840
	atomic64_set((atomic64_t *)data, 0);
841

842
	return 0;
843
}
844

845
static int zip_debugfs_atomic64_get(void *data, u64 *val)
846
{
847
	*val = atomic64_read((atomic64_t *)data);
848

849
	return 0;
850
}
851

852
DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get,
853
			 zip_debugfs_atomic64_set, "%llu\n");
854

855
static int hisi_zip_regs_show(struct seq_file *s, void *unused)
856
{
857
	hisi_qm_regs_dump(s, s->private);
858

859
	return 0;
860
}
861

862
DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs);
863

864
static void __iomem *get_zip_core_addr(struct hisi_qm *qm, int core_num)
865
{
866
	u8 zip_comp_core_num;
867
	u32 zip_core_info;
868

869
	zip_core_info =  qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val;
870
	zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) &
871
			     zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask;
872

873
	if (core_num < zip_comp_core_num)
874
		return qm->io_base + HZIP_CORE_DFX_BASE +
875
			(core_num + 1) * HZIP_CORE_ADDR_INTRVL;
876

877
	return qm->io_base + HZIP_CORE_DFX_DECOMP_BASE +
878
		(core_num - zip_comp_core_num) * HZIP_CORE_ADDR_INTRVL;
879
}
880

881
static int hisi_zip_core_debug_init(struct hisi_qm *qm)
882
{
883
	u32 zip_core_num, zip_comp_core_num;
884
	struct device *dev = &qm->pdev->dev;
885
	struct debugfs_regset32 *regset;
886
	u32 zip_core_info;
887
	struct dentry *tmp_d;
888
	char buf[HZIP_BUF_SIZE];
889
	int i;
890

891
	zip_core_info =  qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val;
892
	zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) &
893
			zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask;
894
	zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) &
895
			zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask;
896

897
	for (i = 0; i < zip_core_num; i++) {
898
		if (i < zip_comp_core_num)
899
			scnprintf(buf, sizeof(buf), "comp_core%d", i);
900
		else
901
			scnprintf(buf, sizeof(buf), "decomp_core%d",
902
				  i - zip_comp_core_num);
903

904
		regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
905
		if (!regset)
906
			return -ENOENT;
907

908
		regset->regs = hzip_dfx_regs;
909
		regset->nregs = ARRAY_SIZE(hzip_dfx_regs);
910
		regset->base = get_zip_core_addr(qm, i);
911
		regset->dev = dev;
912

913
		tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
914
		debugfs_create_file("regs", 0444, tmp_d, regset,
915
				    &hisi_zip_regs_fops);
916
	}
917

918
	return 0;
919
}
920

921
static int zip_cap_regs_show(struct seq_file *s, void *unused)
922
{
923
	struct hisi_qm *qm = s->private;
924
	u32 i, size;
925

926
	size = qm->cap_tables.qm_cap_size;
927
	for (i = 0; i < size; i++)
928
		seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name,
929
			   qm->cap_tables.qm_cap_table[i].cap_val);
930

931
	size = qm->cap_tables.dev_cap_size;
932
	for (i = 0; i < size; i++)
933
		seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name,
934
			   qm->cap_tables.dev_cap_table[i].cap_val);
935

936
	return 0;
937
}
938

939
DEFINE_SHOW_ATTRIBUTE(zip_cap_regs);
940

941
static void hisi_zip_dfx_debug_init(struct hisi_qm *qm)
942
{
943
	struct dfx_diff_registers *hzip_regs = qm->debug.acc_diff_regs;
944
	struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
945
	struct hisi_zip_dfx *dfx = &zip->dfx;
946
	struct dentry *tmp_dir;
947
	void *data;
948
	int i;
949

950
	tmp_dir = debugfs_create_dir("zip_dfx", qm->debug.debug_root);
951
	for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) {
952
		data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset);
953
		debugfs_create_file(zip_dfx_files[i].name,
954
				    0644, tmp_dir, data,
955
				    &zip_atomic64_ops);
956
	}
957

958
	if (qm->fun_type == QM_HW_PF && hzip_regs)
959
		debugfs_create_file("diff_regs", 0444, tmp_dir,
960
				      qm, &hzip_diff_regs_fops);
961

962
	debugfs_create_file("cap_regs", CAP_FILE_PERMISSION,
963
			    qm->debug.debug_root, qm, &zip_cap_regs_fops);
964
}
965

966
static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm)
967
{
968
	struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
969
	int i;
970

971
	for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) {
972
		spin_lock_init(&zip->ctrl->files[i].lock);
973
		zip->ctrl->files[i].ctrl = zip->ctrl;
974
		zip->ctrl->files[i].index = i;
975

976
		debugfs_create_file(ctrl_debug_file_name[i], 0600,
977
				    qm->debug.debug_root,
978
				    zip->ctrl->files + i,
979
				    &ctrl_debug_fops);
980
	}
981

982
	return hisi_zip_core_debug_init(qm);
983
}
984

985
static int hisi_zip_debugfs_init(struct hisi_qm *qm)
986
{
987
	struct device *dev = &qm->pdev->dev;
988
	int ret;
989

990
	ret = hisi_qm_regs_debugfs_init(qm, hzip_diff_regs, ARRAY_SIZE(hzip_diff_regs));
991
	if (ret) {
992
		dev_warn(dev, "Failed to init ZIP diff regs!\n");
993
		return ret;
994
	}
995

996
	qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET;
997
	qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN;
998
	qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
999
							hzip_debugfs_root);
1000

1001
	hisi_qm_debug_init(qm);
1002

1003
	if (qm->fun_type == QM_HW_PF) {
1004
		ret = hisi_zip_ctrl_debug_init(qm);
1005
		if (ret)
1006
			goto debugfs_remove;
1007
	}
1008

1009
	hisi_zip_dfx_debug_init(qm);
1010

1011
	return 0;
1012

1013
debugfs_remove:
1014
	debugfs_remove_recursive(qm->debug.debug_root);
1015
	hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));
1016
	return ret;
1017
}
1018

1019
/* hisi_zip_debug_regs_clear() - clear the zip debug regs */
1020
static void hisi_zip_debug_regs_clear(struct hisi_qm *qm)
1021
{
1022
	u32 zip_core_info;
1023
	u8 zip_core_num;
1024
	int i, j;
1025

1026
	zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val;
1027
	zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) &
1028
			zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask;
1029

1030
	/* enable register read_clear bit */
1031
	writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
1032
	for (i = 0; i < zip_core_num; i++)
1033
		for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++)
1034
			readl(get_zip_core_addr(qm, i) +
1035
			      hzip_dfx_regs[j].offset);
1036

1037
	/* disable register read_clear bit */
1038
	writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
1039

1040
	hisi_qm_debug_regs_clear(qm);
1041
}
1042

1043
static void hisi_zip_debugfs_exit(struct hisi_qm *qm)
1044
{
1045
	debugfs_remove_recursive(qm->debug.debug_root);
1046

1047
	hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));
1048

1049
	if (qm->fun_type == QM_HW_PF) {
1050
		hisi_zip_debug_regs_clear(qm);
1051
		qm->debug.curr_qm_qp_num = 0;
1052
	}
1053
}
1054

1055
static int hisi_zip_show_last_regs_init(struct hisi_qm *qm)
1056
{
1057
	int core_dfx_regs_num =  ARRAY_SIZE(hzip_dump_dfx_regs);
1058
	int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);
1059
	struct qm_debug *debug = &qm->debug;
1060
	void __iomem *io_base;
1061
	u32 zip_core_info;
1062
	u32 zip_core_num;
1063
	int i, j, idx;
1064

1065
	zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val;
1066
	zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) &
1067
			zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask;
1068

1069
	debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num,
1070
				    sizeof(unsigned int), GFP_KERNEL);
1071
	if (!debug->last_words)
1072
		return -ENOMEM;
1073

1074
	for (i = 0; i < com_dfx_regs_num; i++) {
1075
		io_base = qm->io_base + hzip_com_dfx_regs[i].offset;
1076
		debug->last_words[i] = readl_relaxed(io_base);
1077
	}
1078

1079
	for (i = 0; i < zip_core_num; i++) {
1080
		io_base = get_zip_core_addr(qm, i);
1081
		for (j = 0; j < core_dfx_regs_num; j++) {
1082
			idx = com_dfx_regs_num + i * core_dfx_regs_num + j;
1083
			debug->last_words[idx] = readl_relaxed(
1084
				io_base + hzip_dump_dfx_regs[j].offset);
1085
		}
1086
	}
1087

1088
	return 0;
1089
}
1090

1091
static void hisi_zip_show_last_regs_uninit(struct hisi_qm *qm)
1092
{
1093
	struct qm_debug *debug = &qm->debug;
1094

1095
	if (qm->fun_type == QM_HW_VF || !debug->last_words)
1096
		return;
1097

1098
	kfree(debug->last_words);
1099
	debug->last_words = NULL;
1100
}
1101

1102
static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm)
1103
{
1104
	int core_dfx_regs_num =  ARRAY_SIZE(hzip_dump_dfx_regs);
1105
	int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);
1106
	u32 zip_core_num, zip_comp_core_num;
1107
	struct qm_debug *debug = &qm->debug;
1108
	char buf[HZIP_BUF_SIZE];
1109
	u32 zip_core_info;
1110
	void __iomem *base;
1111
	int i, j, idx;
1112
	u32 val;
1113

1114
	if (qm->fun_type == QM_HW_VF || !debug->last_words)
1115
		return;
1116

1117
	for (i = 0; i < com_dfx_regs_num; i++) {
1118
		val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset);
1119
		if (debug->last_words[i] != val)
1120
			pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n",
1121
				 hzip_com_dfx_regs[i].name, debug->last_words[i], val);
1122
	}
1123

1124
	zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val;
1125
	zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) &
1126
			zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask;
1127
	zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) &
1128
			zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask;
1129

1130
	for (i = 0; i < zip_core_num; i++) {
1131
		if (i < zip_comp_core_num)
1132
			scnprintf(buf, sizeof(buf), "Comp_core-%d", i);
1133
		else
1134
			scnprintf(buf, sizeof(buf), "Decomp_core-%d",
1135
				  i - zip_comp_core_num);
1136
		base = get_zip_core_addr(qm, i);
1137

1138
		pci_info(qm->pdev, "==>%s:\n", buf);
1139
		/* dump last word for dfx regs during control resetting */
1140
		for (j = 0; j < core_dfx_regs_num; j++) {
1141
			idx = com_dfx_regs_num + i * core_dfx_regs_num + j;
1142
			val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset);
1143
			if (debug->last_words[idx] != val)
1144
				pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n",
1145
					 hzip_dump_dfx_regs[j].name,
1146
					 debug->last_words[idx], val);
1147
		}
1148
	}
1149
}
1150

1151
static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts)
1152
{
1153
	const struct hisi_zip_hw_error *err = zip_hw_error;
1154
	struct device *dev = &qm->pdev->dev;
1155
	u32 err_val;
1156

1157
	while (err->msg) {
1158
		if (err->int_msk & err_sts) {
1159
			dev_err(dev, "%s [error status=0x%x] found\n",
1160
				err->msg, err->int_msk);
1161

1162
			if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) {
1163
				err_val = readl(qm->io_base +
1164
						HZIP_CORE_SRAM_ECC_ERR_INFO);
1165
				dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n",
1166
					((err_val >>
1167
					HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF));
1168
			}
1169
		}
1170
		err++;
1171
	}
1172
}
1173

1174
static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm)
1175
{
1176
	return readl(qm->io_base + HZIP_CORE_INT_STATUS);
1177
}
1178

1179
static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
1180
{
1181
	writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE);
1182
}
1183

1184
static void hisi_zip_disable_error_report(struct hisi_qm *qm, u32 err_type)
1185
{
1186
	u32 nfe_mask = qm->err_info.dev_err.nfe;
1187

1188
	writel(nfe_mask & (~err_type), qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1189
}
1190

1191
static void hisi_zip_enable_error_report(struct hisi_qm *qm)
1192
{
1193
	u32 nfe_mask = qm->err_info.dev_err.nfe;
1194
	u32 ce_mask = qm->err_info.dev_err.ce;
1195

1196
	writel(nfe_mask, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1197
	writel(ce_mask, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB);
1198
}
1199

1200
static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm)
1201
{
1202
	u32 val;
1203

1204
	val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1205

1206
	writel(val & ~HZIP_AXI_SHUTDOWN_ENABLE,
1207
	       qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1208

1209
	writel(val | HZIP_AXI_SHUTDOWN_ENABLE,
1210
	       qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1211

1212
	hisi_dae_open_axi_master_ooo(qm);
1213
}
1214

1215
static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm)
1216
{
1217
	u32 nfe_enb;
1218

1219
	/* Disable ECC Mbit error report. */
1220
	nfe_enb = readl(qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1221
	writel(nfe_enb & ~HZIP_CORE_INT_STATUS_M_ECC,
1222
	       qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1223

1224
	/* Inject zip ECC Mbit error to block master ooo. */
1225
	writel(HZIP_CORE_INT_STATUS_M_ECC,
1226
	       qm->io_base + HZIP_CORE_INT_SET);
1227
}
1228

1229
static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm)
1230
{
1231
	enum acc_err_result zip_result = ACC_ERR_NONE;
1232
	enum acc_err_result dae_result;
1233
	u32 err_status;
1234

1235
	/* Get device hardware new error status */
1236
	err_status = hisi_zip_get_hw_err_status(qm);
1237
	if (err_status) {
1238
		if (err_status & qm->err_info.dev_err.ecc_2bits_mask)
1239
			qm->err_status.is_dev_ecc_mbit = true;
1240
		hisi_zip_log_hw_error(qm, err_status);
1241

1242
		if (err_status & qm->err_info.dev_err.reset_mask) {
1243
			/* Disable the same error reporting until device is recovered. */
1244
			hisi_zip_disable_error_report(qm, err_status);
1245
			zip_result = ACC_ERR_NEED_RESET;
1246
		} else {
1247
			hisi_zip_clear_hw_err_status(qm, err_status);
1248
			/* Avoid firmware disable error report, re-enable. */
1249
			hisi_zip_enable_error_report(qm);
1250
		}
1251
	}
1252

1253
	dae_result = hisi_dae_get_err_result(qm);
1254

1255
	return (zip_result == ACC_ERR_NEED_RESET ||
1256
		dae_result == ACC_ERR_NEED_RESET) ?
1257
		ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
1258
}
1259

1260
static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm)
1261
{
1262
	u32 err_status;
1263

1264
	err_status = hisi_zip_get_hw_err_status(qm);
1265
	if (err_status & qm->err_info.dev_err.shutdown_mask)
1266
		return true;
1267

1268
	return hisi_dae_dev_is_abnormal(qm);
1269
}
1270

1271
static int hisi_zip_set_priv_status(struct hisi_qm *qm)
1272
{
1273
	return hisi_dae_close_axi_master_ooo(qm);
1274
}
1275

1276
static void hisi_zip_disable_axi_error(struct hisi_qm *qm)
1277
{
1278
	struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
1279
	u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
1280
	u32 val;
1281

1282
	val = ~(err_mask & (~HZIP_AXI_ERROR_MASK));
1283
	writel(val, qm->io_base + HZIP_CORE_INT_MASK_REG);
1284

1285
	if (qm->ver > QM_HW_V2)
1286
		writel(dev_err->shutdown_mask & (~HZIP_AXI_ERROR_MASK),
1287
		       qm->io_base + HZIP_OOO_SHUTDOWN_SEL);
1288
}
1289

1290
static void hisi_zip_enable_axi_error(struct hisi_qm *qm)
1291
{
1292
	struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
1293
	u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
1294

1295
	/* clear axi error source */
1296
	writel(HZIP_AXI_ERROR_MASK, qm->io_base + HZIP_CORE_INT_SOURCE);
1297

1298
	writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG);
1299

1300
	if (qm->ver > QM_HW_V2)
1301
		writel(dev_err->shutdown_mask, qm->io_base + HZIP_OOO_SHUTDOWN_SEL);
1302
}
1303

1304
static void hisi_zip_err_info_init(struct hisi_qm *qm)
1305
{
1306
	struct hisi_qm_err_info *err_info = &qm->err_info;
1307
	struct hisi_qm_err_mask *qm_err = &err_info->qm_err;
1308
	struct hisi_qm_err_mask *dev_err = &err_info->dev_err;
1309

1310
	qm_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK;
1311
	qm_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver);
1312
	qm_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1313
					  ZIP_QM_NFE_MASK_CAP, qm->cap_ver);
1314
	qm_err->ecc_2bits_mask = QM_ECC_MBIT;
1315
	qm_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1316
						 ZIP_QM_RESET_MASK_CAP, qm->cap_ver);
1317
	qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1318
						    ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1319

1320
	dev_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK;
1321
	dev_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);
1322
	dev_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
1323
	dev_err->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC;
1324
	dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1325
						     ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1326
	dev_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1327
						  ZIP_RESET_MASK_CAP, qm->cap_ver);
1328

1329
	err_info->msi_wr_port = HZIP_WR_PORT;
1330
	err_info->acpi_rst = "ZRST";
1331
}
1332

1333
static const struct hisi_qm_err_ini hisi_zip_err_ini = {
1334
	.hw_init		= hisi_zip_set_user_domain_and_cache,
1335
	.hw_err_enable		= hisi_zip_hw_error_enable,
1336
	.hw_err_disable		= hisi_zip_hw_error_disable,
1337
	.get_dev_hw_err_status	= hisi_zip_get_hw_err_status,
1338
	.clear_dev_hw_err_status = hisi_zip_clear_hw_err_status,
1339
	.open_axi_master_ooo	= hisi_zip_open_axi_master_ooo,
1340
	.close_axi_master_ooo	= hisi_zip_close_axi_master_ooo,
1341
	.open_sva_prefetch	= hisi_zip_open_sva_prefetch,
1342
	.close_sva_prefetch	= hisi_zip_close_sva_prefetch,
1343
	.show_last_dfx_regs	= hisi_zip_show_last_dfx_regs,
1344
	.err_info_init		= hisi_zip_err_info_init,
1345
	.get_err_result		= hisi_zip_get_err_result,
1346
	.set_priv_status	= hisi_zip_set_priv_status,
1347
	.dev_is_abnormal	= hisi_zip_dev_is_abnormal,
1348
	.disable_axi_error	= hisi_zip_disable_axi_error,
1349
	.enable_axi_error	= hisi_zip_enable_axi_error,
1350
};
1351

1352
static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
1353
{
1354
	struct hisi_qm *qm = &hisi_zip->qm;
1355
	struct hisi_zip_ctrl *ctrl;
1356
	int ret;
1357

1358
	ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL);
1359
	if (!ctrl)
1360
		return -ENOMEM;
1361

1362
	hisi_zip->ctrl = ctrl;
1363
	ctrl->hisi_zip = hisi_zip;
1364

1365
	ret = hisi_zip_set_user_domain_and_cache(qm);
1366
	if (ret)
1367
		return ret;
1368

1369
	hisi_qm_dev_err_init(qm);
1370
	hisi_zip_debug_regs_clear(qm);
1371

1372
	ret = hisi_zip_show_last_regs_init(qm);
1373
	if (ret)
1374
		pci_err(qm->pdev, "Failed to init last word regs!\n");
1375

1376
	return ret;
1377
}
1378

1379
static int zip_pre_store_cap_reg(struct hisi_qm *qm)
1380
{
1381
	struct hisi_qm_cap_record *zip_cap;
1382
	struct pci_dev *pdev = qm->pdev;
1383
	size_t i, size;
1384

1385
	size = ARRAY_SIZE(zip_cap_query_info);
1386
	zip_cap = devm_kcalloc(&pdev->dev, size, sizeof(*zip_cap), GFP_KERNEL);
1387
	if (!zip_cap)
1388
		return -ENOMEM;
1389

1390
	for (i = 0; i < size; i++) {
1391
		zip_cap[i].type = zip_cap_query_info[i].type;
1392
		zip_cap[i].name = zip_cap_query_info[i].name;
1393
		zip_cap[i].cap_val = hisi_qm_get_cap_value(qm, zip_cap_query_info,
1394
				     i, qm->cap_ver);
1395
	}
1396

1397
	qm->cap_tables.dev_cap_table = zip_cap;
1398
	qm->cap_tables.dev_cap_size = size;
1399

1400
	return 0;
1401
}
1402

1403
static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
1404
{
1405
	u64 alg_msk;
1406
	int ret;
1407

1408
	qm->pdev = pdev;
1409
	qm->mode = uacce_mode;
1410
	qm->sqe_size = HZIP_SQE_SIZE;
1411
	qm->dev_name = hisi_zip_name;
1412

1413
	qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_ZIP_PF) ?
1414
			QM_HW_PF : QM_HW_VF;
1415
	if (qm->fun_type == QM_HW_PF) {
1416
		qm->qp_base = HZIP_PF_DEF_Q_BASE;
1417
		qm->qp_num = pf_q_num;
1418
		qm->debug.curr_qm_qp_num = pf_q_num;
1419
		qm->qm_list = &zip_devices;
1420
		qm->err_ini = &hisi_zip_err_ini;
1421
		if (pf_q_num_flag)
1422
			set_bit(QM_MODULE_PARAM, &qm->misc_ctl);
1423
	} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
1424
		/*
1425
		 * have no way to get qm configure in VM in v1 hardware,
1426
		 * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force
1427
		 * to trigger only one VF in v1 hardware.
1428
		 *
1429
		 * v2 hardware has no such problem.
1430
		 */
1431
		qm->qp_base = HZIP_PF_DEF_Q_NUM;
1432
		qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;
1433
	}
1434

1435
	ret = hisi_qm_init(qm);
1436
	if (ret) {
1437
		pci_err(qm->pdev, "Failed to init zip qm configures!\n");
1438
		return ret;
1439
	}
1440

1441
	/* Fetch and save the value of capability registers */
1442
	ret = zip_pre_store_cap_reg(qm);
1443
	if (ret) {
1444
		pci_err(qm->pdev, "Failed to pre-store capability registers!\n");
1445
		goto err_qm_uninit;
1446
	}
1447

1448
	alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val;
1449
	ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs));
1450
	if (ret) {
1451
		pci_err(qm->pdev, "Failed to set zip algs!\n");
1452
		goto err_qm_uninit;
1453
	}
1454

1455
	ret = hisi_dae_set_alg(qm);
1456
	if (ret)
1457
		goto err_qm_uninit;
1458

1459
	return 0;
1460

1461
err_qm_uninit:
1462
	hisi_qm_uninit(qm);
1463
	return ret;
1464
}
1465

1466
static void hisi_zip_qm_uninit(struct hisi_qm *qm)
1467
{
1468
	hisi_qm_uninit(qm);
1469
}
1470

1471
static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)
1472
{
1473
	u32 type_rate = HZIP_SHAPER_RATE_COMPRESS;
1474
	struct hisi_qm *qm = &hisi_zip->qm;
1475
	int ret;
1476

1477
	if (qm->fun_type == QM_HW_PF) {
1478
		ret = hisi_zip_pf_probe_init(hisi_zip);
1479
		if (ret)
1480
			return ret;
1481
		/* enable shaper type 0 */
1482
		if (qm->ver >= QM_HW_V3) {
1483
			type_rate |= QM_SHAPER_ENABLE;
1484

1485
			/* ZIP need to enable shaper type 1 */
1486
			type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET;
1487
			qm->type_rate = type_rate;
1488
		}
1489
	}
1490

1491
	return 0;
1492
}
1493

1494
static void hisi_zip_probe_uninit(struct hisi_qm *qm)
1495
{
1496
	if (qm->fun_type == QM_HW_VF)
1497
		return;
1498

1499
	hisi_zip_show_last_regs_uninit(qm);
1500
	hisi_zip_close_sva_prefetch(qm);
1501
	hisi_qm_dev_err_uninit(qm);
1502
}
1503

1504
static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1505
{
1506
	struct hisi_zip *hisi_zip;
1507
	struct hisi_qm *qm;
1508
	int ret;
1509

1510
	hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
1511
	if (!hisi_zip)
1512
		return -ENOMEM;
1513

1514
	qm = &hisi_zip->qm;
1515

1516
	ret = hisi_zip_qm_init(qm, pdev);
1517
	if (ret) {
1518
		pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret);
1519
		return ret;
1520
	}
1521

1522
	ret = hisi_zip_probe_init(hisi_zip);
1523
	if (ret) {
1524
		pci_err(pdev, "Failed to probe (%d)!\n", ret);
1525
		goto err_qm_uninit;
1526
	}
1527

1528
	ret = hisi_qm_start(qm);
1529
	if (ret)
1530
		goto err_probe_uninit;
1531

1532
	ret = hisi_zip_debugfs_init(qm);
1533
	if (ret)
1534
		pci_err(pdev, "failed to init debugfs (%d)!\n", ret);
1535

1536
	hisi_qm_add_list(qm, &zip_devices);
1537
	ret = hisi_qm_alg_register(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);
1538
	if (ret < 0) {
1539
		pci_err(pdev, "failed to register driver to crypto!\n");
1540
		goto err_qm_del_list;
1541
	}
1542

1543
	if (qm->uacce) {
1544
		ret = uacce_register(qm->uacce);
1545
		if (ret) {
1546
			pci_err(pdev, "failed to register uacce (%d)!\n", ret);
1547
			goto err_qm_alg_unregister;
1548
		}
1549
	}
1550

1551
	if (qm->fun_type == QM_HW_PF && vfs_num > 0) {
1552
		ret = hisi_qm_sriov_enable(pdev, vfs_num);
1553
		if (ret < 0)
1554
			goto err_qm_alg_unregister;
1555
	}
1556

1557
	hisi_qm_pm_init(qm);
1558

1559
	return 0;
1560

1561
err_qm_alg_unregister:
1562
	hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);
1563

1564
err_qm_del_list:
1565
	hisi_qm_del_list(qm, &zip_devices);
1566
	hisi_zip_debugfs_exit(qm);
1567
	hisi_qm_stop(qm, QM_NORMAL);
1568

1569
err_probe_uninit:
1570
	hisi_zip_probe_uninit(qm);
1571

1572
err_qm_uninit:
1573
	hisi_zip_qm_uninit(qm);
1574

1575
	return ret;
1576
}
1577

1578
static void hisi_zip_remove(struct pci_dev *pdev)
1579
{
1580
	struct hisi_qm *qm = pci_get_drvdata(pdev);
1581

1582
	hisi_qm_pm_uninit(qm);
1583
	hisi_qm_wait_task_finish(qm, &zip_devices);
1584
	hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);
1585
	hisi_qm_del_list(qm, &zip_devices);
1586

1587
	if (qm->fun_type == QM_HW_PF && qm->vfs_num)
1588
		hisi_qm_sriov_disable(pdev, true);
1589

1590
	hisi_zip_debugfs_exit(qm);
1591
	hisi_qm_stop(qm, QM_NORMAL);
1592
	hisi_zip_probe_uninit(qm);
1593
	hisi_zip_qm_uninit(qm);
1594
}
1595

1596
static const struct dev_pm_ops hisi_zip_pm_ops = {
1597
	SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)
1598
};
1599

1600
static const struct pci_error_handlers hisi_zip_err_handler = {
1601
	.error_detected	= hisi_qm_dev_err_detected,
1602
	.slot_reset	= hisi_qm_dev_slot_reset,
1603
	.reset_prepare	= hisi_qm_reset_prepare,
1604
	.reset_done	= hisi_qm_reset_done,
1605
};
1606

1607
static struct pci_driver hisi_zip_pci_driver = {
1608
	.name			= "hisi_zip",
1609
	.id_table		= hisi_zip_dev_ids,
1610
	.probe			= hisi_zip_probe,
1611
	.remove			= hisi_zip_remove,
1612
	.sriov_configure	= IS_ENABLED(CONFIG_PCI_IOV) ?
1613
					hisi_qm_sriov_configure : NULL,
1614
	.err_handler		= &hisi_zip_err_handler,
1615
	.shutdown		= hisi_qm_dev_shutdown,
1616
	.driver.pm		= &hisi_zip_pm_ops,
1617
};
1618

1619
struct pci_driver *hisi_zip_get_pf_driver(void)
1620
{
1621
	return &hisi_zip_pci_driver;
1622
}
1623
EXPORT_SYMBOL_GPL(hisi_zip_get_pf_driver);
1624

1625
static void hisi_zip_register_debugfs(void)
1626
{
1627
	if (!debugfs_initialized())
1628
		return;
1629

1630
	hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL);
1631
}
1632

1633
static void hisi_zip_unregister_debugfs(void)
1634
{
1635
	debugfs_remove_recursive(hzip_debugfs_root);
1636
}
1637

1638
static int __init hisi_zip_init(void)
1639
{
1640
	int ret;
1641

1642
	hisi_qm_init_list(&zip_devices);
1643
	hisi_zip_register_debugfs();
1644

1645
	ret = pci_register_driver(&hisi_zip_pci_driver);
1646
	if (ret < 0) {
1647
		hisi_zip_unregister_debugfs();
1648
		pr_err("Failed to register pci driver.\n");
1649
	}
1650

1651
	return ret;
1652
}
1653

1654
static void __exit hisi_zip_exit(void)
1655
{
1656
	pci_unregister_driver(&hisi_zip_pci_driver);
1657
	hisi_zip_unregister_debugfs();
1658
}
1659

1660
module_init(hisi_zip_init);
1661
module_exit(hisi_zip_exit);
1662

1663
MODULE_LICENSE("GPL v2");
1664
MODULE_AUTHOR("Zhou Wang <[email protected]>");
1665
MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator");
1666

1667