xref: /linux/drivers/crypto/intel/qat/qat_common/qat_hal.c (revision 06d07429858317ded2db7986113a9e0129cd599b)
1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/slab.h>
4 #include <linux/delay.h>
5 #include <linux/pci_ids.h>
6 
7 #include "adf_accel_devices.h"
8 #include "adf_common_drv.h"
9 #include "icp_qat_hal.h"
10 #include "icp_qat_uclo.h"
11 
12 #define BAD_REGADDR	       0xffff
13 #define MAX_RETRY_TIMES	   10000
14 #define INIT_CTX_ARB_VALUE	0x0
15 #define INIT_CTX_ENABLE_VALUE     0x0
16 #define INIT_PC_VALUE	     0x0
17 #define INIT_WAKEUP_EVENTS_VALUE  0x1
18 #define INIT_SIG_EVENTS_VALUE     0x1
19 #define INIT_CCENABLE_VALUE       0x2000
20 #define RST_CSR_QAT_LSB	   20
21 #define RST_CSR_AE_LSB		  0
22 #define MC_TIMESTAMP_ENABLE       (0x1 << 7)
23 
24 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
25 	(~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
26 	(~(1 << CE_REG_PAR_ERR_BITPOS)))
27 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
28 	(inst = ((inst & 0xFFFF00C03FFull) | \
29 		((((const_val) << 12) & 0x0FF00000ull) | \
30 		(((const_val) << 10) & 0x0003FC00ull))))
31 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
32 	(inst = ((inst & 0xFFFF00FFF00ull) | \
33 		((((const_val) << 12) & 0x0FF00000ull) | \
34 		(((const_val) <<  0) & 0x000000FFull))))
35 
36 #define AE(handle, ae) ((handle)->hal_handle->aes[ae])
37 
38 static const u64 inst_4b[] = {
39 	0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
40 	0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
41 	0x0A021000000ull
42 };
43 
44 static const u64 inst[] = {
45 	0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
46 	0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
47 	0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
48 	0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
49 	0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
50 	0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
51 	0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
52 	0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
53 	0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
54 	0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
55 	0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
56 	0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
57 	0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
58 	0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
59 	0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
60 	0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
61 	0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
62 	0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
63 	0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
64 	0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
65 	0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
66 	0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
67 };
68 
qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
70 			  unsigned char ae, unsigned int ctx_mask)
71 {
72 	AE(handle, ae).live_ctx_mask = ctx_mask;
73 }
74 
75 #define CSR_RETRY_TIMES 500
qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr)76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
77 			     unsigned char ae, unsigned int csr)
78 {
79 	unsigned int iterations = CSR_RETRY_TIMES;
80 	int value;
81 
82 	do {
83 		value = GET_AE_CSR(handle, ae, csr);
84 		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
85 			return value;
86 	} while (iterations--);
87 
88 	pr_err("QAT: Read CSR timeout\n");
89 	return 0;
90 }
91 
qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr,unsigned int value)92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
93 			     unsigned char ae, unsigned int csr,
94 			     unsigned int value)
95 {
96 	unsigned int iterations = CSR_RETRY_TIMES;
97 
98 	do {
99 		SET_AE_CSR(handle, ae, csr, value);
100 		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
101 			return 0;
102 	} while (iterations--);
103 
104 	pr_err("QAT: Write CSR Timeout\n");
105 	return -EFAULT;
106 }
107 
qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int * events)108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
109 				     unsigned char ae, unsigned char ctx,
110 				     unsigned int *events)
111 {
112 	unsigned int cur_ctx;
113 
114 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
115 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
116 	*events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
117 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
118 }
119 
qat_hal_wait_cycles(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int cycles,int chk_inactive)120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
121 			       unsigned char ae, unsigned int cycles,
122 			       int chk_inactive)
123 {
124 	unsigned int base_cnt = 0, cur_cnt = 0;
125 	unsigned int csr = (1 << ACS_ABO_BITPOS);
126 	int times = MAX_RETRY_TIMES;
127 	int elapsed_cycles = 0;
128 
129 	base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
130 	base_cnt &= 0xffff;
131 	while ((int)cycles > elapsed_cycles && times--) {
132 		if (chk_inactive)
133 			csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
134 
135 		cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
136 		cur_cnt &= 0xffff;
137 		elapsed_cycles = cur_cnt - base_cnt;
138 
139 		if (elapsed_cycles < 0)
140 			elapsed_cycles += 0x10000;
141 
142 		/* ensure at least 8 time cycles elapsed in wait_cycles */
143 		if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
144 			return 0;
145 	}
146 	if (times < 0) {
147 		pr_err("QAT: wait_num_cycles time out\n");
148 		return -EFAULT;
149 	}
150 	return 0;
151 }
152 
153 #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))
154 #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))
155 
qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
157 			    unsigned char ae, unsigned char mode)
158 {
159 	unsigned int csr, new_csr;
160 
161 	if (mode != 4 && mode != 8) {
162 		pr_err("QAT: bad ctx mode=%d\n", mode);
163 		return -EINVAL;
164 	}
165 
166 	/* Sets the accelaration engine context mode to either four or eight */
167 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
168 	csr = IGNORE_W1C_MASK & csr;
169 	new_csr = (mode == 4) ?
170 		SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
171 		CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
172 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
173 	return 0;
174 }
175 
qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
177 			   unsigned char ae, unsigned char mode)
178 {
179 	unsigned int csr, new_csr;
180 
181 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
182 	csr &= IGNORE_W1C_MASK;
183 
184 	new_csr = (mode) ?
185 		SET_BIT(csr, CE_NN_MODE_BITPOS) :
186 		CLR_BIT(csr, CE_NN_MODE_BITPOS);
187 
188 	if (new_csr != csr)
189 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
190 
191 	return 0;
192 }
193 
qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,enum icp_qat_uof_regtype lm_type,unsigned char mode)194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
195 			   unsigned char ae, enum icp_qat_uof_regtype lm_type,
196 			   unsigned char mode)
197 {
198 	unsigned int csr, new_csr;
199 
200 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
201 	csr &= IGNORE_W1C_MASK;
202 	switch (lm_type) {
203 	case ICP_LMEM0:
204 		new_csr = (mode) ?
205 			SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
206 			CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
207 		break;
208 	case ICP_LMEM1:
209 		new_csr = (mode) ?
210 			SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
211 			CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
212 		break;
213 	case ICP_LMEM2:
214 		new_csr = (mode) ?
215 			SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :
216 			CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);
217 		break;
218 	case ICP_LMEM3:
219 		new_csr = (mode) ?
220 			SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :
221 			CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);
222 		break;
223 	default:
224 		pr_err("QAT: lmType = 0x%x\n", lm_type);
225 		return -EINVAL;
226 	}
227 
228 	if (new_csr != csr)
229 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
230 	return 0;
231 }
232 
qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)233 void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
234 				unsigned char ae, unsigned char mode)
235 {
236 	unsigned int csr, new_csr;
237 
238 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
239 	csr &= IGNORE_W1C_MASK;
240 	new_csr = (mode) ?
241 		  SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :
242 		  CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);
243 	if (new_csr != csr)
244 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
245 }
246 
qat_hal_get_reg_addr(unsigned int type,unsigned short reg_num)247 static unsigned short qat_hal_get_reg_addr(unsigned int type,
248 					   unsigned short reg_num)
249 {
250 	unsigned short reg_addr;
251 
252 	switch (type) {
253 	case ICP_GPA_ABS:
254 	case ICP_GPB_ABS:
255 		reg_addr = 0x80 | (reg_num & 0x7f);
256 		break;
257 	case ICP_GPA_REL:
258 	case ICP_GPB_REL:
259 		reg_addr = reg_num & 0x1f;
260 		break;
261 	case ICP_SR_RD_REL:
262 	case ICP_SR_WR_REL:
263 	case ICP_SR_REL:
264 		reg_addr = 0x180 | (reg_num & 0x1f);
265 		break;
266 	case ICP_SR_ABS:
267 		reg_addr = 0x140 | ((reg_num & 0x3) << 1);
268 		break;
269 	case ICP_DR_RD_REL:
270 	case ICP_DR_WR_REL:
271 	case ICP_DR_REL:
272 		reg_addr = 0x1c0 | (reg_num & 0x1f);
273 		break;
274 	case ICP_DR_ABS:
275 		reg_addr = 0x100 | ((reg_num & 0x3) << 1);
276 		break;
277 	case ICP_NEIGH_REL:
278 		reg_addr = 0x280 | (reg_num & 0x1f);
279 		break;
280 	case ICP_LMEM0:
281 		reg_addr = 0x200;
282 		break;
283 	case ICP_LMEM1:
284 		reg_addr = 0x220;
285 		break;
286 	case ICP_LMEM2:
287 		reg_addr = 0x2c0;
288 		break;
289 	case ICP_LMEM3:
290 		reg_addr = 0x2e0;
291 		break;
292 	case ICP_NO_DEST:
293 		reg_addr = 0x300 | (reg_num & 0xff);
294 		break;
295 	default:
296 		reg_addr = BAD_REGADDR;
297 		break;
298 	}
299 	return reg_addr;
300 }
301 
qat_hal_reset(struct icp_qat_fw_loader_handle * handle)302 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
303 {
304 	unsigned int reset_mask = handle->chip_info->icp_rst_mask;
305 	unsigned int reset_csr = handle->chip_info->icp_rst_csr;
306 	unsigned int csr_val;
307 
308 	csr_val = GET_CAP_CSR(handle, reset_csr);
309 	csr_val |= reset_mask;
310 	SET_CAP_CSR(handle, reset_csr, csr_val);
311 }
312 
qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int ae_csr,unsigned int csr_val)313 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
314 				unsigned char ae, unsigned int ctx_mask,
315 				unsigned int ae_csr, unsigned int csr_val)
316 {
317 	unsigned int ctx, cur_ctx;
318 
319 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
320 
321 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
322 		if (!(ctx_mask & (1 << ctx)))
323 			continue;
324 		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
325 		qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
326 	}
327 
328 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
329 }
330 
qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int ae_csr)331 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
332 				unsigned char ae, unsigned char ctx,
333 				unsigned int ae_csr)
334 {
335 	unsigned int cur_ctx, csr_val;
336 
337 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
338 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
339 	csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
340 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
341 
342 	return csr_val;
343 }
344 
qat_hal_put_sig_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)345 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
346 				  unsigned char ae, unsigned int ctx_mask,
347 				  unsigned int events)
348 {
349 	unsigned int ctx, cur_ctx;
350 
351 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
352 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
353 		if (!(ctx_mask & (1 << ctx)))
354 			continue;
355 		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
356 		qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
357 	}
358 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
359 }
360 
qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)361 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
362 				     unsigned char ae, unsigned int ctx_mask,
363 				     unsigned int events)
364 {
365 	unsigned int ctx, cur_ctx;
366 
367 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
368 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
369 		if (!(ctx_mask & (1 << ctx)))
370 			continue;
371 		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
372 		qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
373 				  events);
374 	}
375 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
376 }
377 
qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle * handle)378 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
379 {
380 	unsigned long ae_mask = handle->hal_handle->ae_mask;
381 	unsigned int base_cnt, cur_cnt;
382 	unsigned char ae;
383 	int times = MAX_RETRY_TIMES;
384 
385 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
386 		base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
387 		base_cnt &= 0xffff;
388 
389 		do {
390 			cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
391 			cur_cnt &= 0xffff;
392 		} while (times-- && (cur_cnt == base_cnt));
393 
394 		if (times < 0) {
395 			pr_err("QAT: AE%d is inactive!!\n", ae);
396 			return -EFAULT;
397 		}
398 	}
399 
400 	return 0;
401 }
402 
qat_hal_check_ae_active(struct icp_qat_fw_loader_handle * handle,unsigned int ae)403 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
404 			    unsigned int ae)
405 {
406 	unsigned int enable = 0, active = 0;
407 
408 	enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
409 	active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
410 	if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
411 	    (active & (1 << ACS_ABO_BITPOS)))
412 		return 1;
413 	else
414 		return 0;
415 }
416 
qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle * handle)417 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
418 {
419 	unsigned long ae_mask = handle->hal_handle->ae_mask;
420 	unsigned int misc_ctl_csr, misc_ctl;
421 	unsigned char ae;
422 
423 	misc_ctl_csr = handle->chip_info->misc_ctl_csr;
424 	/* stop the timestamp timers */
425 	misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr);
426 	if (misc_ctl & MC_TIMESTAMP_ENABLE)
427 		SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl &
428 			    (~MC_TIMESTAMP_ENABLE));
429 
430 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
431 		qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
432 		qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
433 	}
434 	/* start timestamp timers */
435 	SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);
436 }
437 
438 #define ESRAM_AUTO_TINIT	BIT(2)
439 #define ESRAM_AUTO_TINIT_DONE	BIT(3)
440 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
441 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
qat_hal_init_esram(struct icp_qat_fw_loader_handle * handle)442 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
443 {
444 	void __iomem *csr_addr =
445 			(void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
446 			ESRAM_AUTO_INIT_CSR_OFFSET);
447 	unsigned int csr_val;
448 	int times = 30;
449 
450 	if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
451 		return 0;
452 
453 	csr_val = ADF_CSR_RD(csr_addr, 0);
454 	if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
455 		return 0;
456 
457 	csr_val = ADF_CSR_RD(csr_addr, 0);
458 	csr_val |= ESRAM_AUTO_TINIT;
459 	ADF_CSR_WR(csr_addr, 0, csr_val);
460 
461 	do {
462 		qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
463 		csr_val = ADF_CSR_RD(csr_addr, 0);
464 	} while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
465 	if (times < 0) {
466 		pr_err("QAT: Fail to init eSram!\n");
467 		return -EFAULT;
468 	}
469 	return 0;
470 }
471 
472 #define SHRAM_INIT_CYCLES 2060
qat_hal_clr_reset(struct icp_qat_fw_loader_handle * handle)473 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
474 {
475 	unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr;
476 	unsigned int reset_mask = handle->chip_info->icp_rst_mask;
477 	unsigned int reset_csr = handle->chip_info->icp_rst_csr;
478 	unsigned long ae_mask = handle->hal_handle->ae_mask;
479 	unsigned char ae = 0;
480 	unsigned int times = 100;
481 	unsigned int csr_val;
482 
483 	/* write to the reset csr */
484 	csr_val = GET_CAP_CSR(handle, reset_csr);
485 	csr_val &= ~reset_mask;
486 	do {
487 		SET_CAP_CSR(handle, reset_csr, csr_val);
488 		if (!(times--))
489 			goto out_err;
490 		csr_val = GET_CAP_CSR(handle, reset_csr);
491 		csr_val &= reset_mask;
492 	} while (csr_val);
493 	/* enable clock */
494 	csr_val = GET_CAP_CSR(handle, clk_csr);
495 	csr_val |= reset_mask;
496 	SET_CAP_CSR(handle, clk_csr, csr_val);
497 	if (qat_hal_check_ae_alive(handle))
498 		goto out_err;
499 
500 	/* Set undefined power-up/reset states to reasonable default values */
501 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
502 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
503 				  INIT_CTX_ENABLE_VALUE);
504 		qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
505 				    CTX_STS_INDIRECT,
506 				    handle->hal_handle->upc_mask &
507 				    INIT_PC_VALUE);
508 		qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
509 		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
510 		qat_hal_put_wakeup_event(handle, ae,
511 					 ICP_QAT_UCLO_AE_ALL_CTX,
512 					 INIT_WAKEUP_EVENTS_VALUE);
513 		qat_hal_put_sig_event(handle, ae,
514 				      ICP_QAT_UCLO_AE_ALL_CTX,
515 				      INIT_SIG_EVENTS_VALUE);
516 	}
517 	if (qat_hal_init_esram(handle))
518 		goto out_err;
519 	if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
520 		goto out_err;
521 	qat_hal_reset_timestamp(handle);
522 
523 	return 0;
524 out_err:
525 	pr_err("QAT: failed to get device out of reset\n");
526 	return -EFAULT;
527 }
528 
qat_hal_disable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)529 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
530 				unsigned char ae, unsigned int ctx_mask)
531 {
532 	unsigned int ctx;
533 
534 	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
535 	ctx &= IGNORE_W1C_MASK &
536 		(~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
537 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
538 }
539 
qat_hal_parity_64bit(u64 word)540 static u64 qat_hal_parity_64bit(u64 word)
541 {
542 	word ^= word >> 1;
543 	word ^= word >> 2;
544 	word ^= word >> 4;
545 	word ^= word >> 8;
546 	word ^= word >> 16;
547 	word ^= word >> 32;
548 	return word & 1;
549 }
550 
qat_hal_set_uword_ecc(u64 uword)551 static u64 qat_hal_set_uword_ecc(u64 uword)
552 {
553 	u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
554 		bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
555 		bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
556 		bit6_mask = 0xdaf69a46910ULL;
557 
558 	/* clear the ecc bits */
559 	uword &= ~(0x7fULL << 0x2C);
560 	uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
561 	uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
562 	uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
563 	uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
564 	uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
565 	uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
566 	uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
567 	return uword;
568 }
569 
qat_hal_wr_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)570 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
571 		       unsigned char ae, unsigned int uaddr,
572 		       unsigned int words_num, u64 *uword)
573 {
574 	unsigned int ustore_addr;
575 	unsigned int i;
576 
577 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
578 	uaddr |= UA_ECS;
579 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
580 	for (i = 0; i < words_num; i++) {
581 		unsigned int uwrd_lo, uwrd_hi;
582 		u64 tmp;
583 
584 		tmp = qat_hal_set_uword_ecc(uword[i]);
585 		uwrd_lo = (unsigned int)(tmp & 0xffffffff);
586 		uwrd_hi = (unsigned int)(tmp >> 0x20);
587 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
588 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
589 	}
590 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
591 }
592 
qat_hal_enable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)593 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
594 			       unsigned char ae, unsigned int ctx_mask)
595 {
596 	unsigned int ctx;
597 
598 	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
599 	ctx &= IGNORE_W1C_MASK;
600 	ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
601 	ctx |= (ctx_mask << CE_ENABLE_BITPOS);
602 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
603 }
604 
qat_hal_clear_xfer(struct icp_qat_fw_loader_handle * handle)605 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
606 {
607 	unsigned long ae_mask = handle->hal_handle->ae_mask;
608 	unsigned char ae;
609 	unsigned short reg;
610 
611 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
612 		for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
613 			qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
614 					     reg, 0);
615 			qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
616 					     reg, 0);
617 		}
618 	}
619 }
620 
qat_hal_clear_gpr(struct icp_qat_fw_loader_handle * handle)621 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
622 {
623 	unsigned long ae_mask = handle->hal_handle->ae_mask;
624 	unsigned char ae;
625 	unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
626 	int times = MAX_RETRY_TIMES;
627 	unsigned int csr_val = 0;
628 	unsigned int savctx = 0;
629 	int ret = 0;
630 
631 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
632 		csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
633 		csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
634 		qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
635 		csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
636 		csr_val &= IGNORE_W1C_MASK;
637 		if (handle->chip_info->nn)
638 			csr_val |= CE_NN_MODE;
639 
640 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
641 		qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
642 				  (u64 *)inst);
643 		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
644 				    handle->hal_handle->upc_mask &
645 				    INIT_PC_VALUE);
646 		savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
647 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
648 		qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
649 		qat_hal_wr_indr_csr(handle, ae, ctx_mask,
650 				    CTX_SIG_EVENTS_INDIRECT, 0);
651 		qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
652 		qat_hal_enable_ctx(handle, ae, ctx_mask);
653 	}
654 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
655 		/* wait for AE to finish */
656 		do {
657 			ret = qat_hal_wait_cycles(handle, ae, 20, 1);
658 		} while (ret && times--);
659 
660 		if (times < 0) {
661 			pr_err("QAT: clear GPR of AE %d failed", ae);
662 			return -EINVAL;
663 		}
664 		qat_hal_disable_ctx(handle, ae, ctx_mask);
665 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
666 				  savctx & ACS_ACNO);
667 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
668 				  INIT_CTX_ENABLE_VALUE);
669 		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
670 				    handle->hal_handle->upc_mask &
671 				    INIT_PC_VALUE);
672 		qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
673 		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
674 		qat_hal_put_wakeup_event(handle, ae, ctx_mask,
675 					 INIT_WAKEUP_EVENTS_VALUE);
676 		qat_hal_put_sig_event(handle, ae, ctx_mask,
677 				      INIT_SIG_EVENTS_VALUE);
678 	}
679 	return 0;
680 }
681 
qat_hal_chip_init(struct icp_qat_fw_loader_handle * handle,struct adf_accel_dev * accel_dev)682 static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
683 			     struct adf_accel_dev *accel_dev)
684 {
685 	struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
686 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
687 	void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
688 	unsigned int max_en_ae_id = 0;
689 	struct adf_bar *sram_bar;
690 	unsigned int csr_val = 0;
691 	unsigned long ae_mask;
692 	unsigned char ae = 0;
693 	int ret = 0;
694 
695 	handle->pci_dev = pci_info->pci_dev;
696 	switch (handle->pci_dev->device) {
697 	case ADF_4XXX_PCI_DEVICE_ID:
698 	case ADF_401XX_PCI_DEVICE_ID:
699 	case ADF_402XX_PCI_DEVICE_ID:
700 	case ADF_420XX_PCI_DEVICE_ID:
701 		handle->chip_info->mmp_sram_size = 0;
702 		handle->chip_info->nn = false;
703 		handle->chip_info->lm2lm3 = true;
704 		handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;
705 		handle->chip_info->icp_rst_csr = ICP_RESET_CPP0;
706 		if (handle->pci_dev->device == ADF_420XX_PCI_DEVICE_ID)
707 			handle->chip_info->icp_rst_mask = 0x100155;
708 		else
709 			handle->chip_info->icp_rst_mask = 0x100015;
710 		handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0;
711 		handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX;
712 		handle->chip_info->wakeup_event_val = 0x80000000;
713 		handle->chip_info->fw_auth = true;
714 		handle->chip_info->css_3k = true;
715 		handle->chip_info->tgroup_share_ustore = true;
716 		handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;
717 		handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;
718 		handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX;
719 		handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX;
720 		handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;
721 		handle->chip_info->fcu_loaded_ae_pos = 0;
722 
723 		handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX;
724 		handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX;
725 		handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX;
726 		handle->hal_cap_ae_local_csr_addr_v =
727 			(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
728 			+ LOCAL_TO_XFER_REG_OFFSET);
729 		break;
730 	case PCI_DEVICE_ID_INTEL_QAT_C62X:
731 	case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
732 		handle->chip_info->mmp_sram_size = 0;
733 		handle->chip_info->nn = true;
734 		handle->chip_info->lm2lm3 = false;
735 		handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
736 		handle->chip_info->icp_rst_csr = ICP_RESET;
737 		handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
738 						  (hw_data->accel_mask << RST_CSR_QAT_LSB);
739 		handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
740 		handle->chip_info->misc_ctl_csr = MISC_CONTROL;
741 		handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
742 		handle->chip_info->fw_auth = true;
743 		handle->chip_info->css_3k = false;
744 		handle->chip_info->tgroup_share_ustore = false;
745 		handle->chip_info->fcu_ctl_csr = FCU_CONTROL;
746 		handle->chip_info->fcu_sts_csr = FCU_STATUS;
747 		handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI;
748 		handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;
749 		handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;
750 		handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
751 		handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
752 		handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
753 		handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
754 		handle->hal_cap_ae_local_csr_addr_v =
755 			(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
756 			+ LOCAL_TO_XFER_REG_OFFSET);
757 		break;
758 	case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
759 		handle->chip_info->mmp_sram_size = 0x40000;
760 		handle->chip_info->nn = true;
761 		handle->chip_info->lm2lm3 = false;
762 		handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
763 		handle->chip_info->icp_rst_csr = ICP_RESET;
764 		handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
765 						  (hw_data->accel_mask << RST_CSR_QAT_LSB);
766 		handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
767 		handle->chip_info->misc_ctl_csr = MISC_CONTROL;
768 		handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
769 		handle->chip_info->fw_auth = false;
770 		handle->chip_info->css_3k = false;
771 		handle->chip_info->tgroup_share_ustore = false;
772 		handle->chip_info->fcu_ctl_csr = 0;
773 		handle->chip_info->fcu_sts_csr = 0;
774 		handle->chip_info->fcu_dram_addr_hi = 0;
775 		handle->chip_info->fcu_dram_addr_lo = 0;
776 		handle->chip_info->fcu_loaded_ae_csr = 0;
777 		handle->chip_info->fcu_loaded_ae_pos = 0;
778 		handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
779 		handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
780 		handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
781 		handle->hal_cap_ae_local_csr_addr_v =
782 			(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
783 			+ LOCAL_TO_XFER_REG_OFFSET);
784 		break;
785 	default:
786 		ret = -EINVAL;
787 		goto out_err;
788 	}
789 
790 	if (handle->chip_info->mmp_sram_size > 0) {
791 		sram_bar =
792 			&pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
793 		handle->hal_sram_addr_v = sram_bar->virt_addr;
794 	}
795 	handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
796 	handle->hal_handle->ae_mask = hw_data->ae_mask;
797 	handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
798 	handle->hal_handle->slice_mask = hw_data->accel_mask;
799 	handle->cfg_ae_mask = ALL_AE_MASK;
800 	/* create AE objects */
801 	handle->hal_handle->upc_mask = 0x1ffff;
802 	handle->hal_handle->max_ustore = 0x4000;
803 
804 	ae_mask = handle->hal_handle->ae_mask;
805 	for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) {
806 		handle->hal_handle->aes[ae].free_addr = 0;
807 		handle->hal_handle->aes[ae].free_size =
808 		    handle->hal_handle->max_ustore;
809 		handle->hal_handle->aes[ae].ustore_size =
810 		    handle->hal_handle->max_ustore;
811 		handle->hal_handle->aes[ae].live_ctx_mask =
812 						ICP_QAT_UCLO_AE_ALL_CTX;
813 		max_en_ae_id = ae;
814 	}
815 	handle->hal_handle->ae_max_num = max_en_ae_id + 1;
816 
817 	/* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
818 	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
819 		csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
820 		csr_val |= 0x1;
821 		qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
822 	}
823 out_err:
824 	return ret;
825 }
826 
qat_hal_init(struct adf_accel_dev * accel_dev)827 int qat_hal_init(struct adf_accel_dev *accel_dev)
828 {
829 	struct icp_qat_fw_loader_handle *handle;
830 	int ret = 0;
831 
832 	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
833 	if (!handle)
834 		return -ENOMEM;
835 
836 	handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
837 	if (!handle->hal_handle) {
838 		ret = -ENOMEM;
839 		goto out_hal_handle;
840 	}
841 
842 	handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL);
843 	if (!handle->chip_info) {
844 		ret = -ENOMEM;
845 		goto out_chip_info;
846 	}
847 
848 	ret = qat_hal_chip_init(handle, accel_dev);
849 	if (ret) {
850 		dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n");
851 		goto out_err;
852 	}
853 
854 	/* take all AEs out of reset */
855 	ret = qat_hal_clr_reset(handle);
856 	if (ret) {
857 		dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
858 		goto out_err;
859 	}
860 
861 	qat_hal_clear_xfer(handle);
862 	if (!handle->chip_info->fw_auth) {
863 		ret = qat_hal_clear_gpr(handle);
864 		if (ret)
865 			goto out_err;
866 	}
867 
868 	accel_dev->fw_loader->fw_loader = handle;
869 	return 0;
870 
871 out_err:
872 	kfree(handle->chip_info);
873 out_chip_info:
874 	kfree(handle->hal_handle);
875 out_hal_handle:
876 	kfree(handle);
877 	return ret;
878 }
879 
qat_hal_deinit(struct icp_qat_fw_loader_handle * handle)880 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
881 {
882 	if (!handle)
883 		return;
884 	kfree(handle->chip_info);
885 	kfree(handle->hal_handle);
886 	kfree(handle);
887 }
888 
qat_hal_start(struct icp_qat_fw_loader_handle * handle)889 int qat_hal_start(struct icp_qat_fw_loader_handle *handle)
890 {
891 	unsigned long ae_mask = handle->hal_handle->ae_mask;
892 	u32 wakeup_val = handle->chip_info->wakeup_event_val;
893 	u32 fcu_ctl_csr, fcu_sts_csr;
894 	unsigned int fcu_sts;
895 	unsigned char ae;
896 	u32 ae_ctr = 0;
897 	int retry = 0;
898 
899 	if (handle->chip_info->fw_auth) {
900 		fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
901 		fcu_sts_csr = handle->chip_info->fcu_sts_csr;
902 		ae_ctr = hweight32(ae_mask);
903 		SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);
904 		do {
905 			msleep(FW_AUTH_WAIT_PERIOD);
906 			fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
907 			if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
908 				return ae_ctr;
909 		} while (retry++ < FW_AUTH_MAX_RETRY);
910 		pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts);
911 		return 0;
912 	} else {
913 		for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
914 			qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val);
915 			qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
916 			ae_ctr++;
917 		}
918 		return ae_ctr;
919 	}
920 }
921 
qat_hal_stop(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)922 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
923 		  unsigned int ctx_mask)
924 {
925 	if (!handle->chip_info->fw_auth)
926 		qat_hal_disable_ctx(handle, ae, ctx_mask);
927 }
928 
qat_hal_set_pc(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int upc)929 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
930 		    unsigned char ae, unsigned int ctx_mask, unsigned int upc)
931 {
932 	qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
933 			    handle->hal_handle->upc_mask & upc);
934 }
935 
qat_hal_get_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)936 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
937 			       unsigned char ae, unsigned int uaddr,
938 			       unsigned int words_num, u64 *uword)
939 {
940 	unsigned int i, uwrd_lo, uwrd_hi;
941 	unsigned int ustore_addr, misc_control;
942 
943 	misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
944 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
945 			  misc_control & 0xfffffffb);
946 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
947 	uaddr |= UA_ECS;
948 	for (i = 0; i < words_num; i++) {
949 		qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
950 		uaddr++;
951 		uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
952 		uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
953 		uword[i] = uwrd_hi;
954 		uword[i] = (uword[i] << 0x20) | uwrd_lo;
955 	}
956 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
957 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
958 }
959 
qat_hal_wr_umem(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,unsigned int * data)960 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
961 		     unsigned char ae, unsigned int uaddr,
962 		     unsigned int words_num, unsigned int *data)
963 {
964 	unsigned int i, ustore_addr;
965 
966 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
967 	uaddr |= UA_ECS;
968 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
969 	for (i = 0; i < words_num; i++) {
970 		unsigned int uwrd_lo, uwrd_hi, tmp;
971 
972 		uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
973 			  ((data[i] & 0xff00) << 2) |
974 			  (0x3 << 8) | (data[i] & 0xff);
975 		uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
976 		uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
977 		tmp = ((data[i] >> 0x10) & 0xffff);
978 		uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
979 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
980 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
981 	}
982 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
983 }
984 
985 #define MAX_EXEC_INST 100
qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,u64 * micro_inst,unsigned int inst_num,int code_off,unsigned int max_cycle,unsigned int * endpc)986 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
987 				   unsigned char ae, unsigned char ctx,
988 				   u64 *micro_inst, unsigned int inst_num,
989 				   int code_off, unsigned int max_cycle,
990 				   unsigned int *endpc)
991 {
992 	unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0;
993 	unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0;
994 	unsigned int ind_t_index = 0, ind_t_index_byte = 0;
995 	unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0;
996 	unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0;
997 	u64 savuwords[MAX_EXEC_INST];
998 	unsigned int ind_cnt_sig;
999 	unsigned int ind_sig, act_sig;
1000 	unsigned int csr_val = 0, newcsr_val;
1001 	unsigned int savctx;
1002 	unsigned int savcc, wakeup_events, savpc;
1003 	unsigned int ctxarb_ctl, ctx_enables;
1004 
1005 	if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
1006 		pr_err("QAT: invalid instruction num %d\n", inst_num);
1007 		return -EINVAL;
1008 	}
1009 	/* save current context */
1010 	ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
1011 	ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
1012 	ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
1013 						INDIRECT_LM_ADDR_0_BYTE_INDEX);
1014 	ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
1015 						INDIRECT_LM_ADDR_1_BYTE_INDEX);
1016 	if (handle->chip_info->lm2lm3) {
1017 		ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1018 						   LM_ADDR_2_INDIRECT);
1019 		ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1020 						   LM_ADDR_3_INDIRECT);
1021 		ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1022 							INDIRECT_LM_ADDR_2_BYTE_INDEX);
1023 		ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1024 							INDIRECT_LM_ADDR_3_BYTE_INDEX);
1025 		ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx,
1026 						  INDIRECT_T_INDEX);
1027 		ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx,
1028 						       INDIRECT_T_INDEX_BYTE_INDEX);
1029 	}
1030 	if (inst_num <= MAX_EXEC_INST)
1031 		qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
1032 	qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
1033 	savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
1034 	savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
1035 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1036 	ctx_enables &= IGNORE_W1C_MASK;
1037 	savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
1038 	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1039 	ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1040 	ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1041 					  FUTURE_COUNT_SIGNAL_INDIRECT);
1042 	ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1043 				      CTX_SIG_EVENTS_INDIRECT);
1044 	act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
1045 	/* execute micro codes */
1046 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1047 	qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
1048 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
1049 	qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
1050 	if (code_off)
1051 		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
1052 	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
1053 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
1054 	qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1055 	qat_hal_enable_ctx(handle, ae, (1 << ctx));
1056 	/* wait for micro codes to finish */
1057 	if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
1058 		return -EFAULT;
1059 	if (endpc) {
1060 		unsigned int ctx_status;
1061 
1062 		ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
1063 						 CTX_STS_INDIRECT);
1064 		*endpc = ctx_status & handle->hal_handle->upc_mask;
1065 	}
1066 	/* retore to saved context */
1067 	qat_hal_disable_ctx(handle, ae, (1 << ctx));
1068 	if (inst_num <= MAX_EXEC_INST)
1069 		qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
1070 	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
1071 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
1072 			    handle->hal_handle->upc_mask & savpc);
1073 	csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
1074 	newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1075 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1076 	qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
1077 	qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
1078 	qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
1079 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1080 			    LM_ADDR_0_INDIRECT, ind_lm_addr0);
1081 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1082 			    LM_ADDR_1_INDIRECT, ind_lm_addr1);
1083 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1084 			    INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
1085 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1086 			    INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
1087 	if (handle->chip_info->lm2lm3) {
1088 		qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT,
1089 				    ind_lm_addr2);
1090 		qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT,
1091 				    ind_lm_addr3);
1092 		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1093 				    INDIRECT_LM_ADDR_2_BYTE_INDEX,
1094 				    ind_lm_addr_byte2);
1095 		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1096 				    INDIRECT_LM_ADDR_3_BYTE_INDEX,
1097 				    ind_lm_addr_byte3);
1098 		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1099 				    INDIRECT_T_INDEX, ind_t_index);
1100 		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1101 				    INDIRECT_T_INDEX_BYTE_INDEX,
1102 				    ind_t_index_byte);
1103 	}
1104 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1105 			    FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
1106 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1107 			    CTX_SIG_EVENTS_INDIRECT, ind_sig);
1108 	qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
1109 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1110 
1111 	return 0;
1112 }
1113 
qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int * data)1114 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
1115 			      unsigned char ae, unsigned char ctx,
1116 			      enum icp_qat_uof_regtype reg_type,
1117 			      unsigned short reg_num, unsigned int *data)
1118 {
1119 	unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
1120 	unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
1121 	unsigned short reg_addr;
1122 	int status = 0;
1123 	u64 insts, savuword;
1124 
1125 	reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1126 	if (reg_addr == BAD_REGADDR) {
1127 		pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
1128 		return -EINVAL;
1129 	}
1130 	switch (reg_type) {
1131 	case ICP_GPA_REL:
1132 		insts = 0xA070000000ull | (reg_addr & 0x3ff);
1133 		break;
1134 	default:
1135 		insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
1136 		break;
1137 	}
1138 	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1139 	ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1140 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1141 	ctx_enables &= IGNORE_W1C_MASK;
1142 	if (ctx != (savctx & ACS_ACNO))
1143 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1144 				  ctx & ACS_ACNO);
1145 	qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
1146 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1147 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
1148 	uaddr = UA_ECS;
1149 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1150 	insts = qat_hal_set_uword_ecc(insts);
1151 	uwrd_lo = (unsigned int)(insts & 0xffffffff);
1152 	uwrd_hi = (unsigned int)(insts >> 0x20);
1153 	qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1154 	qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1155 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1156 	/* delay for at least 8 cycles */
1157 	qat_hal_wait_cycles(handle, ae, 0x8, 0);
1158 	/*
1159 	 * read ALU output
1160 	 * the instruction should have been executed
1161 	 * prior to clearing the ECS in putUwords
1162 	 */
1163 	*data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
1164 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1165 	qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1166 	if (ctx != (savctx & ACS_ACNO))
1167 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1168 				  savctx & ACS_ACNO);
1169 	qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1170 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1171 
1172 	return status;
1173 }
1174 
qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1175 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1176 			      unsigned char ae, unsigned char ctx,
1177 			      enum icp_qat_uof_regtype reg_type,
1178 			      unsigned short reg_num, unsigned int data)
1179 {
1180 	unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1181 	u64 insts[] = {
1182 		0x0F440000000ull,
1183 		0x0F040000000ull,
1184 		0x0F0000C0300ull,
1185 		0x0E000010000ull
1186 	};
1187 	const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1188 	const int imm_w1 = 0, imm_w0 = 1;
1189 
1190 	dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1191 	if (dest_addr == BAD_REGADDR) {
1192 		pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1193 		return -EINVAL;
1194 	}
1195 
1196 	data16lo = 0xffff & data;
1197 	data16hi = 0xffff & (data >> 0x10);
1198 	src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1199 					  (0xff & data16hi));
1200 	src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1201 					   (0xff & data16lo));
1202 	switch (reg_type) {
1203 	case ICP_GPA_REL:
1204 		insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1205 		    ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1206 		insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1207 		    ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1208 		break;
1209 	default:
1210 		insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1211 		    ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1212 
1213 		insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1214 		    ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1215 		break;
1216 	}
1217 
1218 	return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1219 				       code_off, num_inst * 0x5, NULL);
1220 }
1221 
qat_hal_get_ins_num(void)1222 int qat_hal_get_ins_num(void)
1223 {
1224 	return ARRAY_SIZE(inst_4b);
1225 }
1226 
qat_hal_concat_micro_code(u64 * micro_inst,unsigned int inst_num,unsigned int size,unsigned int addr,unsigned int * value)1227 static int qat_hal_concat_micro_code(u64 *micro_inst,
1228 				     unsigned int inst_num, unsigned int size,
1229 				     unsigned int addr, unsigned int *value)
1230 {
1231 	int i;
1232 	unsigned int cur_value;
1233 	const u64 *inst_arr;
1234 	int fixup_offset;
1235 	int usize = 0;
1236 	int orig_num;
1237 
1238 	orig_num = inst_num;
1239 	cur_value = value[0];
1240 	inst_arr = inst_4b;
1241 	usize = ARRAY_SIZE(inst_4b);
1242 	fixup_offset = inst_num;
1243 	for (i = 0; i < usize; i++)
1244 		micro_inst[inst_num++] = inst_arr[i];
1245 	INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1246 	fixup_offset++;
1247 	INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1248 	fixup_offset++;
1249 	INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1250 	fixup_offset++;
1251 	INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1252 
1253 	return inst_num - orig_num;
1254 }
1255 
qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,int * pfirst_exec,u64 * micro_inst,unsigned int inst_num)1256 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1257 				      unsigned char ae, unsigned char ctx,
1258 				      int *pfirst_exec, u64 *micro_inst,
1259 				      unsigned int inst_num)
1260 {
1261 	int stat = 0;
1262 	unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1263 	unsigned int gprb0 = 0, gprb1 = 0;
1264 
1265 	if (*pfirst_exec) {
1266 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1267 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1268 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1269 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1270 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1271 		*pfirst_exec = 0;
1272 	}
1273 	stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1274 				       inst_num * 0x5, NULL);
1275 	if (stat != 0)
1276 		return -EFAULT;
1277 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1278 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1279 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1280 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1281 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1282 
1283 	return 0;
1284 }
1285 
qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,struct icp_qat_uof_batch_init * lm_init_header)1286 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1287 			unsigned char ae,
1288 			struct icp_qat_uof_batch_init *lm_init_header)
1289 {
1290 	struct icp_qat_uof_batch_init *plm_init;
1291 	u64 *micro_inst_arry;
1292 	int micro_inst_num;
1293 	int alloc_inst_size;
1294 	int first_exec = 1;
1295 	int stat = 0;
1296 
1297 	plm_init = lm_init_header->next;
1298 	alloc_inst_size = lm_init_header->size;
1299 	if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1300 		alloc_inst_size = handle->hal_handle->max_ustore;
1301 	micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1302 					GFP_KERNEL);
1303 	if (!micro_inst_arry)
1304 		return -ENOMEM;
1305 	micro_inst_num = 0;
1306 	while (plm_init) {
1307 		unsigned int addr, *value, size;
1308 
1309 		ae = plm_init->ae;
1310 		addr = plm_init->addr;
1311 		value = plm_init->value;
1312 		size = plm_init->size;
1313 		micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1314 							    micro_inst_num,
1315 							    size, addr, value);
1316 		plm_init = plm_init->next;
1317 	}
1318 	/* exec micro codes */
1319 	if (micro_inst_arry && micro_inst_num > 0) {
1320 		micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1321 		stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1322 						  micro_inst_arry,
1323 						  micro_inst_num);
1324 	}
1325 	kfree(micro_inst_arry);
1326 	return stat;
1327 }
1328 
qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int val)1329 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1330 				   unsigned char ae, unsigned char ctx,
1331 				   enum icp_qat_uof_regtype reg_type,
1332 				   unsigned short reg_num, unsigned int val)
1333 {
1334 	int status = 0;
1335 	unsigned int reg_addr;
1336 	unsigned int ctx_enables;
1337 	unsigned short mask;
1338 	unsigned short dr_offset = 0x10;
1339 
1340 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1341 	if (CE_INUSE_CONTEXTS & ctx_enables) {
1342 		if (ctx & 0x1) {
1343 			pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1344 			return -EINVAL;
1345 		}
1346 		mask = 0x1f;
1347 		dr_offset = 0x20;
1348 	} else {
1349 		mask = 0x0f;
1350 	}
1351 	if (reg_num & ~mask)
1352 		return -EINVAL;
1353 	reg_addr = reg_num + (ctx << 0x5);
1354 	switch (reg_type) {
1355 	case ICP_SR_RD_REL:
1356 	case ICP_SR_REL:
1357 		SET_AE_XFER(handle, ae, reg_addr, val);
1358 		break;
1359 	case ICP_DR_RD_REL:
1360 	case ICP_DR_REL:
1361 		SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1362 		break;
1363 	default:
1364 		status = -EINVAL;
1365 		break;
1366 	}
1367 	return status;
1368 }
1369 
qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1370 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1371 				   unsigned char ae, unsigned char ctx,
1372 				   enum icp_qat_uof_regtype reg_type,
1373 				   unsigned short reg_num, unsigned int data)
1374 {
1375 	unsigned int gprval, ctx_enables;
1376 	unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1377 	    data16low;
1378 	unsigned short reg_mask;
1379 	int status = 0;
1380 	u64 micro_inst[] = {
1381 		0x0F440000000ull,
1382 		0x0F040000000ull,
1383 		0x0A000000000ull,
1384 		0x0F0000C0300ull,
1385 		0x0E000010000ull
1386 	};
1387 	const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1388 	const unsigned short gprnum = 0, dly = num_inst * 0x5;
1389 
1390 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1391 	if (CE_INUSE_CONTEXTS & ctx_enables) {
1392 		if (ctx & 0x1) {
1393 			pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1394 			return -EINVAL;
1395 		}
1396 		reg_mask = (unsigned short)~0x1f;
1397 	} else {
1398 		reg_mask = (unsigned short)~0xf;
1399 	}
1400 	if (reg_num & reg_mask)
1401 		return -EINVAL;
1402 	xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1403 	if (xfr_addr == BAD_REGADDR) {
1404 		pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1405 		return -EINVAL;
1406 	}
1407 	status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1408 	if (status) {
1409 		pr_err("QAT: failed to read register");
1410 		return status;
1411 	}
1412 	gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1413 	data16low = 0xffff & data;
1414 	data16hi = 0xffff & (data >> 0x10);
1415 	src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1416 					  (unsigned short)(0xff & data16hi));
1417 	src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1418 					   (unsigned short)(0xff & data16low));
1419 	micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1420 	    ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1421 	micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1422 	    ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1423 	micro_inst[0x2] = micro_inst[0x2] |
1424 	    ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1425 	status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1426 					 code_off, dly, NULL);
1427 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1428 	return status;
1429 }
1430 
qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned short nn,unsigned int val)1431 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1432 			      unsigned char ae, unsigned char ctx,
1433 			      unsigned short nn, unsigned int val)
1434 {
1435 	unsigned int ctx_enables;
1436 	int stat = 0;
1437 
1438 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1439 	ctx_enables &= IGNORE_W1C_MASK;
1440 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1441 
1442 	stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1443 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1444 	return stat;
1445 }
1446 
qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned short absreg_num,unsigned short * relreg,unsigned char * ctx)1447 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1448 				      *handle, unsigned char ae,
1449 				      unsigned short absreg_num,
1450 				      unsigned short *relreg,
1451 				      unsigned char *ctx)
1452 {
1453 	unsigned int ctx_enables;
1454 
1455 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1456 	if (ctx_enables & CE_INUSE_CONTEXTS) {
1457 		/* 4-ctx mode */
1458 		*relreg = absreg_num & 0x1F;
1459 		*ctx = (absreg_num >> 0x4) & 0x6;
1460 	} else {
1461 		/* 8-ctx mode */
1462 		*relreg = absreg_num & 0x0F;
1463 		*ctx = (absreg_num >> 0x4) & 0x7;
1464 	}
1465 	return 0;
1466 }
1467 
qat_hal_init_gpr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1468 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1469 		     unsigned char ae, unsigned long ctx_mask,
1470 		     enum icp_qat_uof_regtype reg_type,
1471 		     unsigned short reg_num, unsigned int regdata)
1472 {
1473 	int stat = 0;
1474 	unsigned short reg;
1475 	unsigned char ctx = 0;
1476 	enum icp_qat_uof_regtype type;
1477 
1478 	if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1479 		return -EINVAL;
1480 
1481 	do {
1482 		if (ctx_mask == 0) {
1483 			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1484 						   &ctx);
1485 			type = reg_type - 1;
1486 		} else {
1487 			reg = reg_num;
1488 			type = reg_type;
1489 			if (!test_bit(ctx, &ctx_mask))
1490 				continue;
1491 		}
1492 		stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1493 		if (stat) {
1494 			pr_err("QAT: write gpr fail\n");
1495 			return -EINVAL;
1496 		}
1497 	} while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1498 
1499 	return 0;
1500 }
1501 
qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1502 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1503 			 unsigned char ae, unsigned long ctx_mask,
1504 			 enum icp_qat_uof_regtype reg_type,
1505 			 unsigned short reg_num, unsigned int regdata)
1506 {
1507 	int stat = 0;
1508 	unsigned short reg;
1509 	unsigned char ctx = 0;
1510 	enum icp_qat_uof_regtype type;
1511 
1512 	if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1513 		return -EINVAL;
1514 
1515 	do {
1516 		if (ctx_mask == 0) {
1517 			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1518 						   &ctx);
1519 			type = reg_type - 3;
1520 		} else {
1521 			reg = reg_num;
1522 			type = reg_type;
1523 			if (!test_bit(ctx, &ctx_mask))
1524 				continue;
1525 		}
1526 		stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1527 					       regdata);
1528 		if (stat) {
1529 			pr_err("QAT: write wr xfer fail\n");
1530 			return -EINVAL;
1531 		}
1532 	} while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1533 
1534 	return 0;
1535 }
1536 
qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1537 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1538 			 unsigned char ae, unsigned long ctx_mask,
1539 			 enum icp_qat_uof_regtype reg_type,
1540 			 unsigned short reg_num, unsigned int regdata)
1541 {
1542 	int stat = 0;
1543 	unsigned short reg;
1544 	unsigned char ctx = 0;
1545 	enum icp_qat_uof_regtype type;
1546 
1547 	if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1548 		return -EINVAL;
1549 
1550 	do {
1551 		if (ctx_mask == 0) {
1552 			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1553 						   &ctx);
1554 			type = reg_type - 3;
1555 		} else {
1556 			reg = reg_num;
1557 			type = reg_type;
1558 			if (!test_bit(ctx, &ctx_mask))
1559 				continue;
1560 		}
1561 		stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1562 					       regdata);
1563 		if (stat) {
1564 			pr_err("QAT: write rd xfer fail\n");
1565 			return -EINVAL;
1566 		}
1567 	} while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1568 
1569 	return 0;
1570 }
1571 
qat_hal_init_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,unsigned short reg_num,unsigned int regdata)1572 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1573 		    unsigned char ae, unsigned long ctx_mask,
1574 		    unsigned short reg_num, unsigned int regdata)
1575 {
1576 	int stat = 0;
1577 	unsigned char ctx;
1578 	if (!handle->chip_info->nn) {
1579 		dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n",
1580 			handle->pci_dev->device);
1581 		return -EINVAL;
1582 	}
1583 
1584 	if (ctx_mask == 0)
1585 		return -EINVAL;
1586 
1587 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1588 		if (!test_bit(ctx, &ctx_mask))
1589 			continue;
1590 		stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1591 		if (stat) {
1592 			pr_err("QAT: write neigh error\n");
1593 			return -EINVAL;
1594 		}
1595 	}
1596 
1597 	return 0;
1598 }
1599