1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2025 Intel Corporation */
3 #include "qat_freebsd.h"
4 #include "adf_cfg.h"
5 #include "adf_common_drv.h"
6 #include "adf_accel_devices.h"
7 #include "icp_qat_uclo.h"
8 #include "icp_qat_fw.h"
9 #include "icp_qat_fw_init_admin.h"
10 #include "adf_cfg_strings.h"
11 #include "adf_transport_access_macros.h"
12 #include "adf_transport_internal.h"
13 #include <linux/delay.h>
14 #include "adf_accel_devices.h"
15 #include "adf_common_drv.h"
16 #include "icp_qat_hal.h"
17 #include "icp_qat_uclo.h"
18
19 #define BAD_REGADDR 0xffff
20 #define MAX_RETRY_TIMES 1000000
21 #define INIT_CTX_ARB_VALUE 0x0
22 #define INIT_CTX_ENABLE_VALUE 0x0
23 #define INIT_PC_VALUE 0x0
24 #define INIT_WAKEUP_EVENTS_VALUE 0x1
25 #define INIT_SIG_EVENTS_VALUE 0x1
26 #define INIT_CCENABLE_VALUE 0x2000
27 #define RST_CSR_QAT_LSB 20
28 #define RST_CSR_AE_LSB 0
29 #define MC_TIMESTAMP_ENABLE (0x1 << 7)
30
31 #define IGNORE_W1C_MASK \
32 ((~(1 << CE_BREAKPOINT_BITPOS)) & \
33 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
34 (~(1 << CE_REG_PAR_ERR_BITPOS)))
35 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
36 (inst = ((inst & 0xFFFF00C03FFull) | \
37 ((((const_val) << 12) & 0x0FF00000ull) | \
38 (((const_val) << 10) & 0x0003FC00ull))))
39 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
40 (inst = ((inst & 0xFFFF00FFF00ull) | \
41 ((((const_val) << 12) & 0x0FF00000ull) | \
42 (((const_val) << 0) & 0x000000FFull))))
43
44 #define AE(handle, ae) ((handle)->hal_handle->aes[ae])
45
46 static const uint64_t inst_4b[] = { 0x0F0400C0000ull, 0x0F4400C0000ull,
47 0x0F040000300ull, 0x0F440000300ull,
48 0x0FC066C0000ull, 0x0F0000C0300ull,
49 0x0F0000C0300ull, 0x0F0000C0300ull,
50 0x0A021000000ull };
51
52 static const uint64_t inst[] = {
53 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
54 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
55 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
56 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
57 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
58 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
59 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
60 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
61 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
62 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
63 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
64 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
65 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
66 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
67 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
68 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
69 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
70 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
71 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
72 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
73 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
74 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
75 };
76
77 static const uint64_t inst_CPM2X[] = {
78 0x0F0000C0000ull, 0x0D802C00011ull, 0x0F0000C0001ull, 0x0FC066C0001ull,
79 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F000500300ull,
80 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0A0580C0000ull,
81 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 0x0A0584C0000ull,
82 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 0x0A0588C0000ull,
83 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 0x0A058CC0000ull,
84 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 0x0A05C0C0000ull,
85 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 0x0A05C4C0000ull,
86 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 0x0A05C8C0000ull,
87 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 0x0A05CCC0000ull,
88 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 0x0A0400C0000ull,
89 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 0x0A0402C0000ull,
90 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 0x0A0404C0000ull,
91 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 0x0A0406C0000ull,
92 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 0x0A0408C0000ull,
93 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 0x0A040AC0000ull,
94 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 0x0A040CC0000ull,
95 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 0x0A040EC0000ull,
96 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 0x0D81341C010ull,
97 0x0E000000001ull, 0x0E000010000ull,
98 };
99
100 void
qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)101 qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
102 unsigned char ae,
103 unsigned int ctx_mask)
104 {
105 AE(handle, ae).live_ctx_mask = ctx_mask;
106 }
107
108 #define CSR_RETRY_TIMES 500
109 static int
qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr,unsigned int * value)110 qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
111 unsigned char ae,
112 unsigned int csr,
113 unsigned int *value)
114 {
115 unsigned int iterations = CSR_RETRY_TIMES;
116
117 do {
118 *value = GET_AE_CSR(handle, ae, csr);
119 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
120 return 0;
121 } while (iterations--);
122
123 pr_err("QAT: Read CSR timeout\n");
124 return EFAULT;
125 }
126
127 static int
qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr,unsigned int value)128 qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
129 unsigned char ae,
130 unsigned int csr,
131 unsigned int value)
132 {
133 unsigned int iterations = CSR_RETRY_TIMES;
134
135 do {
136 SET_AE_CSR(handle, ae, csr, value);
137 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
138 return 0;
139 } while (iterations--);
140
141 pr_err("QAT: Write CSR Timeout\n");
142 return EFAULT;
143 }
144
145 static void
qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int * events)146 qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
147 unsigned char ae,
148 unsigned char ctx,
149 unsigned int *events)
150 {
151 unsigned int cur_ctx;
152
153 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
154 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
155 qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events);
156 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
157 }
158
159 static int
qat_hal_wait_cycles(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int cycles,int chk_inactive)160 qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
161 unsigned char ae,
162 unsigned int cycles,
163 int chk_inactive)
164 {
165 unsigned int base_cnt = 0, cur_cnt = 0;
166 unsigned int csr = (1 << ACS_ABO_BITPOS);
167 int times = MAX_RETRY_TIMES;
168 int elapsed_cycles = 0;
169
170 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt);
171 base_cnt &= 0xffff;
172 while ((int)cycles > elapsed_cycles && times--) {
173 if (chk_inactive)
174 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr);
175
176 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt);
177 cur_cnt &= 0xffff;
178 elapsed_cycles = cur_cnt - base_cnt;
179
180 if (elapsed_cycles < 0)
181 elapsed_cycles += 0x10000;
182
183 /* ensure at least 8 time cycles elapsed in wait_cycles */
184 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
185 return 0;
186 }
187 if (times < 0) {
188 pr_err("QAT: wait_num_cycles time out\n");
189 return EFAULT;
190 }
191 return 0;
192 }
193
194 void
qat_hal_get_scs_neigh_ae(unsigned char ae,unsigned char * ae_neigh)195 qat_hal_get_scs_neigh_ae(unsigned char ae, unsigned char *ae_neigh)
196 {
197 *ae_neigh = (ae & 0x1) ? (ae - 1) : (ae + 1);
198 }
199
200 #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))
201 #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))
202
203 int
qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)204 qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
205 unsigned char ae,
206 unsigned char mode)
207 {
208 unsigned int csr, new_csr;
209
210 if (mode != 4 && mode != 8) {
211 pr_err("QAT: bad ctx mode=%d\n", mode);
212 return EINVAL;
213 }
214
215 /* Sets the accelaration engine context mode to either four or eight */
216 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
217 csr = IGNORE_W1C_MASK & csr;
218 new_csr = (mode == 4) ? SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
219 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
220 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
221 return 0;
222 }
223
224 int
qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)225 qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
226 unsigned char ae,
227 unsigned char mode)
228 {
229 unsigned int csr, new_csr;
230
231 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
232 pr_err("QAT: No next neigh for CPM2X\n");
233 return EINVAL;
234 }
235
236 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
237 csr &= IGNORE_W1C_MASK;
238
239 new_csr = (mode) ? SET_BIT(csr, CE_NN_MODE_BITPOS) :
240 CLR_BIT(csr, CE_NN_MODE_BITPOS);
241
242 if (new_csr != csr)
243 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
244
245 return 0;
246 }
247
248 int
qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,enum icp_qat_uof_regtype lm_type,unsigned char mode)249 qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
250 unsigned char ae,
251 enum icp_qat_uof_regtype lm_type,
252 unsigned char mode)
253 {
254 unsigned int csr, new_csr;
255
256 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
257 csr &= IGNORE_W1C_MASK;
258 switch (lm_type) {
259 case ICP_LMEM0:
260 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
261 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
262 break;
263 case ICP_LMEM1:
264 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
265 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
266 break;
267 case ICP_LMEM2:
268 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :
269 CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);
270 break;
271 case ICP_LMEM3:
272 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :
273 CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);
274 break;
275 default:
276 pr_err("QAT: lmType = 0x%x\n", lm_type);
277 return EINVAL;
278 }
279
280 if (new_csr != csr)
281 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
282 return 0;
283 }
284
285 void
qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)286 qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
287 unsigned char ae,
288 unsigned char mode)
289 {
290 unsigned int csr, new_csr;
291
292 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
293 csr &= IGNORE_W1C_MASK;
294 new_csr = (mode) ? SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :
295 CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);
296 if (new_csr != csr)
297 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
298 }
299
300 void
qat_hal_set_ae_scs_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)301 qat_hal_set_ae_scs_mode(struct icp_qat_fw_loader_handle *handle,
302 unsigned char ae,
303 unsigned char mode)
304 {
305 unsigned int csr, new_csr;
306
307 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr);
308 new_csr = (mode) ? SET_BIT(csr, MMC_SHARE_CS_BITPOS) :
309 CLR_BIT(csr, MMC_SHARE_CS_BITPOS);
310 if (new_csr != csr)
311 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, new_csr);
312 }
313
314 static unsigned short
qat_hal_get_reg_addr(unsigned int type,unsigned short reg_num)315 qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num)
316 {
317 unsigned short reg_addr;
318
319 switch (type) {
320 case ICP_GPA_ABS:
321 case ICP_GPB_ABS:
322 reg_addr = 0x80 | (reg_num & 0x7f);
323 break;
324 case ICP_GPA_REL:
325 case ICP_GPB_REL:
326 reg_addr = reg_num & 0x1f;
327 break;
328 case ICP_SR_RD_REL:
329 case ICP_SR_WR_REL:
330 case ICP_SR_REL:
331 reg_addr = 0x180 | (reg_num & 0x1f);
332 break;
333 case ICP_SR_ABS:
334 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
335 break;
336 case ICP_DR_RD_REL:
337 case ICP_DR_WR_REL:
338 case ICP_DR_REL:
339 reg_addr = 0x1c0 | (reg_num & 0x1f);
340 break;
341 case ICP_DR_ABS:
342 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
343 break;
344 case ICP_NEIGH_REL:
345 reg_addr = 0x280 | (reg_num & 0x1f);
346 break;
347 case ICP_LMEM0:
348 reg_addr = 0x200;
349 break;
350 case ICP_LMEM1:
351 reg_addr = 0x220;
352 break;
353 case ICP_LMEM2:
354 reg_addr = 0x2c0;
355 break;
356 case ICP_LMEM3:
357 reg_addr = 0x2e0;
358 break;
359 case ICP_NO_DEST:
360 reg_addr = 0x300 | (reg_num & 0xff);
361 break;
362 default:
363 reg_addr = BAD_REGADDR;
364 break;
365 }
366 return reg_addr;
367 }
368
369 static u32
qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle * handle)370 qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle *handle)
371 {
372 u32 tg = 0, ae;
373 u32 valid_ae_mask = 0;
374
375 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
376 if (handle->hal_handle->ae_mask & (1 << ae)) {
377 tg = ae / 4;
378 valid_ae_mask |= (1 << (tg * 2));
379 }
380 }
381 return valid_ae_mask;
382 }
383
384 void
qat_hal_reset(struct icp_qat_fw_loader_handle * handle)385 qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
386 {
387 unsigned int ae_reset_csr[MAX_CPP_NUM];
388 unsigned int ae_reset_val[MAX_CPP_NUM];
389 unsigned int valid_ae_mask, valid_slice_mask;
390 unsigned int cpp_num = 1;
391 unsigned int i;
392
393 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
394 ae_reset_csr[0] = ICP_RESET_CPP0;
395 ae_reset_csr[1] = ICP_RESET_CPP1;
396 if (handle->hal_handle->ae_mask > 0xffff)
397 ++cpp_num;
398 } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
399 ae_reset_csr[0] = ICP_RESET_CPP0;
400 } else {
401 ae_reset_csr[0] = ICP_RESET;
402 }
403
404 for (i = 0; i < cpp_num; i++) {
405 if (i == 0) {
406 if (IS_QAT_GEN4(
407 pci_get_device(GET_DEV(handle->accel_dev)))) {
408 valid_ae_mask =
409 qat_hal_get_ae_mask_gen4(handle);
410 valid_slice_mask =
411 handle->hal_handle->slice_mask;
412 } else {
413 valid_ae_mask =
414 handle->hal_handle->ae_mask & 0xFFFF;
415 valid_slice_mask =
416 handle->hal_handle->slice_mask & 0x3F;
417 }
418 } else {
419 valid_ae_mask =
420 (handle->hal_handle->ae_mask >> AES_PER_CPP) &
421 0xFFFF;
422 valid_slice_mask =
423 (handle->hal_handle->slice_mask >> SLICES_PER_CPP) &
424 0x3F;
425 }
426
427 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]);
428 ae_reset_val[i] |= valid_ae_mask << RST_CSR_AE_LSB;
429 ae_reset_val[i] |= valid_slice_mask << RST_CSR_QAT_LSB;
430 SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]);
431 }
432 }
433
434 static void
qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int ae_csr,unsigned int csr_val)435 qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
436 unsigned char ae,
437 unsigned int ctx_mask,
438 unsigned int ae_csr,
439 unsigned int csr_val)
440 {
441 unsigned int ctx, cur_ctx;
442
443 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
444
445 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
446 if (!(ctx_mask & (1 << ctx)))
447 continue;
448 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
449 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
450 }
451
452 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
453 }
454
455 static void
qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int ae_csr,unsigned int * csr_val)456 qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
457 unsigned char ae,
458 unsigned char ctx,
459 unsigned int ae_csr,
460 unsigned int *csr_val)
461 {
462 unsigned int cur_ctx;
463
464 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
465 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
466 qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val);
467 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
468 }
469
470 static void
qat_hal_put_sig_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)471 qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
472 unsigned char ae,
473 unsigned int ctx_mask,
474 unsigned int events)
475 {
476 unsigned int ctx, cur_ctx;
477
478 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
479 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
480 if (!(ctx_mask & (1 << ctx)))
481 continue;
482 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
483 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
484 }
485 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
486 }
487
488 static void
qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)489 qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
490 unsigned char ae,
491 unsigned int ctx_mask,
492 unsigned int events)
493 {
494 unsigned int ctx, cur_ctx;
495
496 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
497 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
498 if (!(ctx_mask & (1 << ctx)))
499 continue;
500 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
501 qat_hal_wr_ae_csr(handle,
502 ae,
503 CTX_WAKEUP_EVENTS_INDIRECT,
504 events);
505 }
506 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
507 }
508
509 static int
qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle * handle)510 qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
511 {
512 unsigned int base_cnt, cur_cnt;
513 unsigned char ae;
514 unsigned long ae_mask = handle->hal_handle->ae_mask;
515 int times = MAX_RETRY_TIMES;
516
517 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
518 {
519 qat_hal_rd_ae_csr(handle,
520 ae,
521 PROFILE_COUNT,
522 (unsigned int *)&base_cnt);
523 base_cnt &= 0xffff;
524
525 do {
526 qat_hal_rd_ae_csr(handle,
527 ae,
528 PROFILE_COUNT,
529 (unsigned int *)&cur_cnt);
530 cur_cnt &= 0xffff;
531 } while (times-- && (cur_cnt == base_cnt));
532
533 if (times < 0) {
534 pr_err("QAT: AE%d is inactive!!\n", ae);
535 return EFAULT;
536 }
537 }
538
539 return 0;
540 }
541
542 int
qat_hal_check_ae_active(struct icp_qat_fw_loader_handle * handle,unsigned int ae)543 qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
544 unsigned int ae)
545 {
546 unsigned int enable = 0, active = 0;
547
548 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable);
549 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active);
550 if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
551 (active & (1 << ACS_ABO_BITPOS)))
552 return 1;
553 else
554 return 0;
555 }
556
557 static void
qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle * handle)558 qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
559 {
560 unsigned int misc_ctl_csr, misc_ctl;
561 unsigned char ae;
562 unsigned long ae_mask = handle->hal_handle->ae_mask;
563
564 misc_ctl_csr =
565 (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) ?
566 MISC_CONTROL_C4XXX :
567 MISC_CONTROL;
568 /* stop the timestamp timers */
569 misc_ctl = GET_GLB_CSR(handle, misc_ctl_csr);
570 if (misc_ctl & MC_TIMESTAMP_ENABLE)
571 SET_GLB_CSR(handle,
572 misc_ctl_csr,
573 misc_ctl & (~MC_TIMESTAMP_ENABLE));
574
575 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
576 {
577 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
578 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
579 }
580 /* start timestamp timers */
581 SET_GLB_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);
582 }
583
584 #define ESRAM_AUTO_TINIT BIT(2)
585 #define ESRAM_AUTO_TINIT_DONE BIT(3)
586 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
587 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
588
589 static int
qat_hal_init_esram(struct icp_qat_fw_loader_handle * handle)590 qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
591 {
592 uintptr_t csr_addr =
593 ((uintptr_t)handle->hal_ep_csr_addr_v + ESRAM_AUTO_INIT_CSR_OFFSET);
594 unsigned int csr_val;
595 int times = 30;
596
597 if (pci_get_device(GET_DEV(handle->accel_dev)) !=
598 ADF_DH895XCC_PCI_DEVICE_ID)
599 return 0;
600
601 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr);
602 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
603 return 0;
604 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr);
605 csr_val |= ESRAM_AUTO_TINIT;
606
607 ADF_CSR_WR(handle->hal_misc_addr_v, csr_addr, csr_val);
608 do {
609 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
610 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr);
611
612 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
613 if (times < 0) {
614 pr_err("QAT: Fail to init eSram!\n");
615 return EFAULT;
616 }
617 return 0;
618 }
619
620 #define SHRAM_INIT_CYCLES 2060
621 int
qat_hal_clr_reset(struct icp_qat_fw_loader_handle * handle)622 qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
623 {
624 unsigned int ae_reset_csr[MAX_CPP_NUM];
625 unsigned int ae_reset_val[MAX_CPP_NUM];
626 unsigned int cpp_num = 1;
627 unsigned int valid_ae_mask, valid_slice_mask;
628 unsigned char ae;
629 unsigned int i;
630 unsigned int clk_csr[MAX_CPP_NUM];
631 unsigned int clk_val[MAX_CPP_NUM];
632 unsigned int times = 100;
633 unsigned long ae_mask = handle->hal_handle->ae_mask;
634
635 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
636 ae_reset_csr[0] = ICP_RESET_CPP0;
637 ae_reset_csr[1] = ICP_RESET_CPP1;
638 clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0;
639 clk_csr[1] = ICP_GLOBAL_CLK_ENABLE_CPP1;
640 if (handle->hal_handle->ae_mask > 0xffff)
641 ++cpp_num;
642 } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
643 ae_reset_csr[0] = ICP_RESET_CPP0;
644 clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0;
645 } else {
646 ae_reset_csr[0] = ICP_RESET;
647 clk_csr[0] = ICP_GLOBAL_CLK_ENABLE;
648 }
649
650 for (i = 0; i < cpp_num; i++) {
651 if (i == 0) {
652 if (IS_QAT_GEN4(
653 pci_get_device(GET_DEV(handle->accel_dev)))) {
654 valid_ae_mask =
655 qat_hal_get_ae_mask_gen4(handle);
656 valid_slice_mask =
657 handle->hal_handle->slice_mask;
658 } else {
659 valid_ae_mask =
660 handle->hal_handle->ae_mask & 0xFFFF;
661 valid_slice_mask =
662 handle->hal_handle->slice_mask & 0x3F;
663 }
664 } else {
665 valid_ae_mask =
666 (handle->hal_handle->ae_mask >> AES_PER_CPP) &
667 0xFFFF;
668 valid_slice_mask =
669 (handle->hal_handle->slice_mask >> SLICES_PER_CPP) &
670 0x3F;
671 }
672 /* write to the reset csr */
673 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]);
674 ae_reset_val[i] &= ~(valid_ae_mask << RST_CSR_AE_LSB);
675 ae_reset_val[i] &= ~(valid_slice_mask << RST_CSR_QAT_LSB);
676 do {
677 SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]);
678 if (!(times--))
679 goto out_err;
680 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]);
681 } while (
682 (valid_ae_mask | (valid_slice_mask << RST_CSR_QAT_LSB)) &
683 ae_reset_val[i]);
684 /* enable clock */
685 clk_val[i] = GET_GLB_CSR(handle, clk_csr[i]);
686 clk_val[i] |= valid_ae_mask << 0;
687 clk_val[i] |= valid_slice_mask << 20;
688 SET_GLB_CSR(handle, clk_csr[i], clk_val[i]);
689 }
690 if (qat_hal_check_ae_alive(handle))
691 goto out_err;
692
693 /* Set undefined power-up/reset states to reasonable default values */
694 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
695 {
696 qat_hal_wr_ae_csr(handle,
697 ae,
698 CTX_ENABLES,
699 INIT_CTX_ENABLE_VALUE);
700 qat_hal_wr_indr_csr(handle,
701 ae,
702 ICP_QAT_UCLO_AE_ALL_CTX,
703 CTX_STS_INDIRECT,
704 handle->hal_handle->upc_mask &
705 INIT_PC_VALUE);
706 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
707 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
708 qat_hal_put_wakeup_event(handle,
709 ae,
710 ICP_QAT_UCLO_AE_ALL_CTX,
711 INIT_WAKEUP_EVENTS_VALUE);
712 qat_hal_put_sig_event(handle,
713 ae,
714 ICP_QAT_UCLO_AE_ALL_CTX,
715 INIT_SIG_EVENTS_VALUE);
716 }
717 if (qat_hal_init_esram(handle))
718 goto out_err;
719 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
720 goto out_err;
721 qat_hal_reset_timestamp(handle);
722
723 return 0;
724 out_err:
725 pr_err("QAT: failed to get device out of reset\n");
726 return EFAULT;
727 }
728
729 static void
qat_hal_disable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)730 qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
731 unsigned char ae,
732 unsigned int ctx_mask)
733 {
734 unsigned int ctx;
735
736 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
737 ctx &= IGNORE_W1C_MASK &
738 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
739 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
740 }
741
742 static uint64_t
qat_hal_parity_64bit(uint64_t word)743 qat_hal_parity_64bit(uint64_t word)
744 {
745 word ^= word >> 1;
746 word ^= word >> 2;
747 word ^= word >> 4;
748 word ^= word >> 8;
749 word ^= word >> 16;
750 word ^= word >> 32;
751 return word & 1;
752 }
753
754 static uint64_t
qat_hal_set_uword_ecc(uint64_t uword)755 qat_hal_set_uword_ecc(uint64_t uword)
756 {
757 uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
758 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
759 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
760 bit6_mask = 0xdaf69a46910ULL;
761
762 /* clear the ecc bits */
763 uword &= ~(0x7fULL << 0x2C);
764 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
765 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
766 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
767 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
768 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
769 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
770 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
771 return uword;
772 }
773
774 void
qat_hal_wr_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,const uint64_t * uword)775 qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
776 unsigned char ae,
777 unsigned int uaddr,
778 unsigned int words_num,
779 const uint64_t *uword)
780 {
781 unsigned int ustore_addr;
782 unsigned int i, ae_in_group;
783
784 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
785 ae_in_group = ae / 4 * 4;
786
787 for (i = 0; i < AE_TG_NUM_CPM2X; i++) {
788 if (ae_in_group + i == ae)
789 continue;
790 if (ae_in_group + i >= handle->hal_handle->ae_max_num)
791 break;
792 if (qat_hal_check_ae_active(handle, ae_in_group + i)) {
793 pr_err(
794 "ae%d in T_group is active, cannot write to ustore!\n",
795 ae_in_group + i);
796 return;
797 }
798 }
799 }
800
801 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
802 uaddr |= UA_ECS;
803 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
804 for (i = 0; i < words_num; i++) {
805 unsigned int uwrd_lo, uwrd_hi;
806 uint64_t tmp;
807
808 tmp = qat_hal_set_uword_ecc(uword[i]);
809 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
810 uwrd_hi = (unsigned int)(tmp >> 0x20);
811 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
812 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
813 }
814 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
815 }
816
817 void
qat_hal_wr_coalesce_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)818 qat_hal_wr_coalesce_uwords(struct icp_qat_fw_loader_handle *handle,
819 unsigned char ae,
820 unsigned int uaddr,
821 unsigned int words_num,
822 u64 *uword)
823 {
824 u64 *even_uwrods, *odd_uwords;
825 unsigned char neigh_ae, odd_ae, even_ae;
826 int i, even_cpy_cnt = 0, odd_cpy_cnt = 0;
827
828 even_uwrods =
829 malloc(16 * 1024 * sizeof(*uword), M_QAT, M_WAITOK | M_ZERO);
830 odd_uwords =
831 malloc(16 * 1024 * sizeof(*uword), M_QAT, M_WAITOK | M_ZERO);
832 qat_hal_get_scs_neigh_ae(ae, &neigh_ae);
833 if (ae & 1) {
834 odd_ae = ae;
835 even_ae = neigh_ae;
836 } else {
837 odd_ae = neigh_ae;
838 even_ae = ae;
839 }
840 for (i = 0; i < words_num; i++) {
841 if ((uaddr + i) & 1)
842 odd_uwords[odd_cpy_cnt++] = uword[i];
843 else
844 even_uwrods[even_cpy_cnt++] = uword[i];
845 }
846 if (even_cpy_cnt)
847 qat_hal_wr_uwords(handle,
848 even_ae,
849 (uaddr + 1) / 2,
850 even_cpy_cnt,
851 even_uwrods);
852 if (odd_cpy_cnt)
853 qat_hal_wr_uwords(
854 handle, odd_ae, uaddr / 2, odd_cpy_cnt, odd_uwords);
855 free(even_uwrods, M_QAT);
856 free(odd_uwords, M_QAT);
857 }
858
859 static void
qat_hal_enable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)860 qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
861 unsigned char ae,
862 unsigned int ctx_mask)
863 {
864 unsigned int ctx;
865
866 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
867 ctx &= IGNORE_W1C_MASK;
868 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
869 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
870 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
871 }
872
873 static void
qat_hal_clear_xfer(struct icp_qat_fw_loader_handle * handle)874 qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
875 {
876 unsigned char ae;
877 unsigned short reg;
878 unsigned long ae_mask = handle->hal_handle->ae_mask;
879
880 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
881 {
882 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
883 qat_hal_init_rd_xfer(
884 handle, ae, 0, ICP_SR_RD_ABS, reg, 0);
885 qat_hal_init_rd_xfer(
886 handle, ae, 0, ICP_DR_RD_ABS, reg, 0);
887 }
888 }
889 }
890
891 static int
qat_hal_clear_gpr(struct icp_qat_fw_loader_handle * handle)892 qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
893 {
894 unsigned char ae;
895 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
896 int times = MAX_RETRY_TIMES;
897 unsigned int csr_val = 0;
898 unsigned int savctx = 0;
899 unsigned int scs_flag = 0;
900 unsigned long ae_mask = handle->hal_handle->ae_mask;
901 int ret = 0;
902
903 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
904 {
905 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
906 scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS);
907 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
908 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
909 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
910 csr_val &= IGNORE_W1C_MASK;
911 if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
912 csr_val |= CE_NN_MODE;
913 }
914 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
915
916 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
917 if (ae % 4 == 0)
918 qat_hal_wr_uwords(handle,
919 ae,
920 0,
921 ARRAY_SIZE(inst_CPM2X),
922 (const uint64_t *)inst_CPM2X);
923 } else {
924 qat_hal_wr_uwords(handle,
925 ae,
926 0,
927 ARRAY_SIZE(inst),
928 (const uint64_t *)inst);
929 }
930 qat_hal_wr_indr_csr(handle,
931 ae,
932 ctx_mask,
933 CTX_STS_INDIRECT,
934 handle->hal_handle->upc_mask &
935 INIT_PC_VALUE);
936 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
937 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
938 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
939 qat_hal_wr_indr_csr(
940 handle, ae, ctx_mask, CTX_SIG_EVENTS_INDIRECT, 0);
941 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
942 qat_hal_enable_ctx(handle, ae, ctx_mask);
943 }
944
945 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
946 {
947 /* wait for AE to finish */
948 do {
949 ret = qat_hal_wait_cycles(handle, ae, 20, 1);
950 } while (ret && times--);
951
952 if (times < 0) {
953 pr_err("QAT: clear GPR of AE %d failed", ae);
954 return EINVAL;
955 }
956 qat_hal_disable_ctx(handle, ae, ctx_mask);
957 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
958 if (scs_flag)
959 csr_val |= (1 << MMC_SHARE_CS_BITPOS);
960 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
961 qat_hal_wr_ae_csr(handle,
962 ae,
963 ACTIVE_CTX_STATUS,
964 savctx & ACS_ACNO);
965 qat_hal_wr_ae_csr(handle,
966 ae,
967 CTX_ENABLES,
968 INIT_CTX_ENABLE_VALUE);
969 qat_hal_wr_indr_csr(handle,
970 ae,
971 ctx_mask,
972 CTX_STS_INDIRECT,
973 handle->hal_handle->upc_mask &
974 INIT_PC_VALUE);
975 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
976 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
977 qat_hal_put_wakeup_event(handle,
978 ae,
979 ctx_mask,
980 INIT_WAKEUP_EVENTS_VALUE);
981 qat_hal_put_sig_event(handle,
982 ae,
983 ctx_mask,
984 INIT_SIG_EVENTS_VALUE);
985 }
986 return 0;
987 }
988
989 static int
qat_hal_check_imr(struct icp_qat_fw_loader_handle * handle)990 qat_hal_check_imr(struct icp_qat_fw_loader_handle *handle)
991 {
992 device_t dev = accel_to_pci_dev(handle->accel_dev);
993 u8 reg_val = 0;
994
995 if (pci_get_device(GET_DEV(handle->accel_dev)) !=
996 ADF_C3XXX_PCI_DEVICE_ID &&
997 pci_get_device(GET_DEV(handle->accel_dev)) !=
998 ADF_200XX_PCI_DEVICE_ID)
999 return 0;
1000
1001 reg_val = pci_read_config(dev, 0x04, 1);
1002 /*
1003 * PCI command register memory bit and rambaseaddr_lo address
1004 * are checked to confirm IMR2 is enabled in BIOS settings
1005 */
1006 if ((reg_val & 0x2) && GET_FCU_CSR(handle, FCU_RAMBASE_ADDR_LO))
1007 return 0;
1008
1009 return EINVAL;
1010 }
1011
1012 int
qat_hal_init(struct adf_accel_dev * accel_dev)1013 qat_hal_init(struct adf_accel_dev *accel_dev)
1014 {
1015 unsigned char ae;
1016 unsigned int cap_offset, ae_offset, ep_offset;
1017 unsigned int sram_offset = 0;
1018 unsigned int max_en_ae_id = 0;
1019 int ret = 0;
1020 unsigned long ae_mask;
1021 struct icp_qat_fw_loader_handle *handle;
1022 if (!accel_dev) {
1023 return EFAULT;
1024 }
1025 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
1026 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
1027 struct adf_bar *misc_bar =
1028 &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
1029 struct adf_bar *sram_bar;
1030
1031 handle = malloc(sizeof(*handle), M_QAT, M_WAITOK | M_ZERO);
1032
1033 handle->hal_misc_addr_v = misc_bar->virt_addr;
1034 handle->accel_dev = accel_dev;
1035 if (pci_get_device(GET_DEV(handle->accel_dev)) ==
1036 ADF_DH895XCC_PCI_DEVICE_ID ||
1037 IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
1038 sram_bar =
1039 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
1040 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev))))
1041 sram_offset =
1042 0x400000 + accel_dev->aram_info->mmp_region_offset;
1043 handle->hal_sram_addr_v = sram_bar->virt_addr;
1044 handle->hal_sram_offset = sram_offset;
1045 handle->hal_sram_size = sram_bar->size;
1046 }
1047 GET_CSR_OFFSET(pci_get_device(GET_DEV(handle->accel_dev)),
1048 cap_offset,
1049 ae_offset,
1050 ep_offset);
1051 handle->hal_cap_g_ctl_csr_addr_v = cap_offset;
1052 handle->hal_cap_ae_xfer_csr_addr_v = ae_offset;
1053 handle->hal_ep_csr_addr_v = ep_offset;
1054 handle->hal_cap_ae_local_csr_addr_v =
1055 ((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET);
1056 handle->fw_auth = (pci_get_device(GET_DEV(handle->accel_dev)) ==
1057 ADF_DH895XCC_PCI_DEVICE_ID) ?
1058 false :
1059 true;
1060 if (handle->fw_auth && qat_hal_check_imr(handle)) {
1061 device_printf(GET_DEV(accel_dev), "IMR2 not enabled in BIOS\n");
1062 ret = EINVAL;
1063 goto out_hal_handle;
1064 }
1065
1066 handle->hal_handle =
1067 malloc(sizeof(*handle->hal_handle), M_QAT, M_WAITOK | M_ZERO);
1068 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
1069 handle->hal_handle->ae_mask = hw_data->ae_mask;
1070 handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
1071 handle->hal_handle->slice_mask = hw_data->accel_mask;
1072 handle->cfg_ae_mask = 0xFFFFFFFF;
1073 /* create AE objects */
1074 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
1075 handle->hal_handle->upc_mask = 0xffff;
1076 handle->hal_handle->max_ustore = 0x2000;
1077 } else {
1078 handle->hal_handle->upc_mask = 0x1ffff;
1079 handle->hal_handle->max_ustore = 0x4000;
1080 }
1081
1082 ae_mask = hw_data->ae_mask;
1083
1084 for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE)
1085 {
1086 handle->hal_handle->aes[ae].free_addr = 0;
1087 handle->hal_handle->aes[ae].free_size =
1088 handle->hal_handle->max_ustore;
1089 handle->hal_handle->aes[ae].ustore_size =
1090 handle->hal_handle->max_ustore;
1091 handle->hal_handle->aes[ae].live_ctx_mask =
1092 ICP_QAT_UCLO_AE_ALL_CTX;
1093 max_en_ae_id = ae;
1094 }
1095 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
1096 /* take all AEs out of reset */
1097 if (qat_hal_clr_reset(handle)) {
1098 device_printf(GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
1099 ret = EIO;
1100 goto out_err;
1101 }
1102 qat_hal_clear_xfer(handle);
1103 if (!handle->fw_auth) {
1104 if (qat_hal_clear_gpr(handle)) {
1105 ret = EIO;
1106 goto out_err;
1107 }
1108 }
1109
1110 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
1111 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
1112 {
1113 unsigned int csr_val = 0;
1114
1115 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val);
1116 csr_val |= 0x1;
1117 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
1118 }
1119 accel_dev->fw_loader->fw_loader = handle;
1120 return 0;
1121
1122 out_err:
1123 free(handle->hal_handle, M_QAT);
1124 out_hal_handle:
1125 free(handle, M_QAT);
1126 return ret;
1127 }
1128
1129 void
qat_hal_deinit(struct icp_qat_fw_loader_handle * handle)1130 qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
1131 {
1132 if (!handle)
1133 return;
1134 free(handle->hal_handle, M_QAT);
1135 free(handle, M_QAT);
1136 }
1137
1138 int
qat_hal_start(struct icp_qat_fw_loader_handle * handle)1139 qat_hal_start(struct icp_qat_fw_loader_handle *handle)
1140 {
1141 unsigned char ae = 0;
1142 int retry = 0;
1143 unsigned int fcu_sts = 0;
1144 unsigned int fcu_ctl_csr, fcu_sts_csr;
1145 unsigned long ae_mask = handle->hal_handle->ae_mask;
1146 u32 ae_ctr = 0;
1147
1148 if (handle->fw_auth) {
1149 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
1150 {
1151 ae_ctr++;
1152 }
1153 if (IS_QAT_GEN3_OR_GEN4(
1154 pci_get_device(GET_DEV(handle->accel_dev)))) {
1155 fcu_ctl_csr = FCU_CONTROL_C4XXX;
1156 fcu_sts_csr = FCU_STATUS_C4XXX;
1157
1158 } else {
1159 fcu_ctl_csr = FCU_CONTROL;
1160 fcu_sts_csr = FCU_STATUS;
1161 }
1162 SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);
1163 do {
1164 pause_ms("adfstop", FW_AUTH_WAIT_PERIOD);
1165 fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr);
1166 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
1167 return ae_ctr;
1168 } while (retry++ < FW_AUTH_MAX_RETRY);
1169 pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n",
1170 ae,
1171 fcu_sts);
1172 return 0;
1173 } else {
1174 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
1175 {
1176 qat_hal_put_wakeup_event(handle,
1177 ae,
1178 0,
1179 IS_QAT_GEN4(
1180 pci_get_device(GET_DEV(
1181 handle->accel_dev))) ?
1182 0x80000000 :
1183 0x10000);
1184 qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
1185 ae_ctr++;
1186 }
1187 return ae_ctr;
1188 }
1189 }
1190
1191 void
qat_hal_stop(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)1192 qat_hal_stop(struct icp_qat_fw_loader_handle *handle,
1193 unsigned char ae,
1194 unsigned int ctx_mask)
1195 {
1196 if (!handle->fw_auth)
1197 qat_hal_disable_ctx(handle, ae, ctx_mask);
1198 }
1199
1200 void
qat_hal_set_pc(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int upc)1201 qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
1202 unsigned char ae,
1203 unsigned int ctx_mask,
1204 unsigned int upc)
1205 {
1206 qat_hal_wr_indr_csr(handle,
1207 ae,
1208 ctx_mask,
1209 CTX_STS_INDIRECT,
1210 handle->hal_handle->upc_mask & upc);
1211 }
1212
1213 static void
qat_hal_get_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,uint64_t * uword)1214 qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
1215 unsigned char ae,
1216 unsigned int uaddr,
1217 unsigned int words_num,
1218 uint64_t *uword)
1219 {
1220 unsigned int i, uwrd_lo, uwrd_hi;
1221 unsigned int ustore_addr, misc_control;
1222 unsigned int scs_flag = 0;
1223
1224 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control);
1225 scs_flag = misc_control & (0x1 << MMC_SHARE_CS_BITPOS);
1226 /*disable scs*/
1227 qat_hal_wr_ae_csr(handle,
1228 ae,
1229 AE_MISC_CONTROL,
1230 misc_control & 0xfffffffb);
1231 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
1232 uaddr |= UA_ECS;
1233 for (i = 0; i < words_num; i++) {
1234 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1235 uaddr++;
1236 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo);
1237 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi);
1238 uword[i] = uwrd_hi;
1239 uword[i] = (uword[i] << 0x20) | uwrd_lo;
1240 }
1241 if (scs_flag)
1242 misc_control |= (0x1 << MMC_SHARE_CS_BITPOS);
1243 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
1244 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1245 }
1246
1247 void
qat_hal_wr_umem(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,unsigned int * data)1248 qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
1249 unsigned char ae,
1250 unsigned int uaddr,
1251 unsigned int words_num,
1252 unsigned int *data)
1253 {
1254 unsigned int i, ustore_addr;
1255
1256 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
1257 uaddr |= UA_ECS;
1258 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1259 for (i = 0; i < words_num; i++) {
1260 unsigned int uwrd_lo, uwrd_hi, tmp;
1261
1262 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
1263 ((data[i] & 0xff00) << 2) | (0x3 << 8) | (data[i] & 0xff);
1264 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
1265 uwrd_hi |= (bitcount32(data[i] & 0xffff) & 0x1) << 8;
1266 tmp = ((data[i] >> 0x10) & 0xffff);
1267 uwrd_hi |= (bitcount32(tmp) & 0x1) << 9;
1268 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1269 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1270 }
1271 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1272 }
1273
1274 #define MAX_EXEC_INST 100
1275 static int
qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,uint64_t * micro_inst,unsigned int inst_num,int code_off,unsigned int max_cycle,unsigned int * endpc)1276 qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
1277 unsigned char ae,
1278 unsigned char ctx,
1279 uint64_t *micro_inst,
1280 unsigned int inst_num,
1281 int code_off,
1282 unsigned int max_cycle,
1283 unsigned int *endpc)
1284 {
1285 u64 *savuwords = NULL;
1286 unsigned int ind_lm_addr0, ind_lm_addr1;
1287 unsigned int ind_lm_addr2, ind_lm_addr3;
1288 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
1289 unsigned int ind_lm_addr_byte2, ind_lm_addr_byte3;
1290 unsigned int ind_t_index, ind_t_index_byte;
1291 unsigned int ind_cnt_sig;
1292 unsigned int ind_sig, act_sig;
1293 unsigned int csr_val = 0, newcsr_val;
1294 unsigned int savctx, scs_flag;
1295 unsigned int savcc, wakeup_events, savpc;
1296 unsigned int ctxarb_ctl, ctx_enables;
1297
1298 if (inst_num > handle->hal_handle->max_ustore || !micro_inst) {
1299 pr_err("QAT: invalid instruction num %d\n", inst_num);
1300 return EINVAL;
1301 }
1302
1303 savuwords = kzalloc(sizeof(u64) * MAX_EXEC_INST, GFP_KERNEL);
1304 if (!savuwords)
1305 return ENOMEM;
1306
1307 /* save current context */
1308 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
1309 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
1310 qat_hal_rd_indr_csr(
1311 handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &ind_lm_addr_byte0);
1312 qat_hal_rd_indr_csr(
1313 handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &ind_lm_addr_byte1);
1314 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
1315 qat_hal_rd_indr_csr(
1316 handle, ae, ctx, LM_ADDR_2_INDIRECT, &ind_lm_addr2);
1317 qat_hal_rd_indr_csr(
1318 handle, ae, ctx, LM_ADDR_3_INDIRECT, &ind_lm_addr3);
1319 qat_hal_rd_indr_csr(handle,
1320 ae,
1321 ctx,
1322 INDIRECT_LM_ADDR_2_BYTE_INDEX,
1323 &ind_lm_addr_byte2);
1324 qat_hal_rd_indr_csr(handle,
1325 ae,
1326 ctx,
1327 INDIRECT_LM_ADDR_3_BYTE_INDEX,
1328 &ind_lm_addr_byte3);
1329 qat_hal_rd_indr_csr(
1330 handle, ae, ctx, INDIRECT_T_INDEX, &ind_t_index);
1331 qat_hal_rd_indr_csr(handle,
1332 ae,
1333 ctx,
1334 INDIRECT_T_INDEX_BYTE_INDEX,
1335 &ind_t_index_byte);
1336 }
1337 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
1338 scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS);
1339 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1340 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1341 if (inst_num <= MAX_EXEC_INST)
1342 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
1343 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
1344 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc);
1345 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
1346 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1347 ctx_enables &= IGNORE_W1C_MASK;
1348 qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc);
1349 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
1350 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl);
1351 qat_hal_rd_indr_csr(
1352 handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, &ind_cnt_sig);
1353 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig);
1354 qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig);
1355 /* execute micro codes */
1356 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1357 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
1358 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
1359 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
1360 if (code_off)
1361 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
1362 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
1363 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
1364 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1365 qat_hal_enable_ctx(handle, ae, (1 << ctx));
1366 /* wait for micro codes to finish */
1367 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) {
1368 kfree(savuwords);
1369 return EFAULT;
1370 }
1371 if (endpc) {
1372 unsigned int ctx_status;
1373
1374 qat_hal_rd_indr_csr(
1375 handle, ae, ctx, CTX_STS_INDIRECT, &ctx_status);
1376 *endpc = ctx_status & handle->hal_handle->upc_mask;
1377 }
1378 /* retore to saved context */
1379 qat_hal_disable_ctx(handle, ae, (1 << ctx));
1380 if (inst_num <= MAX_EXEC_INST)
1381 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
1382 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
1383 qat_hal_wr_indr_csr(handle,
1384 ae,
1385 (1 << ctx),
1386 CTX_STS_INDIRECT,
1387 handle->hal_handle->upc_mask & savpc);
1388 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
1389 newcsr_val = scs_flag ? SET_BIT(csr_val, MMC_SHARE_CS_BITPOS) :
1390 CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1391 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1392 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
1393 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
1394 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
1395 qat_hal_wr_indr_csr(
1396 handle, ae, (1 << ctx), LM_ADDR_0_INDIRECT, ind_lm_addr0);
1397 qat_hal_wr_indr_csr(
1398 handle, ae, (1 << ctx), LM_ADDR_1_INDIRECT, ind_lm_addr1);
1399 qat_hal_wr_indr_csr(handle,
1400 ae,
1401 (1 << ctx),
1402 INDIRECT_LM_ADDR_0_BYTE_INDEX,
1403 ind_lm_addr_byte0);
1404 qat_hal_wr_indr_csr(handle,
1405 ae,
1406 (1 << ctx),
1407 INDIRECT_LM_ADDR_1_BYTE_INDEX,
1408 ind_lm_addr_byte1);
1409 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
1410 qat_hal_wr_indr_csr(
1411 handle, ae, (1 << ctx), LM_ADDR_2_INDIRECT, ind_lm_addr2);
1412 qat_hal_wr_indr_csr(
1413 handle, ae, (1 << ctx), LM_ADDR_3_INDIRECT, ind_lm_addr3);
1414 qat_hal_wr_indr_csr(handle,
1415 ae,
1416 (1 << ctx),
1417 INDIRECT_LM_ADDR_2_BYTE_INDEX,
1418 ind_lm_addr_byte2);
1419 qat_hal_wr_indr_csr(handle,
1420 ae,
1421 (1 << ctx),
1422 INDIRECT_LM_ADDR_3_BYTE_INDEX,
1423 ind_lm_addr_byte3);
1424 qat_hal_wr_indr_csr(
1425 handle, ae, (1 << ctx), INDIRECT_T_INDEX, ind_t_index);
1426 qat_hal_wr_indr_csr(handle,
1427 ae,
1428 (1 << ctx),
1429 INDIRECT_T_INDEX_BYTE_INDEX,
1430 ind_t_index_byte);
1431 }
1432 qat_hal_wr_indr_csr(
1433 handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
1434 qat_hal_wr_indr_csr(
1435 handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, ind_sig);
1436 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
1437 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1438 kfree(savuwords);
1439
1440 return 0;
1441 }
1442
1443 static int
qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int * data)1444 qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
1445 unsigned char ae,
1446 unsigned char ctx,
1447 enum icp_qat_uof_regtype reg_type,
1448 unsigned short reg_num,
1449 unsigned int *data)
1450 {
1451 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
1452 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
1453 unsigned short reg_addr;
1454 int status = 0;
1455 unsigned int scs_flag = 0;
1456 unsigned int csr_val = 0, newcsr_val = 0;
1457 u64 insts, savuword;
1458
1459 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1460 if (reg_addr == BAD_REGADDR) {
1461 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
1462 return EINVAL;
1463 }
1464 switch (reg_type) {
1465 case ICP_GPA_REL:
1466 insts = 0xA070000000ull | (reg_addr & 0x3ff);
1467 break;
1468 default:
1469 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
1470 break;
1471 }
1472 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
1473 scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS);
1474 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1475 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1476 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
1477 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl);
1478 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1479 ctx_enables &= IGNORE_W1C_MASK;
1480 if (ctx != (savctx & ACS_ACNO))
1481 qat_hal_wr_ae_csr(handle,
1482 ae,
1483 ACTIVE_CTX_STATUS,
1484 ctx & ACS_ACNO);
1485 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
1486 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1487 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
1488 uaddr = UA_ECS;
1489 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1490 insts = qat_hal_set_uword_ecc(insts);
1491 uwrd_lo = (unsigned int)(insts & 0xffffffff);
1492 uwrd_hi = (unsigned int)(insts >> 0x20);
1493 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1494 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1495 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1496 /* delay for at least 8 cycles */
1497 qat_hal_wait_cycles(handle, ae, 0x8, 0);
1498 /*
1499 * read ALU output
1500 * the instruction should have been executed
1501 * prior to clearing the ECS in putUwords
1502 */
1503 qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data);
1504 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1505 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1506 if (ctx != (savctx & ACS_ACNO))
1507 qat_hal_wr_ae_csr(handle,
1508 ae,
1509 ACTIVE_CTX_STATUS,
1510 savctx & ACS_ACNO);
1511 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1512 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
1513 newcsr_val = scs_flag ? SET_BIT(csr_val, MMC_SHARE_CS_BITPOS) :
1514 CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1515 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1516 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1517
1518 return status;
1519 }
1520
1521 static int
qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1522 qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1523 unsigned char ae,
1524 unsigned char ctx,
1525 enum icp_qat_uof_regtype reg_type,
1526 unsigned short reg_num,
1527 unsigned int data)
1528 {
1529 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1530 uint64_t insts[] = { 0x0F440000000ull,
1531 0x0F040000000ull,
1532 0x0F0000C0300ull,
1533 0x0E000010000ull };
1534 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1535 const int imm_w1 = 0, imm_w0 = 1;
1536
1537 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1538 if (dest_addr == BAD_REGADDR) {
1539 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1540 return EINVAL;
1541 }
1542
1543 data16lo = 0xffff & data;
1544 data16hi = 0xffff & (data >> 0x10);
1545 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1546 (unsigned short)(0xff & data16hi));
1547 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1548 (unsigned short)(0xff & data16lo));
1549 switch (reg_type) {
1550 case ICP_GPA_REL:
1551 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1552 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1553 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1554 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1555 break;
1556 default:
1557 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1558 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1559
1560 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1561 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1562 break;
1563 }
1564
1565 return qat_hal_exec_micro_inst(
1566 handle, ae, ctx, insts, num_inst, code_off, num_inst * 0x5, NULL);
1567 }
1568
1569 int
qat_hal_get_ins_num(void)1570 qat_hal_get_ins_num(void)
1571 {
1572 return ARRAY_SIZE(inst_4b);
1573 }
1574
1575 static int
qat_hal_concat_micro_code(uint64_t * micro_inst,unsigned int inst_num,unsigned int size,unsigned int addr,unsigned int * value)1576 qat_hal_concat_micro_code(uint64_t *micro_inst,
1577 unsigned int inst_num,
1578 unsigned int size,
1579 unsigned int addr,
1580 unsigned int *value)
1581 {
1582 int i;
1583 unsigned int cur_value;
1584 const uint64_t *inst_arr;
1585 unsigned int fixup_offset;
1586 int usize = 0;
1587 unsigned int orig_num;
1588 unsigned int delta;
1589
1590 orig_num = inst_num;
1591 fixup_offset = inst_num;
1592 cur_value = value[0];
1593 inst_arr = inst_4b;
1594 usize = ARRAY_SIZE(inst_4b);
1595 for (i = 0; i < usize; i++)
1596 micro_inst[inst_num++] = inst_arr[i];
1597 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1598 fixup_offset++;
1599 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1600 fixup_offset++;
1601 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1602 fixup_offset++;
1603 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1604
1605 delta = inst_num - orig_num;
1606
1607 return (int)delta;
1608 }
1609
1610 static int
qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,int * pfirst_exec,uint64_t * micro_inst,unsigned int inst_num)1611 qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1612 unsigned char ae,
1613 unsigned char ctx,
1614 int *pfirst_exec,
1615 uint64_t *micro_inst,
1616 unsigned int inst_num)
1617 {
1618 int stat = 0;
1619 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1620 unsigned int gprb0 = 0, gprb1 = 0;
1621
1622 if (*pfirst_exec) {
1623 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1624 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1625 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1626 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1627 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1628 *pfirst_exec = 0;
1629 }
1630 stat = qat_hal_exec_micro_inst(
1631 handle, ae, ctx, micro_inst, inst_num, 1, inst_num * 0x5, NULL);
1632 if (stat != 0)
1633 return EFAULT;
1634 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1635 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1636 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1637 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1638 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1639
1640 return 0;
1641 }
1642
1643 int
qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,struct icp_qat_uof_batch_init * lm_init_header)1644 qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1645 unsigned char ae,
1646 struct icp_qat_uof_batch_init *lm_init_header)
1647 {
1648 struct icp_qat_uof_batch_init *plm_init;
1649 uint64_t *micro_inst_arry;
1650 int micro_inst_num;
1651 int alloc_inst_size;
1652 int first_exec = 1;
1653 int stat = 0;
1654
1655 if (!lm_init_header)
1656 return 0;
1657 plm_init = lm_init_header->next;
1658 alloc_inst_size = lm_init_header->size;
1659 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1660 alloc_inst_size = handle->hal_handle->max_ustore;
1661 micro_inst_arry = malloc(alloc_inst_size * sizeof(uint64_t),
1662 M_QAT,
1663 M_WAITOK | M_ZERO);
1664 micro_inst_num = 0;
1665 while (plm_init) {
1666 unsigned int addr, *value, size;
1667
1668 ae = plm_init->ae;
1669 addr = plm_init->addr;
1670 value = plm_init->value;
1671 size = plm_init->size;
1672 micro_inst_num += qat_hal_concat_micro_code(
1673 micro_inst_arry, micro_inst_num, size, addr, value);
1674 plm_init = plm_init->next;
1675 }
1676 /* exec micro codes */
1677 if (micro_inst_arry && micro_inst_num > 0) {
1678 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1679 stat = qat_hal_exec_micro_init_lm(handle,
1680 ae,
1681 0,
1682 &first_exec,
1683 micro_inst_arry,
1684 micro_inst_num);
1685 }
1686 free(micro_inst_arry, M_QAT);
1687 return stat;
1688 }
1689
1690 static int
qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int val)1691 qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1692 unsigned char ae,
1693 unsigned char ctx,
1694 enum icp_qat_uof_regtype reg_type,
1695 unsigned short reg_num,
1696 unsigned int val)
1697 {
1698 int status = 0;
1699 unsigned int reg_addr;
1700 unsigned int ctx_enables;
1701 unsigned short mask;
1702 unsigned short dr_offset = 0x10;
1703
1704 status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1705 if (CE_INUSE_CONTEXTS & ctx_enables) {
1706 if (ctx & 0x1) {
1707 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1708 return EINVAL;
1709 }
1710 mask = 0x1f;
1711 dr_offset = 0x20;
1712 } else {
1713 mask = 0x0f;
1714 }
1715 if (reg_num & ~mask)
1716 return EINVAL;
1717 reg_addr = reg_num + (ctx << 0x5);
1718 switch (reg_type) {
1719 case ICP_SR_RD_REL:
1720 case ICP_SR_REL:
1721 SET_AE_XFER(handle, ae, reg_addr, val);
1722 break;
1723 case ICP_DR_RD_REL:
1724 case ICP_DR_REL:
1725 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1726 break;
1727 default:
1728 status = EINVAL;
1729 break;
1730 }
1731 return status;
1732 }
1733
1734 static int
qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1735 qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1736 unsigned char ae,
1737 unsigned char ctx,
1738 enum icp_qat_uof_regtype reg_type,
1739 unsigned short reg_num,
1740 unsigned int data)
1741 {
1742 unsigned int gprval, ctx_enables;
1743 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1744 data16low;
1745 unsigned short reg_mask;
1746 int status = 0;
1747 uint64_t micro_inst[] = { 0x0F440000000ull,
1748 0x0F040000000ull,
1749 0x0A000000000ull,
1750 0x0F0000C0300ull,
1751 0x0E000010000ull };
1752 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1753 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1754
1755 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1756 if (CE_INUSE_CONTEXTS & ctx_enables) {
1757 if (ctx & 0x1) {
1758 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1759 return EINVAL;
1760 }
1761 reg_mask = (unsigned short)~0x1f;
1762 } else {
1763 reg_mask = (unsigned short)~0xf;
1764 }
1765 if (reg_num & reg_mask)
1766 return EINVAL;
1767 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1768 if (xfr_addr == BAD_REGADDR) {
1769 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1770 return EINVAL;
1771 }
1772 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1773 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1774 data16low = 0xffff & data;
1775 data16hi = 0xffff & (data >> 0x10);
1776 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1777 (unsigned short)(0xff & data16hi));
1778 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1779 (unsigned short)(0xff & data16low));
1780 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1781 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1782 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1783 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1784 micro_inst[0x2] = micro_inst[0x2] | ((xfr_addr & 0x3ff) << 20) |
1785 ((gpr_addr & 0x3ff) << 10);
1786 status = qat_hal_exec_micro_inst(
1787 handle, ae, ctx, micro_inst, num_inst, code_off, dly, NULL);
1788 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1789 return status;
1790 }
1791
1792 static int
qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned short nn,unsigned int val)1793 qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1794 unsigned char ae,
1795 unsigned char ctx,
1796 unsigned short nn,
1797 unsigned int val)
1798 {
1799 unsigned int ctx_enables;
1800 int stat = 0;
1801
1802 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1803 ctx_enables &= IGNORE_W1C_MASK;
1804 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1805
1806 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1807 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1808 return stat;
1809 }
1810
1811 static int
qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned short absreg_num,unsigned short * relreg,unsigned char * ctx)1812 qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle *handle,
1813 unsigned char ae,
1814 unsigned short absreg_num,
1815 unsigned short *relreg,
1816 unsigned char *ctx)
1817 {
1818 unsigned int ctx_enables;
1819
1820 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1821 if (ctx_enables & CE_INUSE_CONTEXTS) {
1822 /* 4-ctx mode */
1823 *relreg = absreg_num & 0x1F;
1824 *ctx = (absreg_num >> 0x4) & 0x6;
1825 } else {
1826 /* 8-ctx mode */
1827 *relreg = absreg_num & 0x0F;
1828 *ctx = (absreg_num >> 0x4) & 0x7;
1829 }
1830 return 0;
1831 }
1832
1833 int
qat_hal_init_gpr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1834 qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1835 unsigned char ae,
1836 unsigned long ctx_mask,
1837 enum icp_qat_uof_regtype reg_type,
1838 unsigned short reg_num,
1839 unsigned int regdata)
1840 {
1841 int stat = 0;
1842 unsigned short reg;
1843 unsigned char ctx = 0;
1844 enum icp_qat_uof_regtype type;
1845
1846 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1847 return EINVAL;
1848
1849 do {
1850 if (ctx_mask == 0) {
1851 qat_hal_convert_abs_to_rel(
1852 handle, ae, reg_num, ®, &ctx);
1853 type = reg_type - 1;
1854 } else {
1855 reg = reg_num;
1856 type = reg_type;
1857 if (!test_bit(ctx, &ctx_mask))
1858 continue;
1859 }
1860 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1861 if (stat) {
1862 pr_err("QAT: write gpr fail\n");
1863 return EINVAL;
1864 }
1865 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1866
1867 return 0;
1868 }
1869
1870 int
qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1871 qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1872 unsigned char ae,
1873 unsigned long ctx_mask,
1874 enum icp_qat_uof_regtype reg_type,
1875 unsigned short reg_num,
1876 unsigned int regdata)
1877 {
1878 int stat = 0;
1879 unsigned short reg;
1880 unsigned char ctx = 0;
1881 enum icp_qat_uof_regtype type;
1882
1883 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1884 return EINVAL;
1885
1886 do {
1887 if (ctx_mask == 0) {
1888 qat_hal_convert_abs_to_rel(
1889 handle, ae, reg_num, ®, &ctx);
1890 type = reg_type - 3;
1891 } else {
1892 reg = reg_num;
1893 type = reg_type;
1894 if (!test_bit(ctx, &ctx_mask))
1895 continue;
1896 }
1897 stat = qat_hal_put_rel_wr_xfer(
1898 handle, ae, ctx, type, reg, regdata);
1899 if (stat) {
1900 pr_err("QAT: write wr xfer fail\n");
1901 return EINVAL;
1902 }
1903 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1904
1905 return 0;
1906 }
1907
1908 int
qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1909 qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1910 unsigned char ae,
1911 unsigned long ctx_mask,
1912 enum icp_qat_uof_regtype reg_type,
1913 unsigned short reg_num,
1914 unsigned int regdata)
1915 {
1916 int stat = 0;
1917 unsigned short reg;
1918 unsigned char ctx = 0;
1919 enum icp_qat_uof_regtype type;
1920
1921 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1922 return EINVAL;
1923
1924 do {
1925 if (ctx_mask == 0) {
1926 qat_hal_convert_abs_to_rel(
1927 handle, ae, reg_num, ®, &ctx);
1928 type = reg_type - 3;
1929 } else {
1930 reg = reg_num;
1931 type = reg_type;
1932 if (!test_bit(ctx, &ctx_mask))
1933 continue;
1934 }
1935 stat = qat_hal_put_rel_rd_xfer(
1936 handle, ae, ctx, type, reg, regdata);
1937 if (stat) {
1938 pr_err("QAT: write rd xfer fail\n");
1939 return EINVAL;
1940 }
1941 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1942
1943 return 0;
1944 }
1945
1946 int
qat_hal_init_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned long ctx_mask,unsigned short reg_num,unsigned int regdata)1947 qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1948 unsigned char ae,
1949 unsigned long ctx_mask,
1950 unsigned short reg_num,
1951 unsigned int regdata)
1952 {
1953 int stat = 0;
1954 unsigned char ctx;
1955
1956 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
1957 pr_err("QAT: No next neigh for CPM2X\n");
1958 return EINVAL;
1959 }
1960
1961 if (ctx_mask == 0)
1962 return EINVAL;
1963
1964 for_each_set_bit(ctx, &ctx_mask, ICP_QAT_UCLO_MAX_CTX)
1965 {
1966 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1967 if (stat) {
1968 pr_err("QAT: write neigh error\n");
1969 return EINVAL;
1970 }
1971 }
1972
1973 return 0;
1974 }
1975