1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
2 /* $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */
3
4 /*
5 * Copyright (c) 2019 Internet Initiative Japan, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 *
37 * * Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * * Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in
41 * the documentation and/or other materials provided with the
42 * distribution.
43 * * Neither the name of Intel Corporation nor the names of its
44 * contributors may be used to endorse or promote products derived
45 * from this software without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 */
59
60 #include <sys/cdefs.h>
61 #if 0
62 __KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
63 #endif
64
65 #include <sys/param.h>
66 #include <sys/bus.h>
67 #include <sys/firmware.h>
68 #include <sys/limits.h>
69 #include <sys/systm.h>
70
71 #include <machine/bus.h>
72
73 #include <dev/pci/pcireg.h>
74 #include <dev/pci/pcivar.h>
75
76 #include "qatreg.h"
77 #include "qatvar.h"
78 #include "qat_aevar.h"
79
80 static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
81 uint32_t);
82 static int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t,
83 uint32_t *);
84 static void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t,
85 bus_size_t, uint32_t);
86 static int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t,
87 bus_size_t, uint32_t *);
88
89 static u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short);
90 static int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char,
91 enum aereg_type, u_short, uint32_t);
92 static int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char,
93 enum aereg_type, u_short, uint32_t *);
94 static int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char,
95 enum aereg_type, u_short, uint32_t);
96 static int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char,
97 enum aereg_type, u_short, uint32_t);
98 static int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char,
99 enum aereg_type, u_short, uint32_t);
100 static int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short,
101 u_short *, u_char *);
102 static int qat_aereg_abs_data_write(struct qat_softc *, u_char,
103 enum aereg_type, u_short, uint32_t);
104
105 static void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int);
106 static void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int);
107 static void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char);
108 static void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char);
109 static void qat_ae_write_lm_mode(struct qat_softc *, u_char,
110 enum aereg_type, u_char);
111 static void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char,
112 u_char);
113 static void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char);
114 static int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int,
115 u_int);
116
117 static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char);
118 static int qat_ae_is_active(struct qat_softc *, u_char);
119 static int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int);
120
121 static int qat_ae_clear_reset(struct qat_softc *);
122 static int qat_ae_check(struct qat_softc *);
123 static int qat_ae_reset_timestamp(struct qat_softc *);
124 static void qat_ae_clear_xfer(struct qat_softc *);
125 static int qat_ae_clear_gprs(struct qat_softc *);
126
127 static void qat_ae_get_shared_ustore_ae(u_char, u_char *);
128 static u_int qat_ae_ucode_parity64(uint64_t);
129 static uint64_t qat_ae_ucode_set_ecc(uint64_t);
130 static int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int,
131 const uint64_t *);
132 static int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int,
133 uint64_t *);
134 static u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *);
135 static int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char,
136 uint64_t *, u_int, int, u_int, u_int *);
137 static int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char,
138 int *, uint64_t *, u_int,
139 u_int *, u_int *, u_int *, u_int *, u_int *);
140 static int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char,
141 u_int, u_int, u_int, u_int, u_int);
142 static int qat_ae_get_inst_num(int);
143 static int qat_ae_batch_put_lm(struct qat_softc *, u_char,
144 struct qat_ae_batch_init_list *, size_t);
145 static int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int);
146
147 static u_int qat_aefw_csum(char *, int);
148 static const char *qat_aefw_uof_string(struct qat_softc *, size_t);
149 static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *,
150 const char *, struct uof_chunk_hdr *);
151
152 static int qat_aefw_load_mof(struct qat_softc *);
153 static void qat_aefw_unload_mof(struct qat_softc *);
154 static int qat_aefw_load_mmp(struct qat_softc *);
155 static void qat_aefw_unload_mmp(struct qat_softc *);
156
157 static int qat_aefw_mof_find_uof0(struct qat_softc *,
158 struct mof_uof_hdr *, struct mof_uof_chunk_hdr *,
159 u_int, size_t, const char *,
160 size_t *, void **);
161 static int qat_aefw_mof_find_uof(struct qat_softc *);
162 static int qat_aefw_mof_parse(struct qat_softc *);
163
164 static int qat_aefw_uof_parse_image(struct qat_softc *,
165 struct qat_uof_image *, struct uof_chunk_hdr *uch);
166 static int qat_aefw_uof_parse_images(struct qat_softc *);
167 static int qat_aefw_uof_parse(struct qat_softc *);
168
169 static int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t,
170 struct qat_dmamem *);
171 static int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *);
172 static int qat_aefw_suof_load(struct qat_softc *sc,
173 struct qat_dmamem *dma);
174 static int qat_aefw_suof_parse_image(struct qat_softc *,
175 struct qat_suof_image *, struct suof_chunk_hdr *);
176 static int qat_aefw_suof_parse(struct qat_softc *);
177 static int qat_aefw_suof_write(struct qat_softc *);
178
179 static int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *,
180 struct qat_uof_image *);
181 static int qat_aefw_uof_init_ae(struct qat_softc *, u_char);
182 static int qat_aefw_uof_init(struct qat_softc *);
183
184 static int qat_aefw_init_memory_one(struct qat_softc *,
185 struct uof_init_mem *);
186 static void qat_aefw_free_lm_init(struct qat_softc *, u_char);
187 static int qat_aefw_init_ustore(struct qat_softc *);
188 static int qat_aefw_init_reg(struct qat_softc *, u_char, u_char,
189 enum aereg_type, u_short, u_int);
190 static int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char,
191 struct qat_uof_image *);
192 static int qat_aefw_init_memory(struct qat_softc *);
193 static int qat_aefw_init_globals(struct qat_softc *);
194 static uint64_t qat_aefw_get_uof_inst(struct qat_softc *,
195 struct qat_uof_page *, u_int);
196 static int qat_aefw_do_pagein(struct qat_softc *, u_char,
197 struct qat_uof_page *);
198 static int qat_aefw_uof_write_one(struct qat_softc *,
199 struct qat_uof_image *);
200 static int qat_aefw_uof_write(struct qat_softc *);
201
202 static int
qat_ae_write_4(struct qat_softc * sc,u_char ae,bus_size_t offset,uint32_t value)203 qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
204 uint32_t value)
205 {
206 int times = TIMEOUT_AE_CSR;
207
208 do {
209 qat_ae_local_write_4(sc, ae, offset, value);
210 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
211 LOCAL_CSR_STATUS_STATUS) == 0)
212 return 0;
213
214 } while (times--);
215
216 device_printf(sc->sc_dev,
217 "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
218 return EFAULT;
219 }
220
221 static int
qat_ae_read_4(struct qat_softc * sc,u_char ae,bus_size_t offset,uint32_t * value)222 qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
223 uint32_t *value)
224 {
225 int times = TIMEOUT_AE_CSR;
226 uint32_t v;
227
228 do {
229 v = qat_ae_local_read_4(sc, ae, offset);
230 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
231 LOCAL_CSR_STATUS_STATUS) == 0) {
232 *value = v;
233 return 0;
234 }
235 } while (times--);
236
237 device_printf(sc->sc_dev,
238 "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
239 return EFAULT;
240 }
241
242 static void
qat_ae_ctx_indr_write(struct qat_softc * sc,u_char ae,uint32_t ctx_mask,bus_size_t offset,uint32_t value)243 qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask,
244 bus_size_t offset, uint32_t value)
245 {
246 int ctx;
247 uint32_t ctxptr;
248
249 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
250 offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
251 offset == CTX_STS_INDIRECT ||
252 offset == CTX_WAKEUP_EVENTS_INDIRECT ||
253 offset == CTX_SIG_EVENTS_INDIRECT ||
254 offset == LM_ADDR_0_INDIRECT ||
255 offset == LM_ADDR_1_INDIRECT ||
256 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
257 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);
258
259 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
260 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
261 if ((ctx_mask & (1 << ctx)) == 0)
262 continue;
263 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
264 qat_ae_write_4(sc, ae, offset, value);
265 }
266 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
267 }
268
269 static int
qat_ae_ctx_indr_read(struct qat_softc * sc,u_char ae,uint32_t ctx,bus_size_t offset,uint32_t * value)270 qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx,
271 bus_size_t offset, uint32_t *value)
272 {
273 int error;
274 uint32_t ctxptr;
275
276 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
277 offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
278 offset == CTX_STS_INDIRECT ||
279 offset == CTX_WAKEUP_EVENTS_INDIRECT ||
280 offset == CTX_SIG_EVENTS_INDIRECT ||
281 offset == LM_ADDR_0_INDIRECT ||
282 offset == LM_ADDR_1_INDIRECT ||
283 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
284 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);
285
286 /* save the ctx ptr */
287 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
288 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
289 (ctx & CSR_CTX_POINTER_CONTEXT))
290 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
291
292 error = qat_ae_read_4(sc, ae, offset, value);
293
294 /* restore ctx ptr */
295 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
296 (ctx & CSR_CTX_POINTER_CONTEXT))
297 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
298
299 return error;
300 }
301
302 static u_short
qat_aereg_get_10bit_addr(enum aereg_type regtype,u_short reg)303 qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg)
304 {
305 u_short addr;
306
307 switch (regtype) {
308 case AEREG_GPA_ABS:
309 case AEREG_GPB_ABS:
310 addr = (reg & 0x7f) | 0x80;
311 break;
312 case AEREG_GPA_REL:
313 case AEREG_GPB_REL:
314 addr = reg & 0x1f;
315 break;
316 case AEREG_SR_RD_REL:
317 case AEREG_SR_WR_REL:
318 case AEREG_SR_REL:
319 addr = 0x180 | (reg & 0x1f);
320 break;
321 case AEREG_SR_INDX:
322 addr = 0x140 | ((reg & 0x3) << 1);
323 break;
324 case AEREG_DR_RD_REL:
325 case AEREG_DR_WR_REL:
326 case AEREG_DR_REL:
327 addr = 0x1c0 | (reg & 0x1f);
328 break;
329 case AEREG_DR_INDX:
330 addr = 0x100 | ((reg & 0x3) << 1);
331 break;
332 case AEREG_NEIGH_INDX:
333 addr = 0x241 | ((reg & 0x3) << 1);
334 break;
335 case AEREG_NEIGH_REL:
336 addr = 0x280 | (reg & 0x1f);
337 break;
338 case AEREG_LMEM0:
339 addr = 0x200;
340 break;
341 case AEREG_LMEM1:
342 addr = 0x220;
343 break;
344 case AEREG_NO_DEST:
345 addr = 0x300 | (reg & 0xff);
346 break;
347 default:
348 addr = AEREG_BAD_REGADDR;
349 break;
350 }
351 return (addr);
352 }
353
354 static int
qat_aereg_rel_data_write(struct qat_softc * sc,u_char ae,u_char ctx,enum aereg_type regtype,u_short relreg,uint32_t value)355 qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx,
356 enum aereg_type regtype, u_short relreg, uint32_t value)
357 {
358 uint16_t srchi, srclo, destaddr, data16hi, data16lo;
359 uint64_t inst[] = {
360 0x0F440000000ull, /* immed_w1[reg, val_hi16] */
361 0x0F040000000ull, /* immed_w0[reg, val_lo16] */
362 0x0F0000C0300ull, /* nop */
363 0x0E000010000ull /* ctx_arb[kill] */
364 };
365 const int ninst = nitems(inst);
366 const int imm_w1 = 0, imm_w0 = 1;
367 unsigned int ctxen;
368 uint16_t mask;
369
370 /* This logic only works for GPRs and LM index registers,
371 not NN or XFER registers! */
372 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
373 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);
374
375 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) {
376 /* determine the context mode */
377 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
378 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
379 /* 4-ctx mode */
380 if (ctx & 0x1)
381 return EINVAL;
382 mask = 0x1f;
383 } else {
384 /* 8-ctx mode */
385 mask = 0x0f;
386 }
387 if (relreg & ~mask)
388 return EINVAL;
389 }
390 if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
391 AEREG_BAD_REGADDR) {
392 return EINVAL;
393 }
394
395 data16lo = 0xffff & value;
396 data16hi = 0xffff & (value >> 16);
397 srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
398 (uint16_t)(0xff & data16hi));
399 srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
400 (uint16_t)(0xff & data16lo));
401
402 switch (regtype) {
403 case AEREG_GPA_REL: /* A rel source */
404 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
405 ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff);
406 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
407 ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff);
408 break;
409 default:
410 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
411 ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff);
412 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
413 ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff);
414 break;
415 }
416
417 return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL);
418 }
419
420 static int
qat_aereg_rel_data_read(struct qat_softc * sc,u_char ae,u_char ctx,enum aereg_type regtype,u_short relreg,uint32_t * value)421 qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx,
422 enum aereg_type regtype, u_short relreg, uint32_t *value)
423 {
424 uint64_t inst, savucode;
425 uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi;
426 u_int uaddr, ustore_addr;
427 int error;
428 u_short mask, regaddr;
429 u_char nae;
430
431 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
432 regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL ||
433 regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL ||
434 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);
435
436 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) ||
437 (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) ||
438 (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL))
439 {
440 /* determine the context mode */
441 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
442 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
443 /* 4-ctx mode */
444 if (ctx & 0x1)
445 return EINVAL;
446 mask = 0x1f;
447 } else {
448 /* 8-ctx mode */
449 mask = 0x0f;
450 }
451 if (relreg & ~mask)
452 return EINVAL;
453 }
454 if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
455 AEREG_BAD_REGADDR) {
456 return EINVAL;
457 }
458
459 /* instruction -- alu[--, --, B, reg] */
460 switch (regtype) {
461 case AEREG_GPA_REL:
462 /* A rel source */
463 inst = 0xA070000000ull | (regaddr & 0x3ff);
464 break;
465 default:
466 inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10));
467 break;
468 }
469
470 /* backup shared control store bit, and force AE to
471 * none-shared mode before executing ucode snippet */
472 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
473 if (misc & AE_MISC_CONTROL_SHARE_CS) {
474 qat_ae_get_shared_ustore_ae(ae, &nae);
475 if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae))
476 return EBUSY;
477 }
478
479 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
480 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
481
482 /* read current context */
483 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
484 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);
485
486 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
487 /* prevent clearing the W1C bits: the breakpoint bit,
488 ECC error bit, and Parity error bit */
489 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
490
491 /* change the context */
492 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO))
493 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
494 ctx & ACTIVE_CTX_STATUS_ACNO);
495 /* save a ustore location */
496 if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) {
497 /* restore AE_MISC_CONTROL csr */
498 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
499
500 /* restore the context */
501 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
502 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
503 savctx & ACTIVE_CTX_STATUS_ACNO);
504 }
505 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
506
507 return (error);
508 }
509
510 /* turn off ustore parity */
511 qat_ae_write_4(sc, ae, CTX_ENABLES,
512 ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE));
513
514 /* save ustore-addr csr */
515 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
516
517 /* write the ALU instruction to ustore, enable ecs bit */
518 uaddr = 0 | USTORE_ADDRESS_ECS;
519
520 /* set the uaddress */
521 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
522 inst = qat_ae_ucode_set_ecc(inst);
523
524 ulo = (uint32_t)(inst & 0xffffffff);
525 uhi = (uint32_t)(inst >> 32);
526
527 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
528
529 /* this will auto increment the address */
530 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);
531
532 /* set the uaddress */
533 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
534
535 /* delay for at least 8 cycles */
536 qat_ae_wait_num_cycles(sc, ae, 0x8, 0);
537
538 /* read ALU output -- the instruction should have been executed
539 prior to clearing the ECS in putUwords */
540 qat_ae_read_4(sc, ae, ALU_OUT, value);
541
542 /* restore ustore-addr csr */
543 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);
544
545 /* restore the ustore */
546 error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode);
547
548 /* restore the context */
549 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
550 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
551 savctx & ACTIVE_CTX_STATUS_ACNO);
552 }
553
554 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
555
556 /* restore AE_MISC_CONTROL csr */
557 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
558
559 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
560
561 return error;
562 }
563
564 static int
qat_aereg_rel_rdxfer_write(struct qat_softc * sc,u_char ae,u_char ctx,enum aereg_type regtype,u_short relreg,uint32_t value)565 qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
566 enum aereg_type regtype, u_short relreg, uint32_t value)
567 {
568 bus_size_t addr;
569 int error;
570 uint32_t ctxen;
571 u_short mask;
572 u_short dr_offset;
573
574 MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL ||
575 regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL);
576
577 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
578 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
579 if (ctx & 0x1) {
580 device_printf(sc->sc_dev,
581 "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx);
582 return EINVAL;
583 }
584 mask = 0x1f;
585 dr_offset = 0x20;
586
587 } else {
588 mask = 0x0f;
589 dr_offset = 0x10;
590 }
591
592 if (relreg & ~mask)
593 return EINVAL;
594
595 addr = relreg + (ctx << 0x5);
596
597 switch (regtype) {
598 case AEREG_SR_REL:
599 case AEREG_SR_RD_REL:
600 qat_ae_xfer_write_4(sc, ae, addr, value);
601 break;
602 case AEREG_DR_REL:
603 case AEREG_DR_RD_REL:
604 qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value);
605 break;
606 default:
607 error = EINVAL;
608 }
609
610 return error;
611 }
612
613 static int
qat_aereg_rel_wrxfer_write(struct qat_softc * sc,u_char ae,u_char ctx,enum aereg_type regtype,u_short relreg,uint32_t value)614 qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
615 enum aereg_type regtype, u_short relreg, uint32_t value)
616 {
617
618 panic("notyet");
619
620 return 0;
621 }
622
623 static int
qat_aereg_rel_nn_write(struct qat_softc * sc,u_char ae,u_char ctx,enum aereg_type regtype,u_short relreg,uint32_t value)624 qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx,
625 enum aereg_type regtype, u_short relreg, uint32_t value)
626 {
627
628 panic("notyet");
629
630 return 0;
631 }
632
633 static int
qat_aereg_abs_to_rel(struct qat_softc * sc,u_char ae,u_short absreg,u_short * relreg,u_char * ctx)634 qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae,
635 u_short absreg, u_short *relreg, u_char *ctx)
636 {
637 uint32_t ctxen;
638
639 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
640 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
641 /* 4-ctx mode */
642 *relreg = absreg & 0x1f;
643 *ctx = (absreg >> 0x4) & 0x6;
644 } else {
645 /* 8-ctx mode */
646 *relreg = absreg & 0x0f;
647 *ctx = (absreg >> 0x4) & 0x7;
648 }
649
650 return 0;
651 }
652
653 static int
qat_aereg_abs_data_write(struct qat_softc * sc,u_char ae,enum aereg_type regtype,u_short absreg,uint32_t value)654 qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae,
655 enum aereg_type regtype, u_short absreg, uint32_t value)
656 {
657 int error;
658 u_short relreg;
659 u_char ctx;
660
661 qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx);
662
663 switch (regtype) {
664 case AEREG_GPA_ABS:
665 MPASS(absreg < MAX_GPR_REG);
666 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL,
667 relreg, value);
668 break;
669 case AEREG_GPB_ABS:
670 MPASS(absreg < MAX_GPR_REG);
671 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL,
672 relreg, value);
673 break;
674 case AEREG_DR_RD_ABS:
675 MPASS(absreg < MAX_XFER_REG);
676 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL,
677 relreg, value);
678 break;
679 case AEREG_SR_RD_ABS:
680 MPASS(absreg < MAX_XFER_REG);
681 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL,
682 relreg, value);
683 break;
684 case AEREG_DR_WR_ABS:
685 MPASS(absreg < MAX_XFER_REG);
686 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL,
687 relreg, value);
688 break;
689 case AEREG_SR_WR_ABS:
690 MPASS(absreg < MAX_XFER_REG);
691 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL,
692 relreg, value);
693 break;
694 case AEREG_NEIGH_ABS:
695 MPASS(absreg < MAX_NN_REG);
696 if (absreg >= MAX_NN_REG)
697 return EINVAL;
698 error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL,
699 relreg, value);
700 break;
701 default:
702 panic("Invalid Register Type");
703 }
704
705 return error;
706 }
707
708 static void
qat_ae_enable_ctx(struct qat_softc * sc,u_char ae,u_int ctx_mask)709 qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
710 {
711 uint32_t ctxen;
712
713 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
714 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
715
716 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
717 ctx_mask &= 0x55;
718 } else {
719 ctx_mask &= 0xff;
720 }
721
722 ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE);
723 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
724 }
725
726 static void
qat_ae_disable_ctx(struct qat_softc * sc,u_char ae,u_int ctx_mask)727 qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
728 {
729 uint32_t ctxen;
730
731 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
732 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
733 ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE));
734 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
735 }
736
737 static void
qat_ae_write_ctx_mode(struct qat_softc * sc,u_char ae,u_char mode)738 qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode)
739 {
740 uint32_t val, nval;
741
742 qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
743 val &= CTX_ENABLES_IGNORE_W1C_MASK;
744
745 if (mode == 4)
746 nval = val | CTX_ENABLES_INUSE_CONTEXTS;
747 else
748 nval = val & ~CTX_ENABLES_INUSE_CONTEXTS;
749
750 if (val != nval)
751 qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
752 }
753
754 static void
qat_ae_write_nn_mode(struct qat_softc * sc,u_char ae,u_char mode)755 qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode)
756 {
757 uint32_t val, nval;
758
759 qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
760 val &= CTX_ENABLES_IGNORE_W1C_MASK;
761
762 if (mode)
763 nval = val | CTX_ENABLES_NN_MODE;
764 else
765 nval = val & ~CTX_ENABLES_NN_MODE;
766
767 if (val != nval)
768 qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
769 }
770
771 static void
qat_ae_write_lm_mode(struct qat_softc * sc,u_char ae,enum aereg_type lm,u_char mode)772 qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae,
773 enum aereg_type lm, u_char mode)
774 {
775 uint32_t val, nval;
776 uint32_t bit;
777
778 qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
779 val &= CTX_ENABLES_IGNORE_W1C_MASK;
780
781 switch (lm) {
782 case AEREG_LMEM0:
783 bit = CTX_ENABLES_LMADDR_0_GLOBAL;
784 break;
785 case AEREG_LMEM1:
786 bit = CTX_ENABLES_LMADDR_1_GLOBAL;
787 break;
788 default:
789 panic("invalid lmem reg type");
790 break;
791 }
792
793 if (mode)
794 nval = val | bit;
795 else
796 nval = val & ~bit;
797
798 if (val != nval)
799 qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
800 }
801
802 static void
qat_ae_write_shared_cs_mode0(struct qat_softc * sc,u_char ae,u_char mode)803 qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode)
804 {
805 uint32_t val, nval;
806
807 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
808
809 if (mode == 1)
810 nval = val | AE_MISC_CONTROL_SHARE_CS;
811 else
812 nval = val & ~AE_MISC_CONTROL_SHARE_CS;
813
814 if (val != nval)
815 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval);
816 }
817
818 static void
qat_ae_write_shared_cs_mode(struct qat_softc * sc,u_char ae,u_char mode)819 qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode)
820 {
821 u_char nae;
822
823 qat_ae_get_shared_ustore_ae(ae, &nae);
824
825 qat_ae_write_shared_cs_mode0(sc, ae, mode);
826
827 if ((sc->sc_ae_mask & (1 << nae))) {
828 qat_ae_write_shared_cs_mode0(sc, nae, mode);
829 }
830 }
831
832 static int
qat_ae_set_reload_ustore(struct qat_softc * sc,u_char ae,u_int reload_size,int shared_mode,u_int ustore_dram_addr)833 qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae,
834 u_int reload_size, int shared_mode, u_int ustore_dram_addr)
835 {
836 uint32_t val, cs_reload;
837
838 switch (reload_size) {
839 case 0:
840 cs_reload = 0x0;
841 break;
842 case QAT_2K:
843 cs_reload = 0x1;
844 break;
845 case QAT_4K:
846 cs_reload = 0x2;
847 break;
848 case QAT_8K:
849 cs_reload = 0x3;
850 break;
851 default:
852 return EINVAL;
853 }
854
855 if (cs_reload)
856 QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr;
857
858 QAT_AE(sc, ae).qae_reload_size = reload_size;
859
860 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
861 val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD |
862 AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS);
863 val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) |
864 __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD);
865 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);
866
867 return 0;
868 }
869
870 static enum qat_ae_status
qat_ae_get_status(struct qat_softc * sc,u_char ae)871 qat_ae_get_status(struct qat_softc *sc, u_char ae)
872 {
873 int error;
874 uint32_t val = 0;
875
876 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
877 if (error || val & CTX_ENABLES_ENABLE)
878 return QAT_AE_ENABLED;
879
880 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
881 if (val & ACTIVE_CTX_STATUS_ABO)
882 return QAT_AE_ACTIVE;
883
884 return QAT_AE_DISABLED;
885 }
886
887
888 static int
qat_ae_is_active(struct qat_softc * sc,u_char ae)889 qat_ae_is_active(struct qat_softc *sc, u_char ae)
890 {
891 uint32_t val;
892
893 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
894 return 1;
895
896 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
897 if (val & ACTIVE_CTX_STATUS_ABO)
898 return 1;
899 else
900 return 0;
901 }
902
903 /* returns 1 if actually waited for specified number of cycles */
904 static int
qat_ae_wait_num_cycles(struct qat_softc * sc,u_char ae,int cycles,int check)905 qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check)
906 {
907 uint32_t cnt, actx;
908 int pcnt, ccnt, elapsed, times;
909
910 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
911 pcnt = cnt & 0xffff;
912
913 times = TIMEOUT_AE_CHECK;
914 do {
915 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
916 ccnt = cnt & 0xffff;
917
918 elapsed = ccnt - pcnt;
919 if (elapsed == 0) {
920 times--;
921 }
922 if (times <= 0) {
923 device_printf(sc->sc_dev,
924 "qat_ae_wait_num_cycles timeout\n");
925 return -1;
926 }
927
928 if (elapsed < 0)
929 elapsed += 0x10000;
930
931 if (elapsed >= CYCLES_FROM_READY2EXE && check) {
932 if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS,
933 &actx) == 0) {
934 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
935 return 0;
936 }
937 }
938 } while (cycles > elapsed);
939
940 if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) {
941 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
942 return 0;
943 }
944
945 return 1;
946 }
947
948 int
qat_ae_init(struct qat_softc * sc)949 qat_ae_init(struct qat_softc *sc)
950 {
951 int error;
952 uint32_t mask, val = 0;
953 u_char ae;
954
955 /* XXX adf_initSysMemInfo */
956
957 /* XXX Disable clock gating for some chip if debug mode */
958
959 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
960 struct qat_ae *qae = &sc->sc_ae[ae];
961 if (!(mask & 1))
962 continue;
963
964 qae->qae_ustore_size = USTORE_SIZE;
965
966 qae->qae_free_addr = 0;
967 qae->qae_free_size = USTORE_SIZE;
968 qae->qae_live_ctx_mask = AE_ALL_CTX;
969 qae->qae_ustore_dram_addr = 0;
970 qae->qae_reload_size = 0;
971 }
972
973 /* XXX Enable attention interrupt */
974
975 error = qat_ae_clear_reset(sc);
976 if (error)
977 return error;
978
979 qat_ae_clear_xfer(sc);
980
981 if (!sc->sc_hw.qhw_fw_auth) {
982 error = qat_ae_clear_gprs(sc);
983 if (error)
984 return error;
985 }
986
987 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
988 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
989 if (!(mask & 1))
990 continue;
991 qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val);
992 val |= 0x1;
993 qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val);
994 }
995
996 error = qat_ae_clear_reset(sc);
997 if (error)
998 return error;
999
1000 /* XXX XXX XXX Clean MMP memory if mem scrub is supported */
1001 /* halMem_ScrubMMPMemory */
1002
1003 return 0;
1004 }
1005
1006 int
qat_ae_start(struct qat_softc * sc)1007 qat_ae_start(struct qat_softc *sc)
1008 {
1009 int error;
1010 u_char ae;
1011
1012 for (ae = 0; ae < sc->sc_ae_num; ae++) {
1013 if ((sc->sc_ae_mask & (1 << ae)) == 0)
1014 continue;
1015
1016 error = qat_aefw_start(sc, ae, 0xff);
1017 if (error)
1018 return error;
1019 }
1020
1021 return 0;
1022 }
1023
1024 void
qat_ae_cluster_intr(void * arg)1025 qat_ae_cluster_intr(void *arg)
1026 {
1027 /* Nothing to implement until we support SRIOV. */
1028 printf("qat_ae_cluster_intr\n");
1029 }
1030
1031 static int
qat_ae_clear_reset(struct qat_softc * sc)1032 qat_ae_clear_reset(struct qat_softc *sc)
1033 {
1034 int error;
1035 uint32_t times, reset, clock, reg, mask;
1036 u_char ae;
1037
1038 reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
1039 reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK));
1040 reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK));
1041 times = TIMEOUT_AE_RESET;
1042 do {
1043 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset);
1044 if ((times--) == 0) {
1045 device_printf(sc->sc_dev, "couldn't reset AEs\n");
1046 return EBUSY;
1047 }
1048 reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
1049 } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) |
1050 __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK))
1051 & reg);
1052
1053 /* Enable clock for AE and QAT */
1054 clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN);
1055 clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK);
1056 clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK);
1057 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock);
1058
1059 error = qat_ae_check(sc);
1060 if (error)
1061 return error;
1062
1063 /*
1064 * Set undefined power-up/reset states to reasonable default values...
1065 * just to make sure we're starting from a known point
1066 */
1067 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
1068 if (!(mask & 1))
1069 continue;
1070
1071 /* init the ctx_enable */
1072 qat_ae_write_4(sc, ae, CTX_ENABLES,
1073 CTX_ENABLES_INIT);
1074
1075 /* initialize the PCs */
1076 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1077 CTX_STS_INDIRECT,
1078 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
1079
1080 /* init the ctx_arb */
1081 qat_ae_write_4(sc, ae, CTX_ARB_CNTL,
1082 CTX_ARB_CNTL_INIT);
1083
1084 /* enable cc */
1085 qat_ae_write_4(sc, ae, CC_ENABLE,
1086 CC_ENABLE_INIT);
1087 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1088 CTX_WAKEUP_EVENTS_INDIRECT,
1089 CTX_WAKEUP_EVENTS_INDIRECT_INIT);
1090 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1091 CTX_SIG_EVENTS_INDIRECT,
1092 CTX_SIG_EVENTS_INDIRECT_INIT);
1093 }
1094
1095 if ((sc->sc_ae_mask != 0) &&
1096 sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) {
1097 /* XXX XXX XXX init eSram only when this is boot time */
1098 }
1099
1100 if ((sc->sc_ae_mask != 0) &&
1101 sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) {
1102 /* XXX XXX XXX wait shram to complete initialization */
1103 }
1104
1105 qat_ae_reset_timestamp(sc);
1106
1107 return 0;
1108 }
1109
1110 static int
qat_ae_check(struct qat_softc * sc)1111 qat_ae_check(struct qat_softc *sc)
1112 {
1113 int error, times, ae;
1114 uint32_t cnt, pcnt, mask;
1115
1116 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
1117 if (!(mask & 1))
1118 continue;
1119
1120 times = TIMEOUT_AE_CHECK;
1121 error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
1122 if (error) {
1123 device_printf(sc->sc_dev,
1124 "couldn't access AE %d CSR\n", ae);
1125 return error;
1126 }
1127 pcnt = cnt & 0xffff;
1128
1129 while (1) {
1130 error = qat_ae_read_4(sc, ae,
1131 PROFILE_COUNT, &cnt);
1132 if (error) {
1133 device_printf(sc->sc_dev,
1134 "couldn't access AE %d CSR\n", ae);
1135 return error;
1136 }
1137 cnt &= 0xffff;
1138 if (cnt == pcnt)
1139 times--;
1140 else
1141 break;
1142 if (times <= 0) {
1143 device_printf(sc->sc_dev,
1144 "AE %d CSR is useless\n", ae);
1145 return EFAULT;
1146 }
1147 }
1148 }
1149
1150 return 0;
1151 }
1152
1153 static int
qat_ae_reset_timestamp(struct qat_softc * sc)1154 qat_ae_reset_timestamp(struct qat_softc *sc)
1155 {
1156 uint32_t misc, mask;
1157 u_char ae;
1158
1159 /* stop the timestamp timers */
1160 misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC);
1161 if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) {
1162 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
1163 misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN));
1164 }
1165
1166 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
1167 if (!(mask & 1))
1168 continue;
1169 qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0);
1170 qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0);
1171 }
1172
1173 /* start timestamp timers */
1174 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
1175 misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN);
1176
1177 return 0;
1178 }
1179
1180 static void
qat_ae_clear_xfer(struct qat_softc * sc)1181 qat_ae_clear_xfer(struct qat_softc *sc)
1182 {
1183 u_int mask, reg;
1184 u_char ae;
1185
1186 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
1187 if (!(mask & 1))
1188 continue;
1189
1190 for (reg = 0; reg < MAX_GPR_REG; reg++) {
1191 qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS,
1192 reg, 0);
1193 qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS,
1194 reg, 0);
1195 }
1196 }
1197 }
1198
1199 static int
qat_ae_clear_gprs(struct qat_softc * sc)1200 qat_ae_clear_gprs(struct qat_softc *sc)
1201 {
1202 uint32_t val;
1203 uint32_t saved_ctx = 0;
1204 int times = TIMEOUT_AE_CHECK, rv;
1205 u_char ae;
1206 u_int mask;
1207
1208 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
1209 if (!(mask & 1))
1210 continue;
1211
1212 /* turn off share control store bit */
1213 val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
1214 val &= ~AE_MISC_CONTROL_SHARE_CS;
1215 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);
1216
1217 /* turn off ucode parity */
1218 /* make sure nn_mode is set to self */
1219 qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
1220 val &= CTX_ENABLES_IGNORE_W1C_MASK;
1221 val |= CTX_ENABLES_NN_MODE;
1222 val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE;
1223 qat_ae_write_4(sc, ae, CTX_ENABLES, val);
1224
1225 /* copy instructions to ustore */
1226 qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst),
1227 ae_clear_gprs_inst);
1228
1229 /* set PC */
1230 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT,
1231 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
1232
1233 /* save current context */
1234 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx);
1235 /* change the active context */
1236 /* start the context from ctx 0 */
1237 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0);
1238
1239 /* wakeup-event voluntary */
1240 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1241 CTX_WAKEUP_EVENTS_INDIRECT,
1242 CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
1243 /* clean signals */
1244 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1245 CTX_SIG_EVENTS_INDIRECT, 0);
1246 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1247
1248 qat_ae_enable_ctx(sc, ae, AE_ALL_CTX);
1249 }
1250
1251 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
1252 if (!(mask & 1))
1253 continue;
1254 /* wait for AE to finish */
1255 do {
1256 rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1);
1257 } while (rv && times--);
1258 if (times <= 0) {
1259 device_printf(sc->sc_dev,
1260 "qat_ae_clear_gprs timeout");
1261 return ETIMEDOUT;
1262 }
1263 qat_ae_disable_ctx(sc, ae, AE_ALL_CTX);
1264 /* change the active context */
1265 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
1266 saved_ctx & ACTIVE_CTX_STATUS_ACNO);
1267 /* init the ctx_enable */
1268 qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT);
1269 /* initialize the PCs */
1270 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1271 CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
1272 /* init the ctx_arb */
1273 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT);
1274 /* enable cc */
1275 qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT);
1276 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
1277 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT);
1278 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT,
1279 CTX_SIG_EVENTS_INDIRECT_INIT);
1280 }
1281
1282 return 0;
1283 }
1284
1285 static void
qat_ae_get_shared_ustore_ae(u_char ae,u_char * nae)1286 qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae)
1287 {
1288 if (ae & 0x1)
1289 *nae = ae - 1;
1290 else
1291 *nae = ae + 1;
1292 }
1293
1294 static u_int
qat_ae_ucode_parity64(uint64_t ucode)1295 qat_ae_ucode_parity64(uint64_t ucode)
1296 {
1297
1298 ucode ^= ucode >> 1;
1299 ucode ^= ucode >> 2;
1300 ucode ^= ucode >> 4;
1301 ucode ^= ucode >> 8;
1302 ucode ^= ucode >> 16;
1303 ucode ^= ucode >> 32;
1304
1305 return ((u_int)(ucode & 1));
1306 }
1307
1308 static uint64_t
qat_ae_ucode_set_ecc(uint64_t ucode)1309 qat_ae_ucode_set_ecc(uint64_t ucode)
1310 {
1311 static const uint64_t
1312 bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL,
1313 bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL,
1314 bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL,
1315 bit6mask=0xdaf69a46910ULL;
1316
1317 /* clear the ecc bits */
1318 ucode &= ~(0x7fULL << USTORE_ECC_BIT_0);
1319
1320 ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) <<
1321 USTORE_ECC_BIT_0;
1322 ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) <<
1323 USTORE_ECC_BIT_1;
1324 ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) <<
1325 USTORE_ECC_BIT_2;
1326 ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) <<
1327 USTORE_ECC_BIT_3;
1328 ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) <<
1329 USTORE_ECC_BIT_4;
1330 ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) <<
1331 USTORE_ECC_BIT_5;
1332 ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) <<
1333 USTORE_ECC_BIT_6;
1334
1335 return (ucode);
1336 }
1337
1338 static int
qat_ae_ucode_write(struct qat_softc * sc,u_char ae,u_int uaddr,u_int ninst,const uint64_t * ucode)1339 qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
1340 const uint64_t *ucode)
1341 {
1342 uint64_t tmp;
1343 uint32_t ustore_addr, ulo, uhi;
1344 int i;
1345
1346 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
1347 uaddr |= USTORE_ADDRESS_ECS;
1348
1349 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
1350 for (i = 0; i < ninst; i++) {
1351 tmp = qat_ae_ucode_set_ecc(ucode[i]);
1352 ulo = (uint32_t)(tmp & 0xffffffff);
1353 uhi = (uint32_t)(tmp >> 32);
1354
1355 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
1356 /* this will auto increment the address */
1357 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);
1358 }
1359 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);
1360
1361 return 0;
1362 }
1363
1364 static int
qat_ae_ucode_read(struct qat_softc * sc,u_char ae,u_int uaddr,u_int ninst,uint64_t * ucode)1365 qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
1366 uint64_t *ucode)
1367 {
1368 uint32_t misc, ustore_addr, ulo, uhi;
1369 u_int ii;
1370 u_char nae;
1371
1372 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
1373 return EBUSY;
1374
1375 /* determine whether it neighbour AE runs in shared control store
1376 * status */
1377 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
1378 if (misc & AE_MISC_CONTROL_SHARE_CS) {
1379 qat_ae_get_shared_ustore_ae(ae, &nae);
1380 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
1381 return EBUSY;
1382 }
1383
1384 /* if reloadable, then get it all from dram-ustore */
1385 if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD))
1386 panic("notyet"); /* XXX getReloadUwords */
1387
1388 /* disable SHARE_CS bit to workaround silicon bug */
1389 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb);
1390
1391 MPASS(uaddr + ninst <= USTORE_SIZE);
1392
1393 /* save ustore-addr csr */
1394 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
1395
1396 uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */
1397 for (ii = 0; ii < ninst; ii++) {
1398 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
1399
1400 uaddr++;
1401 qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo);
1402 qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi);
1403 ucode[ii] = uhi;
1404 ucode[ii] = (ucode[ii] << 32) | ulo;
1405 }
1406
1407 /* restore SHARE_CS bit to workaround silicon bug */
1408 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
1409 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);
1410
1411 return 0;
1412 }
1413
1414 static u_int
qat_ae_concat_ucode(uint64_t * ucode,u_int ninst,u_int size,u_int addr,u_int * value)1415 qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr,
1416 u_int *value)
1417 {
1418 const uint64_t *inst_arr;
1419 u_int ninst0, curvalue;
1420 int ii, vali, fixup, usize = 0;
1421
1422 if (size == 0)
1423 return 0;
1424
1425 ninst0 = ninst;
1426 vali = 0;
1427 curvalue = value[vali++];
1428
1429 switch (size) {
1430 case 0x1:
1431 inst_arr = ae_inst_1b;
1432 usize = nitems(ae_inst_1b);
1433 break;
1434 case 0x2:
1435 inst_arr = ae_inst_2b;
1436 usize = nitems(ae_inst_2b);
1437 break;
1438 case 0x3:
1439 inst_arr = ae_inst_3b;
1440 usize = nitems(ae_inst_3b);
1441 break;
1442 default:
1443 inst_arr = ae_inst_4b;
1444 usize = nitems(ae_inst_4b);
1445 break;
1446 }
1447
1448 fixup = ninst;
1449 for (ii = 0; ii < usize; ii++)
1450 ucode[ninst++] = inst_arr[ii];
1451
1452 INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr));
1453 fixup++;
1454 INSERT_IMMED_GPRA_CONST(ucode[fixup], 0);
1455 fixup++;
1456 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
1457 fixup++;
1458 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
1459 /* XXX fixup++ ? */
1460
1461 if (size <= 0x4)
1462 return (ninst - ninst0);
1463
1464 size -= sizeof(u_int);
1465 while (size >= sizeof(u_int)) {
1466 curvalue = value[vali++];
1467 fixup = ninst;
1468 ucode[ninst++] = ae_inst_4b[0x2];
1469 ucode[ninst++] = ae_inst_4b[0x3];
1470 ucode[ninst++] = ae_inst_4b[0x8];
1471 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
1472 fixup++;
1473 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
1474 /* XXX fixup++ ? */
1475
1476 addr += sizeof(u_int);
1477 size -= sizeof(u_int);
1478 }
1479 /* call this function recusive when the left size less than 4 */
1480 ninst +=
1481 qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali);
1482
1483 return (ninst - ninst0);
1484 }
1485
1486 static int
qat_ae_exec_ucode(struct qat_softc * sc,u_char ae,u_char ctx,uint64_t * ucode,u_int ninst,int cond_code_off,u_int max_cycles,u_int * endpc)1487 qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx,
1488 uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles,
1489 u_int *endpc)
1490 {
1491 int error = 0, share_cs = 0;
1492 uint64_t savucode[MAX_EXEC_INST];
1493 uint32_t indr_lm_addr_0, indr_lm_addr_1;
1494 uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1;
1495 uint32_t indr_future_cnt_sig;
1496 uint32_t indr_sig, active_sig;
1497 uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl;
1498 uint32_t misc, nmisc, ctxen;
1499 u_char nae;
1500
1501 MPASS(ninst <= USTORE_SIZE);
1502
1503 if (qat_ae_is_active(sc, ae))
1504 return EBUSY;
1505
1506 /* save current LM addr */
1507 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0);
1508 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1);
1509 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
1510 &indr_lm_addr_byte_0);
1511 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
1512 &indr_lm_addr_byte_1);
1513
1514 /* backup shared control store bit, and force AE to
1515 none-shared mode before executing ucode snippet */
1516 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
1517 if (misc & AE_MISC_CONTROL_SHARE_CS) {
1518 share_cs = 1;
1519 qat_ae_get_shared_ustore_ae(ae, &nae);
1520 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
1521 return EBUSY;
1522 }
1523 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
1524 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
1525
1526 /* save current states: */
1527 if (ninst <= MAX_EXEC_INST) {
1528 error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode);
1529 if (error) {
1530 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
1531 return error;
1532 }
1533 }
1534
1535 /* save wakeup-events */
1536 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT,
1537 &wakeup_ev);
1538 /* save PC */
1539 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc);
1540 savpc &= UPC_MASK;
1541
1542 /* save ctx enables */
1543 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
1544 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
1545 /* save conditional-code */
1546 qat_ae_read_4(sc, ae, CC_ENABLE, &savcc);
1547 /* save current context */
1548 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
1549 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);
1550
1551 /* save indirect csrs */
1552 qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
1553 &indr_future_cnt_sig);
1554 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig);
1555 qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig);
1556
1557 /* turn off ucode parity */
1558 qat_ae_write_4(sc, ae, CTX_ENABLES,
1559 ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE);
1560
1561 /* copy instructions to ustore */
1562 qat_ae_ucode_write(sc, ae, 0, ninst, ucode);
1563 /* set PC */
1564 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0);
1565 /* change the active context */
1566 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
1567 ctx & ACTIVE_CTX_STATUS_ACNO);
1568
1569 if (cond_code_off) {
1570 /* disable conditional-code*/
1571 qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff);
1572 }
1573
1574 /* wakeup-event voluntary */
1575 qat_ae_ctx_indr_write(sc, ae, 1 << ctx,
1576 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
1577
1578 /* clean signals */
1579 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0);
1580 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1581
1582 /* enable context */
1583 qat_ae_enable_ctx(sc, ae, 1 << ctx);
1584
1585 /* wait for it to finish */
1586 if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0)
1587 error = ETIMEDOUT;
1588
1589 /* see if we need to get the current PC */
1590 if (endpc != NULL) {
1591 uint32_t ctx_status;
1592
1593 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
1594 &ctx_status);
1595 *endpc = ctx_status & UPC_MASK;
1596 }
1597 #if 0
1598 {
1599 uint32_t ctx_status;
1600
1601 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
1602 &ctx_status);
1603 printf("%s: endpc 0x%08x\n", __func__,
1604 ctx_status & UPC_MASK);
1605 }
1606 #endif
1607
1608 /* retore to previous states: */
1609 /* disable context */
1610 qat_ae_disable_ctx(sc, ae, 1 << ctx);
1611 if (ninst <= MAX_EXEC_INST) {
1612 /* instructions */
1613 qat_ae_ucode_write(sc, ae, 0, ninst, savucode);
1614 }
1615 /* wakeup-events */
1616 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT,
1617 wakeup_ev);
1618 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc);
1619
1620 /* only restore shared control store bit,
1621 other bit might be changed by AE code snippet */
1622 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
1623 if (share_cs)
1624 nmisc = misc | AE_MISC_CONTROL_SHARE_CS;
1625 else
1626 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
1627 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
1628 /* conditional-code */
1629 qat_ae_write_4(sc, ae, CC_ENABLE, savcc);
1630 /* change the active context */
1631 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
1632 savctx & ACTIVE_CTX_STATUS_ACNO);
1633 /* restore the nxt ctx to run */
1634 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
1635 /* restore current LM addr */
1636 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT,
1637 indr_lm_addr_0);
1638 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT,
1639 indr_lm_addr_1);
1640 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
1641 indr_lm_addr_byte_0);
1642 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
1643 indr_lm_addr_byte_1);
1644
1645 /* restore indirect csrs */
1646 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
1647 indr_future_cnt_sig);
1648 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT,
1649 indr_sig);
1650 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig);
1651
1652 /* ctx-enables */
1653 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
1654
1655 return error;
1656 }
1657
1658 static int
qat_ae_exec_ucode_init_lm(struct qat_softc * sc,u_char ae,u_char ctx,int * first_exec,uint64_t * ucode,u_int ninst,u_int * gpr_a0,u_int * gpr_a1,u_int * gpr_a2,u_int * gpr_b0,u_int * gpr_b1)1659 qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx,
1660 int *first_exec, uint64_t *ucode, u_int ninst,
1661 u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1)
1662 {
1663
1664 if (*first_exec) {
1665 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
1666 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
1667 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
1668 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
1669 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
1670 *first_exec = 0;
1671 }
1672
1673 return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL);
1674 }
1675
1676 static int
qat_ae_restore_init_lm_gprs(struct qat_softc * sc,u_char ae,u_char ctx,u_int gpr_a0,u_int gpr_a1,u_int gpr_a2,u_int gpr_b0,u_int gpr_b1)1677 qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx,
1678 u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1)
1679 {
1680 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
1681 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
1682 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
1683 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
1684 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
1685
1686 return 0;
1687 }
1688
1689 static int
qat_ae_get_inst_num(int lmsize)1690 qat_ae_get_inst_num(int lmsize)
1691 {
1692 int ninst, left;
1693
1694 if (lmsize == 0)
1695 return 0;
1696
1697 left = lmsize % sizeof(u_int);
1698
1699 if (left) {
1700 ninst = nitems(ae_inst_1b) +
1701 qat_ae_get_inst_num(lmsize - left);
1702 } else {
1703 /* 3 instruction is needed for further code */
1704 ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b);
1705 }
1706
1707 return (ninst);
1708 }
1709
1710 static int
qat_ae_batch_put_lm(struct qat_softc * sc,u_char ae,struct qat_ae_batch_init_list * qabi_list,size_t nqabi)1711 qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae,
1712 struct qat_ae_batch_init_list *qabi_list, size_t nqabi)
1713 {
1714 struct qat_ae_batch_init *qabi;
1715 size_t alloc_ninst, ninst;
1716 uint64_t *ucode;
1717 u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1;
1718 int insnsz, error = 0, execed = 0, first_exec = 1;
1719
1720 if (STAILQ_FIRST(qabi_list) == NULL)
1721 return 0;
1722
1723 alloc_ninst = min(USTORE_SIZE, nqabi);
1724 ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst);
1725
1726 ninst = 0;
1727 STAILQ_FOREACH(qabi, qabi_list, qabi_next) {
1728 insnsz = qat_ae_get_inst_num(qabi->qabi_size);
1729 if (insnsz + ninst > alloc_ninst) {
1730 /* add ctx_arb[kill] */
1731 ucode[ninst++] = 0x0E000010000ull;
1732 execed = 1;
1733
1734 error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
1735 &first_exec, ucode, ninst,
1736 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
1737 if (error) {
1738 qat_ae_restore_init_lm_gprs(sc, ae, 0,
1739 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
1740 qat_free_mem(ucode);
1741 return error;
1742 }
1743 /* run microExec to execute the microcode */
1744 ninst = 0;
1745 }
1746 ninst += qat_ae_concat_ucode(ucode, ninst,
1747 qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value);
1748 }
1749
1750 if (ninst > 0) {
1751 ucode[ninst++] = 0x0E000010000ull;
1752 execed = 1;
1753
1754 error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
1755 &first_exec, ucode, ninst,
1756 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
1757 }
1758 if (execed) {
1759 qat_ae_restore_init_lm_gprs(sc, ae, 0,
1760 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
1761 }
1762
1763 qat_free_mem(ucode);
1764
1765 return error;
1766 }
1767
1768 static int
qat_ae_write_pc(struct qat_softc * sc,u_char ae,u_int ctx_mask,u_int upc)1769 qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc)
1770 {
1771
1772 if (qat_ae_is_active(sc, ae))
1773 return EBUSY;
1774
1775 qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT,
1776 UPC_MASK & upc);
1777 return 0;
1778 }
1779
1780 static inline u_int
qat_aefw_csum_calc(u_int reg,int ch)1781 qat_aefw_csum_calc(u_int reg, int ch)
1782 {
1783 int i;
1784 u_int topbit = CRC_BITMASK(CRC_WIDTH - 1);
1785 u_int inbyte = (u_int)((reg >> 0x18) ^ ch);
1786
1787 reg ^= inbyte << (CRC_WIDTH - 0x8);
1788 for (i = 0; i < 0x8; i++) {
1789 if (reg & topbit)
1790 reg = (reg << 1) ^ CRC_POLY;
1791 else
1792 reg <<= 1;
1793 }
1794
1795 return (reg & CRC_WIDTHMASK(CRC_WIDTH));
1796 }
1797
1798 static u_int
qat_aefw_csum(char * buf,int size)1799 qat_aefw_csum(char *buf, int size)
1800 {
1801 u_int csum = 0;
1802
1803 while (size--) {
1804 csum = qat_aefw_csum_calc(csum, *buf++);
1805 }
1806
1807 return csum;
1808 }
1809
1810 static const char *
qat_aefw_uof_string(struct qat_softc * sc,size_t offset)1811 qat_aefw_uof_string(struct qat_softc *sc, size_t offset)
1812 {
1813 if (offset >= sc->sc_aefw_uof.qafu_str_tab_size)
1814 return NULL;
1815 if (sc->sc_aefw_uof.qafu_str_tab == NULL)
1816 return NULL;
1817
1818 return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset);
1819 }
1820
1821 static struct uof_chunk_hdr *
qat_aefw_uof_find_chunk(struct qat_softc * sc,const char * id,struct uof_chunk_hdr * cur)1822 qat_aefw_uof_find_chunk(struct qat_softc *sc,
1823 const char *id, struct uof_chunk_hdr *cur)
1824 {
1825 struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr;
1826 struct uof_chunk_hdr *uch;
1827 int i;
1828
1829 uch = (struct uof_chunk_hdr *)(uoh + 1);
1830 for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) {
1831 if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size)
1832 return NULL;
1833
1834 if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN))
1835 return uch;
1836 }
1837
1838 return NULL;
1839 }
1840
1841 static int
qat_aefw_load_mof(struct qat_softc * sc)1842 qat_aefw_load_mof(struct qat_softc *sc)
1843 {
1844 const struct firmware *fw;
1845
1846 fw = firmware_get(sc->sc_hw.qhw_mof_fwname);
1847 if (fw == NULL) {
1848 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
1849 sc->sc_hw.qhw_mof_fwname);
1850 return ENXIO;
1851 }
1852
1853 sc->sc_fw_mof = qat_alloc_mem(fw->datasize);
1854 sc->sc_fw_mof_size = fw->datasize;
1855 memcpy(sc->sc_fw_mof, fw->data, fw->datasize);
1856 firmware_put(fw, FIRMWARE_UNLOAD);
1857 return 0;
1858 }
1859
1860 static void
qat_aefw_unload_mof(struct qat_softc * sc)1861 qat_aefw_unload_mof(struct qat_softc *sc)
1862 {
1863 if (sc->sc_fw_mof != NULL) {
1864 qat_free_mem(sc->sc_fw_mof);
1865 sc->sc_fw_mof = NULL;
1866 }
1867 }
1868
1869 static int
qat_aefw_load_mmp(struct qat_softc * sc)1870 qat_aefw_load_mmp(struct qat_softc *sc)
1871 {
1872 const struct firmware *fw;
1873
1874 fw = firmware_get(sc->sc_hw.qhw_mmp_fwname);
1875 if (fw == NULL) {
1876 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
1877 sc->sc_hw.qhw_mmp_fwname);
1878 return ENXIO;
1879 }
1880
1881 sc->sc_fw_mmp = qat_alloc_mem(fw->datasize);
1882 sc->sc_fw_mmp_size = fw->datasize;
1883 memcpy(sc->sc_fw_mmp, fw->data, fw->datasize);
1884 firmware_put(fw, FIRMWARE_UNLOAD);
1885 return 0;
1886 }
1887
1888 static void
qat_aefw_unload_mmp(struct qat_softc * sc)1889 qat_aefw_unload_mmp(struct qat_softc *sc)
1890 {
1891 if (sc->sc_fw_mmp != NULL) {
1892 qat_free_mem(sc->sc_fw_mmp);
1893 sc->sc_fw_mmp = NULL;
1894 }
1895 }
1896
1897 static int
qat_aefw_mof_find_uof0(struct qat_softc * sc,struct mof_uof_hdr * muh,struct mof_uof_chunk_hdr * head,u_int nchunk,size_t size,const char * id,size_t * fwsize,void ** fwptr)1898 qat_aefw_mof_find_uof0(struct qat_softc *sc,
1899 struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head,
1900 u_int nchunk, size_t size, const char *id,
1901 size_t *fwsize, void **fwptr)
1902 {
1903 int i;
1904 char *uof_name;
1905
1906 for (i = 0; i < nchunk; i++) {
1907 struct mof_uof_chunk_hdr *much = &head[i];
1908
1909 if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN))
1910 return EINVAL;
1911
1912 if (much->much_offset + much->much_size > size)
1913 return EINVAL;
1914
1915 if (sc->sc_mof.qmf_sym_size <= much->much_name)
1916 return EINVAL;
1917
1918 uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym +
1919 much->much_name);
1920
1921 if (!strcmp(uof_name, sc->sc_fw_uof_name)) {
1922 *fwptr = (void *)((uintptr_t)muh +
1923 (uintptr_t)much->much_offset);
1924 *fwsize = (size_t)much->much_size;
1925 return 0;
1926 }
1927 }
1928
1929 return ENOENT;
1930 }
1931
1932 static int
qat_aefw_mof_find_uof(struct qat_softc * sc)1933 qat_aefw_mof_find_uof(struct qat_softc *sc)
1934 {
1935 struct mof_uof_hdr *uof_hdr, *suof_hdr;
1936 u_int nuof_chunks = 0, nsuof_chunks = 0;
1937 int error;
1938
1939 uof_hdr = sc->sc_mof.qmf_uof_objs;
1940 suof_hdr = sc->sc_mof.qmf_suof_objs;
1941
1942 if (uof_hdr != NULL) {
1943 if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) {
1944 return EINVAL;
1945 }
1946 nuof_chunks = uof_hdr->muh_num_chunks;
1947 }
1948 if (suof_hdr != NULL) {
1949 if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks)
1950 return EINVAL;
1951 nsuof_chunks = suof_hdr->muh_num_chunks;
1952 }
1953
1954 if (nuof_chunks + nsuof_chunks == 0)
1955 return EINVAL;
1956
1957 if (uof_hdr != NULL) {
1958 error = qat_aefw_mof_find_uof0(sc, uof_hdr,
1959 (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks,
1960 sc->sc_mof.qmf_uof_objs_size, UOF_IMAG,
1961 &sc->sc_fw_uof_size, &sc->sc_fw_uof);
1962 if (error && error != ENOENT)
1963 return error;
1964 }
1965
1966 if (suof_hdr != NULL) {
1967 error = qat_aefw_mof_find_uof0(sc, suof_hdr,
1968 (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks,
1969 sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG,
1970 &sc->sc_fw_suof_size, &sc->sc_fw_suof);
1971 if (error && error != ENOENT)
1972 return error;
1973 }
1974
1975 if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL)
1976 return ENOENT;
1977
1978 return 0;
1979 }
1980
1981 static int
qat_aefw_mof_parse(struct qat_softc * sc)1982 qat_aefw_mof_parse(struct qat_softc *sc)
1983 {
1984 const struct mof_file_hdr *mfh;
1985 const struct mof_file_chunk_hdr *mfch;
1986 size_t size;
1987 u_int csum;
1988 int error, i;
1989
1990 size = sc->sc_fw_mof_size;
1991
1992 if (size < sizeof(struct mof_file_hdr))
1993 return EINVAL;
1994 size -= sizeof(struct mof_file_hdr);
1995
1996 mfh = sc->sc_fw_mof;
1997
1998 if (mfh->mfh_fid != MOF_FID)
1999 return EINVAL;
2000
2001 csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof +
2002 offsetof(struct mof_file_hdr, mfh_min_ver)),
2003 sc->sc_fw_mof_size -
2004 offsetof(struct mof_file_hdr, mfh_min_ver));
2005 if (mfh->mfh_csum != csum)
2006 return EINVAL;
2007
2008 if (mfh->mfh_min_ver != MOF_MIN_VER ||
2009 mfh->mfh_maj_ver != MOF_MAJ_VER)
2010 return EINVAL;
2011
2012 if (mfh->mfh_max_chunks < mfh->mfh_num_chunks)
2013 return EINVAL;
2014
2015 if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks)
2016 return EINVAL;
2017 mfch = (const struct mof_file_chunk_hdr *)(mfh + 1);
2018
2019 for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) {
2020 if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size)
2021 return EINVAL;
2022
2023 if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) {
2024 if (sc->sc_mof.qmf_sym != NULL)
2025 return EINVAL;
2026
2027 sc->sc_mof.qmf_sym =
2028 (void *)((uintptr_t)sc->sc_fw_mof +
2029 (uintptr_t)mfch->mfch_offset + sizeof(u_int));
2030 sc->sc_mof.qmf_sym_size =
2031 *(u_int *)((uintptr_t)sc->sc_fw_mof +
2032 (uintptr_t)mfch->mfch_offset);
2033
2034 if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0)
2035 return EINVAL;
2036 if (mfch->mfch_size != sc->sc_mof.qmf_sym_size +
2037 sizeof(u_int) || mfch->mfch_size == 0)
2038 return EINVAL;
2039 if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym +
2040 sc->sc_mof.qmf_sym_size - 1) != '\0')
2041 return EINVAL;
2042
2043 } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) {
2044 if (sc->sc_mof.qmf_uof_objs != NULL)
2045 return EINVAL;
2046
2047 sc->sc_mof.qmf_uof_objs =
2048 (void *)((uintptr_t)sc->sc_fw_mof +
2049 (uintptr_t)mfch->mfch_offset);
2050 sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size;
2051
2052 } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) {
2053 if (sc->sc_mof.qmf_suof_objs != NULL)
2054 return EINVAL;
2055
2056 sc->sc_mof.qmf_suof_objs =
2057 (void *)((uintptr_t)sc->sc_fw_mof +
2058 (uintptr_t)mfch->mfch_offset);
2059 sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size;
2060 }
2061 }
2062
2063 if (sc->sc_mof.qmf_sym == NULL ||
2064 (sc->sc_mof.qmf_uof_objs == NULL &&
2065 sc->sc_mof.qmf_suof_objs == NULL))
2066 return EINVAL;
2067
2068 error = qat_aefw_mof_find_uof(sc);
2069 if (error)
2070 return error;
2071 return 0;
2072 }
2073
2074 static int
qat_aefw_uof_parse_image(struct qat_softc * sc,struct qat_uof_image * qui,struct uof_chunk_hdr * uch)2075 qat_aefw_uof_parse_image(struct qat_softc *sc,
2076 struct qat_uof_image *qui, struct uof_chunk_hdr *uch)
2077 {
2078 struct uof_image *image;
2079 struct uof_code_page *page;
2080 uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
2081 size_t lim = uch->uch_offset + uch->uch_size, size;
2082 int i, p;
2083
2084 size = uch->uch_size;
2085 if (size < sizeof(struct uof_image))
2086 return EINVAL;
2087 size -= sizeof(struct uof_image);
2088
2089 qui->qui_image = image =
2090 (struct uof_image *)(base + uch->uch_offset);
2091
2092 #define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \
2093 do { \
2094 u_int nent; \
2095 nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\
2096 if ((lim) < off + sizeof(struct uof_obj_table) + \
2097 sizeof(type) * nent) \
2098 return EINVAL; \
2099 *(np) = nent; \
2100 if (nent > 0) \
2101 *(typep) = (type)((struct uof_obj_table *) \
2102 ((base) + (off)) + 1); \
2103 else \
2104 *(typep) = NULL; \
2105 } while (0)
2106
2107 ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg,
2108 struct uof_ae_reg *, base, image->ui_reg_tab, lim);
2109 ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym,
2110 struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim);
2111 ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak,
2112 struct qui_sbreak *, base, image->ui_sbreak_tab, lim);
2113
2114 if (size < sizeof(struct uof_code_page) * image->ui_num_pages)
2115 return EINVAL;
2116 if (nitems(qui->qui_pages) < image->ui_num_pages)
2117 return EINVAL;
2118
2119 page = (struct uof_code_page *)(image + 1);
2120
2121 for (p = 0; p < image->ui_num_pages; p++, page++) {
2122 struct qat_uof_page *qup = &qui->qui_pages[p];
2123 struct uof_code_area *uca;
2124
2125 qup->qup_page_num = page->ucp_page_num;
2126 qup->qup_def_page = page->ucp_def_page;
2127 qup->qup_page_region = page->ucp_page_region;
2128 qup->qup_beg_vaddr = page->ucp_beg_vaddr;
2129 qup->qup_beg_paddr = page->ucp_beg_paddr;
2130
2131 ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var,
2132 struct uof_uword_fixup *, base,
2133 page->ucp_uc_var_tab, lim);
2134 ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var,
2135 struct uof_import_var *, base,
2136 page->ucp_imp_var_tab, lim);
2137 ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr,
2138 struct uof_uword_fixup *, base,
2139 page->ucp_imp_expr_tab, lim);
2140 ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg,
2141 struct uof_uword_fixup *, base,
2142 page->ucp_neigh_reg_tab, lim);
2143
2144 if (lim < page->ucp_code_area + sizeof(struct uof_code_area))
2145 return EINVAL;
2146
2147 uca = (struct uof_code_area *)(base + page->ucp_code_area);
2148 qup->qup_num_micro_words = uca->uca_num_micro_words;
2149
2150 ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks,
2151 struct qat_uof_uword_block *, base,
2152 uca->uca_uword_block_tab, lim);
2153
2154 for (i = 0; i < qup->qup_num_uw_blocks; i++) {
2155 u_int uwordoff = ((struct uof_uword_block *)(
2156 &qup->qup_uw_blocks[i]))->uub_uword_offset;
2157
2158 if (lim < uwordoff)
2159 return EINVAL;
2160
2161 qup->qup_uw_blocks[i].quub_micro_words =
2162 (base + uwordoff);
2163 }
2164 }
2165
2166 #undef ASSIGN_OBJ_TAB
2167
2168 return 0;
2169 }
2170
2171 static int
qat_aefw_uof_parse_images(struct qat_softc * sc)2172 qat_aefw_uof_parse_images(struct qat_softc *sc)
2173 {
2174 struct uof_chunk_hdr *uch = NULL;
2175 int i, error;
2176
2177 for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) {
2178 uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch);
2179 if (uch == NULL)
2180 break;
2181
2182 if (i >= nitems(sc->sc_aefw_uof.qafu_imgs))
2183 return ENOENT;
2184
2185 error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch);
2186 if (error)
2187 return error;
2188
2189 sc->sc_aefw_uof.qafu_num_imgs++;
2190 }
2191
2192 return 0;
2193 }
2194
2195 static int
qat_aefw_uof_parse(struct qat_softc * sc)2196 qat_aefw_uof_parse(struct qat_softc *sc)
2197 {
2198 struct uof_file_hdr *ufh;
2199 struct uof_file_chunk_hdr *ufch;
2200 struct uof_obj_hdr *uoh;
2201 struct uof_chunk_hdr *uch;
2202 void *uof = NULL;
2203 size_t size, uof_size, hdr_size;
2204 uintptr_t base;
2205 u_int csum;
2206 int i;
2207
2208 size = sc->sc_fw_uof_size;
2209 if (size < MIN_UOF_SIZE)
2210 return EINVAL;
2211 size -= sizeof(struct uof_file_hdr);
2212
2213 ufh = sc->sc_fw_uof;
2214
2215 if (ufh->ufh_id != UOF_FID)
2216 return EINVAL;
2217 if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER)
2218 return EINVAL;
2219
2220 if (ufh->ufh_max_chunks < ufh->ufh_num_chunks)
2221 return EINVAL;
2222 if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks)
2223 return EINVAL;
2224 ufch = (struct uof_file_chunk_hdr *)(ufh + 1);
2225
2226 uof_size = 0;
2227 for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) {
2228 if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size)
2229 return EINVAL;
2230
2231 if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) {
2232 if (uof != NULL)
2233 return EINVAL;
2234
2235 uof =
2236 (void *)((uintptr_t)sc->sc_fw_uof +
2237 ufch->ufch_offset);
2238 uof_size = ufch->ufch_size;
2239
2240 csum = qat_aefw_csum(uof, uof_size);
2241 if (csum != ufch->ufch_csum)
2242 return EINVAL;
2243 }
2244 }
2245
2246 if (uof == NULL)
2247 return ENOENT;
2248
2249 size = uof_size;
2250 if (size < sizeof(struct uof_obj_hdr))
2251 return EINVAL;
2252 size -= sizeof(struct uof_obj_hdr);
2253
2254 uoh = uof;
2255
2256 if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks)
2257 return EINVAL;
2258
2259 /* Check if the UOF objects are compatible with the chip */
2260 if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0)
2261 return ENOTSUP;
2262
2263 if (uoh->uoh_min_cpu_ver > sc->sc_rev ||
2264 uoh->uoh_max_cpu_ver < sc->sc_rev)
2265 return ENOTSUP;
2266
2267 sc->sc_aefw_uof.qafu_size = uof_size;
2268 sc->sc_aefw_uof.qafu_obj_hdr = uoh;
2269
2270 base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
2271
2272 /* map uof string-table */
2273 uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL);
2274 if (uch != NULL) {
2275 hdr_size = offsetof(struct uof_str_tab, ust_strings);
2276 sc->sc_aefw_uof.qafu_str_tab =
2277 (void *)(base + uch->uch_offset + hdr_size);
2278 sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size;
2279 }
2280
2281 /* get ustore mem inits table -- should be only one */
2282 uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL);
2283 if (uch != NULL) {
2284 if (uch->uch_size < sizeof(struct uof_obj_table))
2285 return EINVAL;
2286 sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base +
2287 uch->uch_offset))->uot_nentries;
2288 if (sc->sc_aefw_uof.qafu_num_init_mem) {
2289 sc->sc_aefw_uof.qafu_init_mem =
2290 (struct uof_init_mem *)(base + uch->uch_offset +
2291 sizeof(struct uof_obj_table));
2292 sc->sc_aefw_uof.qafu_init_mem_size =
2293 uch->uch_size - sizeof(struct uof_obj_table);
2294 }
2295 }
2296
2297 uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL);
2298 if (uch != NULL) {
2299 if (uch->uch_size < sizeof(struct uof_obj_table) +
2300 sizeof(struct uof_var_mem_seg))
2301 return EINVAL;
2302 sc->sc_aefw_uof.qafu_var_mem_seg =
2303 (struct uof_var_mem_seg *)(base + uch->uch_offset +
2304 sizeof(struct uof_obj_table));
2305 }
2306
2307 return qat_aefw_uof_parse_images(sc);
2308 }
2309
2310 static int
qat_aefw_suof_parse_image(struct qat_softc * sc,struct qat_suof_image * qsi,struct suof_chunk_hdr * sch)2311 qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi,
2312 struct suof_chunk_hdr *sch)
2313 {
2314 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
2315 struct simg_ae_mode *ae_mode;
2316 u_int maj_ver;
2317
2318 qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset +
2319 sizeof(struct suof_obj_hdr);
2320 qsi->qsi_simg_len =
2321 ((struct suof_obj_hdr *)
2322 (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length;
2323
2324 qsi->qsi_css_header = qsi->qsi_simg_buf;
2325 qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr);
2326 qsi->qsi_css_signature = qsi->qsi_css_key +
2327 CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
2328 qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN;
2329
2330 ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg;
2331 qsi->qsi_ae_mask = ae_mode->sam_ae_mask;
2332 qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name;
2333 qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data;
2334 qsi->qsi_fw_type = ae_mode->sam_fw_type;
2335
2336 if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type)
2337 return EINVAL;
2338
2339 maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff;
2340 if ((maj_ver > ae_mode->sam_devmax_ver) ||
2341 (maj_ver < ae_mode->sam_devmin_ver)) {
2342 return EINVAL;
2343 }
2344
2345 return 0;
2346 }
2347
2348 static int
qat_aefw_suof_parse(struct qat_softc * sc)2349 qat_aefw_suof_parse(struct qat_softc *sc)
2350 {
2351 struct suof_file_hdr *sfh;
2352 struct suof_chunk_hdr *sch;
2353 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
2354 struct qat_suof_image *qsi;
2355 size_t size;
2356 u_int csum;
2357 int ae0_img = MAX_AE;
2358 int i, error;
2359
2360 size = sc->sc_fw_suof_size;
2361 if (size < sizeof(struct suof_file_hdr))
2362 return EINVAL;
2363
2364 sfh = sc->sc_fw_suof;
2365
2366 if (sfh->sfh_file_id != SUOF_FID)
2367 return EINVAL;
2368 if (sfh->sfh_fw_type != 0)
2369 return EINVAL;
2370 if (sfh->sfh_num_chunks <= 1)
2371 return EINVAL;
2372 if (sfh->sfh_min_ver != SUOF_MIN_VER ||
2373 sfh->sfh_maj_ver != SUOF_MAJ_VER)
2374 return EINVAL;
2375
2376 csum = qat_aefw_csum((char *)&sfh->sfh_min_ver,
2377 size - offsetof(struct suof_file_hdr, sfh_min_ver));
2378 if (csum != sfh->sfh_check_sum)
2379 return EINVAL;
2380
2381 size -= sizeof(struct suof_file_hdr);
2382
2383 qafs->qafs_file_id = SUOF_FID;
2384 qafs->qafs_suof_buf = sc->sc_fw_suof;
2385 qafs->qafs_suof_size = sc->sc_fw_suof_size;
2386 qafs->qafs_check_sum = sfh->sfh_check_sum;
2387 qafs->qafs_min_ver = sfh->sfh_min_ver;
2388 qafs->qafs_maj_ver = sfh->sfh_maj_ver;
2389 qafs->qafs_fw_type = sfh->sfh_fw_type;
2390
2391 if (size < sizeof(struct suof_chunk_hdr))
2392 return EINVAL;
2393 sch = (struct suof_chunk_hdr *)(sfh + 1);
2394 size -= sizeof(struct suof_chunk_hdr);
2395
2396 if (size < sizeof(struct suof_str_tab))
2397 return EINVAL;
2398 size -= offsetof(struct suof_str_tab, sst_strings);
2399
2400 qafs->qafs_sym_size = ((struct suof_str_tab *)
2401 (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length;
2402 if (size < qafs->qafs_sym_size)
2403 return EINVAL;
2404 qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset +
2405 offsetof(struct suof_str_tab, sst_strings);
2406
2407 qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1;
2408 if (qafs->qafs_num_simgs == 0)
2409 return EINVAL;
2410
2411 qsi = qat_alloc_mem(
2412 sizeof(struct qat_suof_image) * qafs->qafs_num_simgs);
2413 qafs->qafs_simg = qsi;
2414
2415 for (i = 0; i < qafs->qafs_num_simgs; i++) {
2416 error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]);
2417 if (error)
2418 return error;
2419 if ((qsi[i].qsi_ae_mask & 0x1) != 0)
2420 ae0_img = i;
2421 }
2422
2423 if (ae0_img != qafs->qafs_num_simgs - 1) {
2424 struct qat_suof_image last_qsi;
2425
2426 memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1],
2427 sizeof(struct qat_suof_image));
2428 memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img],
2429 sizeof(struct qat_suof_image));
2430 memcpy(&qsi[ae0_img], &last_qsi,
2431 sizeof(struct qat_suof_image));
2432 }
2433
2434 return 0;
2435 }
2436
2437 static int
qat_aefw_alloc_auth_dmamem(struct qat_softc * sc,char * image,size_t size,struct qat_dmamem * dma)2438 qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size,
2439 struct qat_dmamem *dma)
2440 {
2441 struct css_hdr *css = (struct css_hdr *)image;
2442 struct auth_chunk *auth_chunk;
2443 struct fw_auth_desc *auth_desc;
2444 size_t mapsize, simg_offset = sizeof(struct auth_chunk);
2445 bus_size_t bus_addr;
2446 uintptr_t virt_addr;
2447 int error;
2448
2449 if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN)
2450 return EINVAL;
2451
2452 mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ?
2453 CSS_AE_SIMG_LEN + simg_offset :
2454 size + CSS_FWSK_PAD_LEN + simg_offset;
2455 error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE);
2456 if (error)
2457 return error;
2458
2459 memset(dma->qdm_dma_vaddr, 0, mapsize);
2460
2461 auth_chunk = dma->qdm_dma_vaddr;
2462 auth_chunk->ac_chunk_size = mapsize;
2463 auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr;
2464
2465 virt_addr = (uintptr_t)dma->qdm_dma_vaddr;
2466 virt_addr += simg_offset;
2467 bus_addr = auth_chunk->ac_chunk_bus_addr;
2468 bus_addr += simg_offset;
2469
2470 auth_desc = &auth_chunk->ac_fw_auth_desc;
2471 auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32;
2472 auth_desc->fad_css_hdr_low = bus_addr;
2473
2474 memcpy((void *)virt_addr, image, sizeof(struct css_hdr));
2475 /* pub key */
2476 virt_addr += sizeof(struct css_hdr);
2477 bus_addr += sizeof(struct css_hdr);
2478 image += sizeof(struct css_hdr);
2479
2480 auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32;
2481 auth_desc->fad_fwsk_pub_low = bus_addr;
2482
2483 memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN);
2484 memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN);
2485 memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN),
2486 image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t));
2487
2488 virt_addr += CSS_FWSK_PUB_LEN;
2489 bus_addr += CSS_FWSK_PUB_LEN;
2490 image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
2491
2492 auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32;
2493 auth_desc->fad_signature_low = bus_addr;
2494
2495 memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN);
2496
2497 virt_addr += CSS_SIGNATURE_LEN;
2498 bus_addr += CSS_SIGNATURE_LEN;
2499 image += CSS_SIGNATURE_LEN;
2500
2501 auth_desc->fad_img_high = (uint64_t)bus_addr >> 32;
2502 auth_desc->fad_img_low = bus_addr;
2503 auth_desc->fad_img_len = size - AE_IMG_OFFSET;
2504
2505 memcpy((void *)virt_addr, image, auth_desc->fad_img_len);
2506
2507 if (css->css_fw_type == CSS_AE_FIRMWARE) {
2508 auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high;
2509 auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low;
2510
2511 bus_addr += sizeof(struct simg_ae_mode);
2512
2513 auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32;
2514 auth_desc->fad_img_ae_init_data_low = bus_addr;
2515
2516 bus_addr += SIMG_AE_INIT_SEQ_LEN;
2517
2518 auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32;
2519 auth_desc->fad_img_ae_insts_low = bus_addr;
2520 } else {
2521 auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high;
2522 auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low;
2523 }
2524
2525 bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map,
2526 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2527
2528 return 0;
2529 }
2530
2531 static int
qat_aefw_auth(struct qat_softc * sc,struct qat_dmamem * dma)2532 qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma)
2533 {
2534 bus_addr_t addr;
2535 uint32_t fcu, sts;
2536 int retry = 0;
2537
2538 addr = dma->qdm_dma_seg.ds_addr;
2539 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32);
2540 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr);
2541 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH);
2542
2543 do {
2544 DELAY(FW_AUTH_WAIT_PERIOD * 1000);
2545 fcu = qat_cap_global_read_4(sc, FCU_STATUS);
2546 sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
2547 if (sts == FCU_STATUS_STS_VERI_FAIL)
2548 goto fail;
2549 if (fcu & FCU_STATUS_AUTHFWLD &&
2550 sts == FCU_STATUS_STS_VERI_DONE) {
2551 return 0;
2552 }
2553 } while (retry++ < FW_AUTH_MAX_RETRY);
2554
2555 fail:
2556 device_printf(sc->sc_dev,
2557 "firmware authentication error: status 0x%08x retry %d\n",
2558 fcu, retry);
2559 return EINVAL;
2560 }
2561
2562 static int
qat_aefw_suof_load(struct qat_softc * sc,struct qat_dmamem * dma)2563 qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma)
2564 {
2565 struct simg_ae_mode *ae_mode;
2566 uint32_t fcu, sts, loaded;
2567 u_int mask;
2568 u_char ae;
2569 int retry = 0;
2570
2571 ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr +
2572 sizeof(struct auth_chunk) + sizeof(struct css_hdr) +
2573 CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN);
2574
2575 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
2576 if (!(mask & 1))
2577 continue;
2578 if (!((ae_mode->sam_ae_mask >> ae) & 0x1))
2579 continue;
2580 if (qat_ae_is_active(sc, ae)) {
2581 device_printf(sc->sc_dev, "AE %d is active\n", ae);
2582 return EINVAL;
2583 }
2584 qat_cap_global_write_4(sc, FCU_CTRL,
2585 FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE));
2586 do {
2587 DELAY(FW_AUTH_WAIT_PERIOD * 1000);
2588 fcu = qat_cap_global_read_4(sc, FCU_STATUS);
2589 sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
2590 loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE);
2591 if (sts == FCU_STATUS_STS_LOAD_DONE &&
2592 (loaded & (1 << ae))) {
2593 break;
2594 }
2595 } while (retry++ < FW_AUTH_MAX_RETRY);
2596
2597 if (retry > FW_AUTH_MAX_RETRY) {
2598 device_printf(sc->sc_dev,
2599 "firmware load timeout: status %08x\n", fcu);
2600 return EINVAL;
2601 }
2602 }
2603
2604 return 0;
2605 }
2606
2607 static int
qat_aefw_suof_write(struct qat_softc * sc)2608 qat_aefw_suof_write(struct qat_softc *sc)
2609 {
2610 struct qat_suof_image *qsi;
2611 int i, error = 0;
2612
2613 for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) {
2614 qsi = &sc->sc_aefw_suof.qafs_simg[i];
2615 error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf,
2616 qsi->qsi_simg_len, &qsi->qsi_dma);
2617 if (error)
2618 return error;
2619 error = qat_aefw_auth(sc, &qsi->qsi_dma);
2620 if (error) {
2621 qat_free_dmamem(sc, &qsi->qsi_dma);
2622 return error;
2623 }
2624 error = qat_aefw_suof_load(sc, &qsi->qsi_dma);
2625 if (error) {
2626 qat_free_dmamem(sc, &qsi->qsi_dma);
2627 return error;
2628 }
2629 qat_free_dmamem(sc, &qsi->qsi_dma);
2630 }
2631 qat_free_mem(sc->sc_aefw_suof.qafs_simg);
2632
2633 return 0;
2634 }
2635
2636 static int
qat_aefw_uof_assign_image(struct qat_softc * sc,struct qat_ae * qae,struct qat_uof_image * qui)2637 qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae,
2638 struct qat_uof_image *qui)
2639 {
2640 struct qat_ae_slice *slice;
2641 int i, npages, nregions;
2642
2643 if (qae->qae_num_slices >= nitems(qae->qae_slices))
2644 return ENOENT;
2645
2646 if (qui->qui_image->ui_ae_mode &
2647 (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) {
2648 /* XXX */
2649 device_printf(sc->sc_dev,
2650 "shared ae mode is not supported yet\n");
2651 return ENOTSUP;
2652 }
2653
2654 qae->qae_shareable_ustore = 0; /* XXX */
2655 qae->qae_effect_ustore_size = USTORE_SIZE;
2656
2657 slice = &qae->qae_slices[qae->qae_num_slices];
2658
2659 slice->qas_image = qui;
2660 slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned;
2661
2662 nregions = qui->qui_image->ui_num_page_regions;
2663 npages = qui->qui_image->ui_num_pages;
2664
2665 if (nregions > nitems(slice->qas_regions))
2666 return ENOENT;
2667 if (npages > nitems(slice->qas_pages))
2668 return ENOENT;
2669
2670 for (i = 0; i < nregions; i++) {
2671 STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages);
2672 }
2673 for (i = 0; i < npages; i++) {
2674 struct qat_ae_page *page = &slice->qas_pages[i];
2675 int region;
2676
2677 page->qap_page = &qui->qui_pages[i];
2678 region = page->qap_page->qup_page_region;
2679 if (region >= nregions)
2680 return EINVAL;
2681
2682 page->qap_region = &slice->qas_regions[region];
2683 }
2684
2685 qae->qae_num_slices++;
2686
2687 return 0;
2688 }
2689
2690 static int
qat_aefw_uof_init_ae(struct qat_softc * sc,u_char ae)2691 qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae)
2692 {
2693 struct uof_image *image;
2694 struct qat_ae *qae = &(QAT_AE(sc, ae));
2695 int s;
2696 u_char nn_mode;
2697
2698 for (s = 0; s < qae->qae_num_slices; s++) {
2699 if (qae->qae_slices[s].qas_image == NULL)
2700 continue;
2701
2702 image = qae->qae_slices[s].qas_image->qui_image;
2703 qat_ae_write_ctx_mode(sc, ae,
2704 __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE));
2705
2706 nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE);
2707 if (nn_mode != AE_MODE_NN_MODE_DONTCARE)
2708 qat_ae_write_nn_mode(sc, ae, nn_mode);
2709
2710 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0,
2711 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0));
2712 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1,
2713 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1));
2714
2715 qat_ae_write_shared_cs_mode(sc, ae,
2716 __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE));
2717 qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size,
2718 __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED),
2719 qae->qae_reloc_ustore_dram);
2720 }
2721
2722 return 0;
2723 }
2724
2725 static int
qat_aefw_uof_init(struct qat_softc * sc)2726 qat_aefw_uof_init(struct qat_softc *sc)
2727 {
2728 int ae, i, error;
2729 uint32_t mask;
2730
2731 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
2732 struct qat_ae *qae;
2733
2734 if (!(mask & 1))
2735 continue;
2736
2737 qae = &(QAT_AE(sc, ae));
2738
2739 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
2740 if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned &
2741 (1 << ae)) == 0)
2742 continue;
2743
2744 error = qat_aefw_uof_assign_image(sc, qae,
2745 &sc->sc_aefw_uof.qafu_imgs[i]);
2746 if (error)
2747 return error;
2748 }
2749
2750 /* XXX UcLo_initNumUwordUsed */
2751
2752 qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */
2753
2754 error = qat_aefw_uof_init_ae(sc, ae);
2755 if (error)
2756 return error;
2757 }
2758
2759 return 0;
2760 }
2761
2762 int
qat_aefw_load(struct qat_softc * sc)2763 qat_aefw_load(struct qat_softc *sc)
2764 {
2765 int error;
2766
2767 error = qat_aefw_load_mof(sc);
2768 if (error)
2769 return error;
2770
2771 error = qat_aefw_load_mmp(sc);
2772 if (error)
2773 return error;
2774
2775 error = qat_aefw_mof_parse(sc);
2776 if (error) {
2777 device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error);
2778 return error;
2779 }
2780
2781 if (sc->sc_hw.qhw_fw_auth) {
2782 error = qat_aefw_suof_parse(sc);
2783 if (error) {
2784 device_printf(sc->sc_dev, "couldn't parse suof: %d\n",
2785 error);
2786 return error;
2787 }
2788
2789 error = qat_aefw_suof_write(sc);
2790 if (error) {
2791 device_printf(sc->sc_dev,
2792 "could not write firmware: %d\n", error);
2793 return error;
2794 }
2795
2796 } else {
2797 error = qat_aefw_uof_parse(sc);
2798 if (error) {
2799 device_printf(sc->sc_dev, "couldn't parse uof: %d\n",
2800 error);
2801 return error;
2802 }
2803
2804 error = qat_aefw_uof_init(sc);
2805 if (error) {
2806 device_printf(sc->sc_dev,
2807 "couldn't init for aefw: %d\n", error);
2808 return error;
2809 }
2810
2811 error = qat_aefw_uof_write(sc);
2812 if (error) {
2813 device_printf(sc->sc_dev,
2814 "Could not write firmware: %d\n", error);
2815 return error;
2816 }
2817 }
2818
2819 return 0;
2820 }
2821
2822 void
qat_aefw_unload(struct qat_softc * sc)2823 qat_aefw_unload(struct qat_softc *sc)
2824 {
2825 qat_aefw_unload_mmp(sc);
2826 qat_aefw_unload_mof(sc);
2827 }
2828
2829 int
qat_aefw_start(struct qat_softc * sc,u_char ae,u_int ctx_mask)2830 qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask)
2831 {
2832 uint32_t fcu;
2833 int retry = 0;
2834
2835 if (sc->sc_hw.qhw_fw_auth) {
2836 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START);
2837 do {
2838 DELAY(FW_AUTH_WAIT_PERIOD * 1000);
2839 fcu = qat_cap_global_read_4(sc, FCU_STATUS);
2840 if (fcu & FCU_STATUS_DONE)
2841 return 0;
2842 } while (retry++ < FW_AUTH_MAX_RETRY);
2843
2844 device_printf(sc->sc_dev,
2845 "firmware start timeout: status %08x\n", fcu);
2846 return EINVAL;
2847 } else {
2848 qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX,
2849 CTX_WAKEUP_EVENTS_INDIRECT,
2850 CTX_WAKEUP_EVENTS_INDIRECT_SLEEP);
2851 qat_ae_enable_ctx(sc, ae, ctx_mask);
2852 }
2853
2854 return 0;
2855 }
2856
2857 static int
qat_aefw_init_memory_one(struct qat_softc * sc,struct uof_init_mem * uim)2858 qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim)
2859 {
2860 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
2861 struct qat_ae_batch_init_list *qabi_list;
2862 struct uof_mem_val_attr *memattr;
2863 size_t *curinit;
2864 u_long ael;
2865 int i;
2866 const char *sym;
2867 char *ep;
2868
2869 memattr = (struct uof_mem_val_attr *)(uim + 1);
2870
2871 switch (uim->uim_region) {
2872 case LMEM_REGION:
2873 if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) {
2874 device_printf(sc->sc_dev,
2875 "Invalid lmem addr or bytes\n");
2876 return ENOBUFS;
2877 }
2878 if (uim->uim_scope != UOF_SCOPE_LOCAL)
2879 return EINVAL;
2880 sym = qat_aefw_uof_string(sc, uim->uim_sym_name);
2881 ael = strtoul(sym, &ep, 10);
2882 if (ep == sym || ael > MAX_AE)
2883 return EINVAL;
2884 if ((sc->sc_ae_mask & (1 << ael)) == 0)
2885 return 0; /* ae is fused out */
2886
2887 curinit = &qafu->qafu_num_lm_init[ael];
2888 qabi_list = &qafu->qafu_lm_init[ael];
2889
2890 for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) {
2891 struct qat_ae_batch_init *qabi;
2892
2893 qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init));
2894 if (*curinit == 0)
2895 STAILQ_INIT(qabi_list);
2896 STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next);
2897
2898 qabi->qabi_ae = (u_int)ael;
2899 qabi->qabi_addr =
2900 uim->uim_addr + memattr->umva_byte_offset;
2901 qabi->qabi_value = &memattr->umva_value;
2902 qabi->qabi_size = 4;
2903 qafu->qafu_num_lm_init_inst[ael] +=
2904 qat_ae_get_inst_num(qabi->qabi_size);
2905 (*curinit)++;
2906 if (*curinit >= MAX_LMEM_REG) {
2907 device_printf(sc->sc_dev,
2908 "Invalid lmem val attr\n");
2909 return ENOBUFS;
2910 }
2911 }
2912 break;
2913 case SRAM_REGION:
2914 case DRAM_REGION:
2915 case DRAM1_REGION:
2916 case SCRATCH_REGION:
2917 case UMEM_REGION:
2918 /* XXX */
2919 /* fallthrough */
2920 default:
2921 device_printf(sc->sc_dev,
2922 "unsupported memory region to init: %d\n",
2923 uim->uim_region);
2924 return ENOTSUP;
2925 }
2926
2927 return 0;
2928 }
2929
2930 static void
qat_aefw_free_lm_init(struct qat_softc * sc,u_char ae)2931 qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae)
2932 {
2933 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
2934 struct qat_ae_batch_init *qabi;
2935
2936 while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) {
2937 STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next);
2938 qat_free_mem(qabi);
2939 }
2940
2941 qafu->qafu_num_lm_init[ae] = 0;
2942 qafu->qafu_num_lm_init_inst[ae] = 0;
2943 }
2944
2945 static int
qat_aefw_init_ustore(struct qat_softc * sc)2946 qat_aefw_init_ustore(struct qat_softc *sc)
2947 {
2948 uint64_t *fill;
2949 uint32_t dont_init;
2950 int a, i, p;
2951 int error = 0;
2952 int usz, end, start;
2953 u_char ae, nae;
2954
2955 fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t));
2956
2957 for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) {
2958 struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a];
2959 struct uof_image *ui = qui->qui_image;
2960
2961 for (i = 0; i < MAX_USTORE; i++)
2962 memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t));
2963 /*
2964 * Compute do_not_init value as a value that will not be equal
2965 * to fill data when cast to an int
2966 */
2967 dont_init = 0;
2968 if (dont_init == (uint32_t)fill[0])
2969 dont_init = 0xffffffff;
2970
2971 for (p = 0; p < ui->ui_num_pages; p++) {
2972 struct qat_uof_page *qup = &qui->qui_pages[p];
2973 if (!qup->qup_def_page)
2974 continue;
2975
2976 for (i = qup->qup_beg_paddr;
2977 i < qup->qup_beg_paddr + qup->qup_num_micro_words;
2978 i++ ) {
2979 fill[i] = (uint64_t)dont_init;
2980 }
2981 }
2982
2983 for (ae = 0; ae < sc->sc_ae_num; ae++) {
2984 MPASS(ae < UOF_MAX_NUM_OF_AE);
2985 if ((ui->ui_ae_assigned & (1 << ae)) == 0)
2986 continue;
2987
2988 if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) {
2989 qat_ae_get_shared_ustore_ae(ae, &nae);
2990 if (ui->ui_ae_assigned & (1 << ae))
2991 continue;
2992 }
2993 usz = QAT_AE(sc, ae).qae_effect_ustore_size;
2994
2995 /* initialize the areas not going to be overwritten */
2996 end = -1;
2997 do {
2998 /* find next uword that needs to be initialized */
2999 for (start = end + 1; start < usz; start++) {
3000 if ((uint32_t)fill[start] != dont_init)
3001 break;
3002 }
3003 /* see if there are no more such uwords */
3004 if (start >= usz)
3005 break;
3006 for (end = start + 1; end < usz; end++) {
3007 if ((uint32_t)fill[end] == dont_init)
3008 break;
3009 }
3010 if (QAT_AE(sc, ae).qae_shareable_ustore) {
3011 error = ENOTSUP; /* XXX */
3012 goto out;
3013 } else {
3014 error = qat_ae_ucode_write(sc, ae,
3015 start, end - start, &fill[start]);
3016 if (error) {
3017 goto out;
3018 }
3019 }
3020
3021 } while (end < usz);
3022 }
3023 }
3024
3025 out:
3026 qat_free_mem(fill);
3027 return error;
3028 }
3029
3030 static int
qat_aefw_init_reg(struct qat_softc * sc,u_char ae,u_char ctx_mask,enum aereg_type regtype,u_short regaddr,u_int value)3031 qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask,
3032 enum aereg_type regtype, u_short regaddr, u_int value)
3033 {
3034 int error = 0;
3035 u_char ctx;
3036
3037 switch (regtype) {
3038 case AEREG_GPA_REL:
3039 case AEREG_GPB_REL:
3040 case AEREG_SR_REL:
3041 case AEREG_SR_RD_REL:
3042 case AEREG_SR_WR_REL:
3043 case AEREG_DR_REL:
3044 case AEREG_DR_RD_REL:
3045 case AEREG_DR_WR_REL:
3046 case AEREG_NEIGH_REL:
3047 /* init for all valid ctx */
3048 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
3049 if ((ctx_mask & (1 << ctx)) == 0)
3050 continue;
3051 error = qat_aereg_rel_data_write(sc, ae, ctx, regtype,
3052 regaddr, value);
3053 }
3054 break;
3055 case AEREG_GPA_ABS:
3056 case AEREG_GPB_ABS:
3057 case AEREG_SR_ABS:
3058 case AEREG_SR_RD_ABS:
3059 case AEREG_SR_WR_ABS:
3060 case AEREG_DR_ABS:
3061 case AEREG_DR_RD_ABS:
3062 case AEREG_DR_WR_ABS:
3063 error = qat_aereg_abs_data_write(sc, ae, regtype,
3064 regaddr, value);
3065 break;
3066 default:
3067 error = EINVAL;
3068 break;
3069 }
3070
3071 return error;
3072 }
3073
3074 static int
qat_aefw_init_reg_sym_expr(struct qat_softc * sc,u_char ae,struct qat_uof_image * qui)3075 qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae,
3076 struct qat_uof_image *qui)
3077 {
3078 u_int i, expres;
3079 u_char ctx_mask;
3080
3081 for (i = 0; i < qui->qui_num_init_reg_sym; i++) {
3082 struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i];
3083
3084 if (uirs->uirs_value_type == EXPR_VAL) {
3085 /* XXX */
3086 device_printf(sc->sc_dev,
3087 "does not support initializing EXPR_VAL\n");
3088 return ENOTSUP;
3089 } else {
3090 expres = uirs->uirs_value;
3091 }
3092
3093 switch (uirs->uirs_init_type) {
3094 case INIT_REG:
3095 if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
3096 AE_MODE_CTX_MODE) == MAX_AE_CTX) {
3097 ctx_mask = 0xff; /* 8-ctx mode */
3098 } else {
3099 ctx_mask = 0x55; /* 4-ctx mode */
3100 }
3101 qat_aefw_init_reg(sc, ae, ctx_mask,
3102 (enum aereg_type)uirs->uirs_reg_type,
3103 (u_short)uirs->uirs_addr_offset, expres);
3104 break;
3105 case INIT_REG_CTX:
3106 if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
3107 AE_MODE_CTX_MODE) == MAX_AE_CTX) {
3108 ctx_mask = 0xff; /* 8-ctx mode */
3109 } else {
3110 ctx_mask = 0x55; /* 4-ctx mode */
3111 }
3112 if (((1 << uirs->uirs_ctx) & ctx_mask) == 0)
3113 return EINVAL;
3114 qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx,
3115 (enum aereg_type)uirs->uirs_reg_type,
3116 (u_short)uirs->uirs_addr_offset, expres);
3117 break;
3118 case INIT_EXPR:
3119 case INIT_EXPR_ENDIAN_SWAP:
3120 default:
3121 device_printf(sc->sc_dev,
3122 "does not support initializing init_type %d\n",
3123 uirs->uirs_init_type);
3124 return ENOTSUP;
3125 }
3126 }
3127
3128 return 0;
3129 }
3130
3131 static int
qat_aefw_init_memory(struct qat_softc * sc)3132 qat_aefw_init_memory(struct qat_softc *sc)
3133 {
3134 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
3135 size_t uimsz, initmemsz = qafu->qafu_init_mem_size;
3136 struct uof_init_mem *uim;
3137 int error, i;
3138 u_char ae;
3139
3140 uim = qafu->qafu_init_mem;
3141 for (i = 0; i < qafu->qafu_num_init_mem; i++) {
3142 uimsz = sizeof(struct uof_init_mem) +
3143 sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr;
3144 if (uimsz > initmemsz) {
3145 device_printf(sc->sc_dev,
3146 "invalid uof_init_mem or uof_mem_val_attr size\n");
3147 return EINVAL;
3148 }
3149
3150 if (uim->uim_num_bytes > 0) {
3151 error = qat_aefw_init_memory_one(sc, uim);
3152 if (error) {
3153 device_printf(sc->sc_dev,
3154 "Could not init ae memory: %d\n", error);
3155 return error;
3156 }
3157 }
3158 uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz);
3159 initmemsz -= uimsz;
3160 }
3161
3162 /* run Batch put LM API */
3163 for (ae = 0; ae < MAX_AE; ae++) {
3164 error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae],
3165 qafu->qafu_num_lm_init_inst[ae]);
3166 if (error)
3167 device_printf(sc->sc_dev, "Could not put lm\n");
3168
3169 qat_aefw_free_lm_init(sc, ae);
3170 }
3171
3172 error = qat_aefw_init_ustore(sc);
3173
3174 /* XXX run Batch put LM API */
3175
3176 return error;
3177 }
3178
3179 static int
qat_aefw_init_globals(struct qat_softc * sc)3180 qat_aefw_init_globals(struct qat_softc *sc)
3181 {
3182 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
3183 int error, i, p, s;
3184 u_char ae;
3185
3186 /* initialize the memory segments */
3187 if (qafu->qafu_num_init_mem > 0) {
3188 error = qat_aefw_init_memory(sc);
3189 if (error)
3190 return error;
3191 } else {
3192 error = qat_aefw_init_ustore(sc);
3193 if (error)
3194 return error;
3195 }
3196
3197 /* XXX bind import variables with ivd values */
3198
3199 /* XXX bind the uC global variables
3200 * local variables will done on-the-fly */
3201 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
3202 for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) {
3203 struct qat_uof_page *qup =
3204 &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p];
3205 if (qup->qup_num_uw_blocks &&
3206 (qup->qup_num_uc_var || qup->qup_num_imp_var)) {
3207 device_printf(sc->sc_dev,
3208 "not support uC global variables\n");
3209 return ENOTSUP;
3210 }
3211 }
3212 }
3213
3214 for (ae = 0; ae < sc->sc_ae_num; ae++) {
3215 struct qat_ae *qae = &(QAT_AE(sc, ae));
3216
3217 for (s = 0; s < qae->qae_num_slices; s++) {
3218 struct qat_ae_slice *qas = &qae->qae_slices[s];
3219
3220 if (qas->qas_image == NULL)
3221 continue;
3222
3223 error =
3224 qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image);
3225 if (error)
3226 return error;
3227 }
3228 }
3229
3230 return 0;
3231 }
3232
3233 static uint64_t
qat_aefw_get_uof_inst(struct qat_softc * sc,struct qat_uof_page * qup,u_int addr)3234 qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup,
3235 u_int addr)
3236 {
3237 uint64_t uinst = 0;
3238 u_int i;
3239
3240 /* find the block */
3241 for (i = 0; i < qup->qup_num_uw_blocks; i++) {
3242 struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i];
3243
3244 if ((addr >= quub->quub_start_addr) &&
3245 (addr <= (quub->quub_start_addr +
3246 (quub->quub_num_words - 1)))) {
3247 /* unpack n bytes and assigned to the 64-bit uword value.
3248 note: the microwords are stored as packed bytes.
3249 */
3250 addr -= quub->quub_start_addr;
3251 addr *= AEV2_PACKED_UWORD_BYTES;
3252 memcpy(&uinst,
3253 (void *)((uintptr_t)quub->quub_micro_words + addr),
3254 AEV2_PACKED_UWORD_BYTES);
3255 uinst = uinst & UWORD_MASK;
3256
3257 return uinst;
3258 }
3259 }
3260
3261 return INVLD_UWORD;
3262 }
3263
3264 static int
qat_aefw_do_pagein(struct qat_softc * sc,u_char ae,struct qat_uof_page * qup)3265 qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup)
3266 {
3267 struct qat_ae *qae = &(QAT_AE(sc, ae));
3268 uint64_t fill, *ucode_cpybuf;
3269 u_int error, i, upaddr, ninst, cpylen;
3270
3271 if (qup->qup_num_uc_var || qup->qup_num_neigh_reg ||
3272 qup->qup_num_imp_var || qup->qup_num_imp_expr) {
3273 device_printf(sc->sc_dev,
3274 "does not support fixup locals\n");
3275 return ENOTSUP;
3276 }
3277
3278 ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t));
3279
3280 /* XXX get fill-pattern from an image -- they are all the same */
3281 memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern,
3282 sizeof(uint64_t));
3283
3284 upaddr = qup->qup_beg_paddr;
3285 ninst = qup->qup_num_micro_words;
3286 while (ninst > 0) {
3287 cpylen = min(ninst, UWORD_CPYBUF_SIZE);
3288
3289 /* load the buffer */
3290 for (i = 0; i < cpylen; i++) {
3291 /* keep below code structure in case there are
3292 * different handling for shared secnarios */
3293 if (!qae->qae_shareable_ustore) {
3294 /* qat_aefw_get_uof_inst() takes an address that
3295 * is relative to the start of the page.
3296 * So we don't need to add in the physical
3297 * offset of the page. */
3298 if (qup->qup_page_region != 0) {
3299 /* XXX */
3300 device_printf(sc->sc_dev,
3301 "region != 0 is not supported\n");
3302 qat_free_mem(ucode_cpybuf);
3303 return ENOTSUP;
3304 } else {
3305 /* for mixing case, it should take
3306 * physical address */
3307 ucode_cpybuf[i] = qat_aefw_get_uof_inst(
3308 sc, qup, upaddr + i);
3309 if (ucode_cpybuf[i] == INVLD_UWORD) {
3310 /* fill hole in the uof */
3311 ucode_cpybuf[i] = fill;
3312 }
3313 }
3314 } else {
3315 /* XXX */
3316 qat_free_mem(ucode_cpybuf);
3317 return ENOTSUP;
3318 }
3319 }
3320
3321 /* copy the buffer to ustore */
3322 if (!qae->qae_shareable_ustore) {
3323 error = qat_ae_ucode_write(sc, ae, upaddr, cpylen,
3324 ucode_cpybuf);
3325 if (error)
3326 return error;
3327 } else {
3328 /* XXX */
3329 qat_free_mem(ucode_cpybuf);
3330 return ENOTSUP;
3331 }
3332 upaddr += cpylen;
3333 ninst -= cpylen;
3334 }
3335
3336 qat_free_mem(ucode_cpybuf);
3337
3338 return 0;
3339 }
3340
3341 static int
qat_aefw_uof_write_one(struct qat_softc * sc,struct qat_uof_image * qui)3342 qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui)
3343 {
3344 struct uof_image *ui = qui->qui_image;
3345 struct qat_ae_page *qap;
3346 u_int s, p, c;
3347 int error;
3348 u_char ae, ctx_mask;
3349
3350 if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX)
3351 ctx_mask = 0xff; /* 8-ctx mode */
3352 else
3353 ctx_mask = 0x55; /* 4-ctx mode */
3354
3355 /* load the default page and set assigned CTX PC
3356 * to the entrypoint address */
3357 for (ae = 0; ae < sc->sc_ae_num; ae++) {
3358 struct qat_ae *qae = &(QAT_AE(sc, ae));
3359 struct qat_ae_slice *qas;
3360 u_int metadata;
3361
3362 MPASS(ae < UOF_MAX_NUM_OF_AE);
3363
3364 if ((ui->ui_ae_assigned & (1 << ae)) == 0)
3365 continue;
3366
3367 /* find the slice to which this image is assigned */
3368 for (s = 0; s < qae->qae_num_slices; s++) {
3369 qas = &qae->qae_slices[s];
3370 if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask)
3371 break;
3372 }
3373 if (s >= qae->qae_num_slices)
3374 continue;
3375
3376 qas = &qae->qae_slices[s];
3377
3378 for (p = 0; p < ui->ui_num_pages; p++) {
3379 qap = &qas->qas_pages[p];
3380
3381 /* Only load pages loaded by default */
3382 if (!qap->qap_page->qup_def_page)
3383 continue;
3384
3385 error = qat_aefw_do_pagein(sc, ae, qap->qap_page);
3386 if (error)
3387 return error;
3388 }
3389
3390 metadata = qas->qas_image->qui_image->ui_app_metadata;
3391 if (metadata != 0xffffffff && bootverbose) {
3392 device_printf(sc->sc_dev,
3393 "loaded firmware: %s\n",
3394 qat_aefw_uof_string(sc, metadata));
3395 }
3396
3397 /* Assume starting page is page 0 */
3398 qap = &qas->qas_pages[0];
3399 for (c = 0; c < MAX_AE_CTX; c++) {
3400 if (ctx_mask & (1 << c))
3401 qas->qas_cur_pages[c] = qap;
3402 else
3403 qas->qas_cur_pages[c] = NULL;
3404 }
3405
3406 /* set the live context */
3407 qae->qae_live_ctx_mask = ui->ui_ctx_assigned;
3408
3409 /* set context PC to the image entrypoint address */
3410 error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned,
3411 ui->ui_entry_address);
3412 if (error)
3413 return error;
3414 }
3415
3416 /* XXX store the checksum for convenience */
3417
3418 return 0;
3419 }
3420
3421 static int
qat_aefw_uof_write(struct qat_softc * sc)3422 qat_aefw_uof_write(struct qat_softc *sc)
3423 {
3424 int error = 0;
3425 int i;
3426
3427 error = qat_aefw_init_globals(sc);
3428 if (error) {
3429 device_printf(sc->sc_dev,
3430 "Could not initialize globals\n");
3431 return error;
3432 }
3433
3434 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
3435 error = qat_aefw_uof_write_one(sc,
3436 &sc->sc_aefw_uof.qafu_imgs[i]);
3437 if (error)
3438 break;
3439 }
3440
3441 /* XXX UcLo_computeFreeUstore */
3442
3443 return error;
3444 }
3445