1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */ 2 /* $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ 3 4 /* 5 * Copyright (c) 2019 Internet Initiative Japan, Inc. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 /* 31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 37 * * Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * * Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in 41 * the documentation and/or other materials provided with the 42 * distribution. 43 * * Neither the name of Intel Corporation nor the names of its 44 * contributors may be used to endorse or promote products derived 45 * from this software without specific prior written permission. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58 */ 59 60 #include <sys/cdefs.h> 61 #if 0 62 __KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); 63 #endif 64 65 #include <sys/param.h> 66 #include <sys/bus.h> 67 #include <sys/firmware.h> 68 #include <sys/limits.h> 69 #include <sys/systm.h> 70 71 #include <machine/bus.h> 72 73 #include <dev/pci/pcireg.h> 74 #include <dev/pci/pcivar.h> 75 76 #include "qatreg.h" 77 #include "qatvar.h" 78 #include "qat_aevar.h" 79 80 static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t, 81 uint32_t); 82 static int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t, 83 uint32_t *); 84 static void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t, 85 bus_size_t, uint32_t); 86 static int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t, 87 bus_size_t, uint32_t *); 88 89 static u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short); 90 static int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char, 91 enum aereg_type, u_short, uint32_t); 92 static int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char, 93 enum aereg_type, u_short, uint32_t *); 94 static int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char, 95 enum aereg_type, u_short, uint32_t); 96 static int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char, 97 enum aereg_type, u_short, uint32_t); 98 static int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char, 99 enum aereg_type, u_short, uint32_t); 100 static int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short, 101 u_short *, u_char *); 102 static int qat_aereg_abs_data_write(struct qat_softc *, u_char, 103 enum aereg_type, u_short, uint32_t); 104 105 static void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int); 106 static void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int); 107 static void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char); 108 static void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char); 109 static void qat_ae_write_lm_mode(struct qat_softc *, u_char, 110 enum aereg_type, u_char); 111 static void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char, 112 u_char); 113 static void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char); 114 static int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int, 115 u_int); 116 117 static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char); 118 static int qat_ae_is_active(struct qat_softc *, u_char); 119 static int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int); 120 121 static int qat_ae_clear_reset(struct qat_softc *); 122 static int qat_ae_check(struct qat_softc *); 123 static int qat_ae_reset_timestamp(struct qat_softc *); 124 static void qat_ae_clear_xfer(struct qat_softc *); 125 static int qat_ae_clear_gprs(struct qat_softc *); 126 127 static void qat_ae_get_shared_ustore_ae(u_char, u_char *); 128 static u_int qat_ae_ucode_parity64(uint64_t); 129 static uint64_t qat_ae_ucode_set_ecc(uint64_t); 130 static int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int, 131 const uint64_t *); 132 static int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int, 133 uint64_t *); 134 static u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *); 135 static int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char, 136 uint64_t *, u_int, int, u_int, u_int *); 137 static int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char, 138 int *, uint64_t *, u_int, 139 u_int *, u_int *, u_int *, u_int *, u_int *); 140 static int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char, 141 u_int, u_int, u_int, u_int, u_int); 142 static int qat_ae_get_inst_num(int); 143 static int qat_ae_batch_put_lm(struct qat_softc *, u_char, 144 struct qat_ae_batch_init_list *, size_t); 145 static int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int); 146 147 static u_int qat_aefw_csum(char *, int); 148 static const char *qat_aefw_uof_string(struct qat_softc *, size_t); 149 static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *, 150 const char *, struct uof_chunk_hdr *); 151 152 static int qat_aefw_load_mof(struct qat_softc *); 153 static void qat_aefw_unload_mof(struct qat_softc *); 154 static int qat_aefw_load_mmp(struct qat_softc *); 155 static void qat_aefw_unload_mmp(struct qat_softc *); 156 157 static int qat_aefw_mof_find_uof0(struct qat_softc *, 158 struct mof_uof_hdr *, struct mof_uof_chunk_hdr *, 159 u_int, size_t, const char *, 160 size_t *, void **); 161 static int qat_aefw_mof_find_uof(struct qat_softc *); 162 static int qat_aefw_mof_parse(struct qat_softc *); 163 164 static int qat_aefw_uof_parse_image(struct qat_softc *, 165 struct qat_uof_image *, struct uof_chunk_hdr *uch); 166 static int qat_aefw_uof_parse_images(struct qat_softc *); 167 static int qat_aefw_uof_parse(struct qat_softc *); 168 169 static int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t, 170 struct qat_dmamem *); 171 static int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *); 172 static int qat_aefw_suof_load(struct qat_softc *sc, 173 struct qat_dmamem *dma); 174 static int qat_aefw_suof_parse_image(struct qat_softc *, 175 struct qat_suof_image *, struct suof_chunk_hdr *); 176 static int qat_aefw_suof_parse(struct qat_softc *); 177 static int qat_aefw_suof_write(struct qat_softc *); 178 179 static int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *, 180 struct qat_uof_image *); 181 static int qat_aefw_uof_init_ae(struct qat_softc *, u_char); 182 static int qat_aefw_uof_init(struct qat_softc *); 183 184 static int qat_aefw_init_memory_one(struct qat_softc *, 185 struct uof_init_mem *); 186 static void qat_aefw_free_lm_init(struct qat_softc *, u_char); 187 static int qat_aefw_init_ustore(struct qat_softc *); 188 static int qat_aefw_init_reg(struct qat_softc *, u_char, u_char, 189 enum aereg_type, u_short, u_int); 190 static int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char, 191 struct qat_uof_image *); 192 static int qat_aefw_init_memory(struct qat_softc *); 193 static int qat_aefw_init_globals(struct qat_softc *); 194 static uint64_t qat_aefw_get_uof_inst(struct qat_softc *, 195 struct qat_uof_page *, u_int); 196 static int qat_aefw_do_pagein(struct qat_softc *, u_char, 197 struct qat_uof_page *); 198 static int qat_aefw_uof_write_one(struct qat_softc *, 199 struct qat_uof_image *); 200 static int qat_aefw_uof_write(struct qat_softc *); 201 202 static int 203 qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, 204 uint32_t value) 205 { 206 int times = TIMEOUT_AE_CSR; 207 208 do { 209 qat_ae_local_write_4(sc, ae, offset, value); 210 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & 211 LOCAL_CSR_STATUS_STATUS) == 0) 212 return 0; 213 214 } while (times--); 215 216 device_printf(sc->sc_dev, 217 "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); 218 return EFAULT; 219 } 220 221 static int 222 qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset, 223 uint32_t *value) 224 { 225 int times = TIMEOUT_AE_CSR; 226 uint32_t v; 227 228 do { 229 v = qat_ae_local_read_4(sc, ae, offset); 230 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & 231 LOCAL_CSR_STATUS_STATUS) == 0) { 232 *value = v; 233 return 0; 234 } 235 } while (times--); 236 237 device_printf(sc->sc_dev, 238 "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); 239 return EFAULT; 240 } 241 242 static void 243 qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask, 244 bus_size_t offset, uint32_t value) 245 { 246 int ctx; 247 uint32_t ctxptr; 248 249 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || 250 offset == FUTURE_COUNT_SIGNAL_INDIRECT || 251 offset == CTX_STS_INDIRECT || 252 offset == CTX_WAKEUP_EVENTS_INDIRECT || 253 offset == CTX_SIG_EVENTS_INDIRECT || 254 offset == LM_ADDR_0_INDIRECT || 255 offset == LM_ADDR_1_INDIRECT || 256 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || 257 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); 258 259 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); 260 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { 261 if ((ctx_mask & (1 << ctx)) == 0) 262 continue; 263 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); 264 qat_ae_write_4(sc, ae, offset, value); 265 } 266 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); 267 } 268 269 static int 270 qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx, 271 bus_size_t offset, uint32_t *value) 272 { 273 int error; 274 uint32_t ctxptr; 275 276 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || 277 offset == FUTURE_COUNT_SIGNAL_INDIRECT || 278 offset == CTX_STS_INDIRECT || 279 offset == CTX_WAKEUP_EVENTS_INDIRECT || 280 offset == CTX_SIG_EVENTS_INDIRECT || 281 offset == LM_ADDR_0_INDIRECT || 282 offset == LM_ADDR_1_INDIRECT || 283 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || 284 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); 285 286 /* save the ctx ptr */ 287 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); 288 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != 289 (ctx & CSR_CTX_POINTER_CONTEXT)) 290 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); 291 292 error = qat_ae_read_4(sc, ae, offset, value); 293 294 /* restore ctx ptr */ 295 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != 296 (ctx & CSR_CTX_POINTER_CONTEXT)) 297 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); 298 299 return error; 300 } 301 302 static u_short 303 qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg) 304 { 305 u_short addr; 306 307 switch (regtype) { 308 case AEREG_GPA_ABS: 309 case AEREG_GPB_ABS: 310 addr = (reg & 0x7f) | 0x80; 311 break; 312 case AEREG_GPA_REL: 313 case AEREG_GPB_REL: 314 addr = reg & 0x1f; 315 break; 316 case AEREG_SR_RD_REL: 317 case AEREG_SR_WR_REL: 318 case AEREG_SR_REL: 319 addr = 0x180 | (reg & 0x1f); 320 break; 321 case AEREG_SR_INDX: 322 addr = 0x140 | ((reg & 0x3) << 1); 323 break; 324 case AEREG_DR_RD_REL: 325 case AEREG_DR_WR_REL: 326 case AEREG_DR_REL: 327 addr = 0x1c0 | (reg & 0x1f); 328 break; 329 case AEREG_DR_INDX: 330 addr = 0x100 | ((reg & 0x3) << 1); 331 break; 332 case AEREG_NEIGH_INDX: 333 addr = 0x241 | ((reg & 0x3) << 1); 334 break; 335 case AEREG_NEIGH_REL: 336 addr = 0x280 | (reg & 0x1f); 337 break; 338 case AEREG_LMEM0: 339 addr = 0x200; 340 break; 341 case AEREG_LMEM1: 342 addr = 0x220; 343 break; 344 case AEREG_NO_DEST: 345 addr = 0x300 | (reg & 0xff); 346 break; 347 default: 348 addr = AEREG_BAD_REGADDR; 349 break; 350 } 351 return (addr); 352 } 353 354 static int 355 qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx, 356 enum aereg_type regtype, u_short relreg, uint32_t value) 357 { 358 uint16_t srchi, srclo, destaddr, data16hi, data16lo; 359 uint64_t inst[] = { 360 0x0F440000000ull, /* immed_w1[reg, val_hi16] */ 361 0x0F040000000ull, /* immed_w0[reg, val_lo16] */ 362 0x0F0000C0300ull, /* nop */ 363 0x0E000010000ull /* ctx_arb[kill] */ 364 }; 365 const int ninst = nitems(inst); 366 const int imm_w1 = 0, imm_w0 = 1; 367 unsigned int ctxen; 368 uint16_t mask; 369 370 /* This logic only works for GPRs and LM index registers, 371 not NN or XFER registers! */ 372 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || 373 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); 374 375 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) { 376 /* determine the context mode */ 377 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 378 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 379 /* 4-ctx mode */ 380 if (ctx & 0x1) 381 return EINVAL; 382 mask = 0x1f; 383 } else { 384 /* 8-ctx mode */ 385 mask = 0x0f; 386 } 387 if (relreg & ~mask) 388 return EINVAL; 389 } 390 if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == 391 AEREG_BAD_REGADDR) { 392 return EINVAL; 393 } 394 395 data16lo = 0xffff & value; 396 data16hi = 0xffff & (value >> 16); 397 srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST, 398 (uint16_t)(0xff & data16hi)); 399 srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST, 400 (uint16_t)(0xff & data16lo)); 401 402 switch (regtype) { 403 case AEREG_GPA_REL: /* A rel source */ 404 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | 405 ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff); 406 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | 407 ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff); 408 break; 409 default: 410 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | 411 ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff); 412 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | 413 ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff); 414 break; 415 } 416 417 return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL); 418 } 419 420 static int 421 qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx, 422 enum aereg_type regtype, u_short relreg, uint32_t *value) 423 { 424 uint64_t inst, savucode; 425 uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi; 426 u_int uaddr, ustore_addr; 427 int error; 428 u_short mask, regaddr; 429 u_char nae; 430 431 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || 432 regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL || 433 regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL || 434 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); 435 436 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) || 437 (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) || 438 (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL)) 439 { 440 /* determine the context mode */ 441 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 442 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 443 /* 4-ctx mode */ 444 if (ctx & 0x1) 445 return EINVAL; 446 mask = 0x1f; 447 } else { 448 /* 8-ctx mode */ 449 mask = 0x0f; 450 } 451 if (relreg & ~mask) 452 return EINVAL; 453 } 454 if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == 455 AEREG_BAD_REGADDR) { 456 return EINVAL; 457 } 458 459 /* instruction -- alu[--, --, B, reg] */ 460 switch (regtype) { 461 case AEREG_GPA_REL: 462 /* A rel source */ 463 inst = 0xA070000000ull | (regaddr & 0x3ff); 464 break; 465 default: 466 inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10)); 467 break; 468 } 469 470 /* backup shared control store bit, and force AE to 471 * none-shared mode before executing ucode snippet */ 472 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 473 if (misc & AE_MISC_CONTROL_SHARE_CS) { 474 qat_ae_get_shared_ustore_ae(ae, &nae); 475 if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae)) 476 return EBUSY; 477 } 478 479 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 480 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 481 482 /* read current context */ 483 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); 484 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); 485 486 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 487 /* prevent clearing the W1C bits: the breakpoint bit, 488 ECC error bit, and Parity error bit */ 489 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 490 491 /* change the context */ 492 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) 493 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 494 ctx & ACTIVE_CTX_STATUS_ACNO); 495 /* save a ustore location */ 496 if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) { 497 /* restore AE_MISC_CONTROL csr */ 498 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 499 500 /* restore the context */ 501 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { 502 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 503 savctx & ACTIVE_CTX_STATUS_ACNO); 504 } 505 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 506 507 return (error); 508 } 509 510 /* turn off ustore parity */ 511 qat_ae_write_4(sc, ae, CTX_ENABLES, 512 ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE)); 513 514 /* save ustore-addr csr */ 515 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 516 517 /* write the ALU instruction to ustore, enable ecs bit */ 518 uaddr = 0 | USTORE_ADDRESS_ECS; 519 520 /* set the uaddress */ 521 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 522 inst = qat_ae_ucode_set_ecc(inst); 523 524 ulo = (uint32_t)(inst & 0xffffffff); 525 uhi = (uint32_t)(inst >> 32); 526 527 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); 528 529 /* this will auto increment the address */ 530 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); 531 532 /* set the uaddress */ 533 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 534 535 /* delay for at least 8 cycles */ 536 qat_ae_wait_num_cycles(sc, ae, 0x8, 0); 537 538 /* read ALU output -- the instruction should have been executed 539 prior to clearing the ECS in putUwords */ 540 qat_ae_read_4(sc, ae, ALU_OUT, value); 541 542 /* restore ustore-addr csr */ 543 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 544 545 /* restore the ustore */ 546 error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode); 547 548 /* restore the context */ 549 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { 550 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 551 savctx & ACTIVE_CTX_STATUS_ACNO); 552 } 553 554 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 555 556 /* restore AE_MISC_CONTROL csr */ 557 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 558 559 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 560 561 return error; 562 } 563 564 static int 565 qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, 566 enum aereg_type regtype, u_short relreg, uint32_t value) 567 { 568 bus_size_t addr; 569 int error; 570 uint32_t ctxen; 571 u_short mask; 572 u_short dr_offset; 573 574 MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL || 575 regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL); 576 577 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 578 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 579 if (ctx & 0x1) { 580 device_printf(sc->sc_dev, 581 "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx); 582 return EINVAL; 583 } 584 mask = 0x1f; 585 dr_offset = 0x20; 586 587 } else { 588 mask = 0x0f; 589 dr_offset = 0x10; 590 } 591 592 if (relreg & ~mask) 593 return EINVAL; 594 595 addr = relreg + (ctx << 0x5); 596 597 switch (regtype) { 598 case AEREG_SR_REL: 599 case AEREG_SR_RD_REL: 600 qat_ae_xfer_write_4(sc, ae, addr, value); 601 break; 602 case AEREG_DR_REL: 603 case AEREG_DR_RD_REL: 604 qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value); 605 break; 606 default: 607 error = EINVAL; 608 } 609 610 return error; 611 } 612 613 static int 614 qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, 615 enum aereg_type regtype, u_short relreg, uint32_t value) 616 { 617 618 panic("notyet"); 619 620 return 0; 621 } 622 623 static int 624 qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx, 625 enum aereg_type regtype, u_short relreg, uint32_t value) 626 { 627 628 panic("notyet"); 629 630 return 0; 631 } 632 633 static int 634 qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae, 635 u_short absreg, u_short *relreg, u_char *ctx) 636 { 637 uint32_t ctxen; 638 639 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 640 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 641 /* 4-ctx mode */ 642 *relreg = absreg & 0x1f; 643 *ctx = (absreg >> 0x4) & 0x6; 644 } else { 645 /* 8-ctx mode */ 646 *relreg = absreg & 0x0f; 647 *ctx = (absreg >> 0x4) & 0x7; 648 } 649 650 return 0; 651 } 652 653 static int 654 qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae, 655 enum aereg_type regtype, u_short absreg, uint32_t value) 656 { 657 int error; 658 u_short relreg; 659 u_char ctx; 660 661 qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx); 662 663 switch (regtype) { 664 case AEREG_GPA_ABS: 665 MPASS(absreg < MAX_GPR_REG); 666 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 667 relreg, value); 668 break; 669 case AEREG_GPB_ABS: 670 MPASS(absreg < MAX_GPR_REG); 671 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 672 relreg, value); 673 break; 674 case AEREG_DR_RD_ABS: 675 MPASS(absreg < MAX_XFER_REG); 676 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL, 677 relreg, value); 678 break; 679 case AEREG_SR_RD_ABS: 680 MPASS(absreg < MAX_XFER_REG); 681 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL, 682 relreg, value); 683 break; 684 case AEREG_DR_WR_ABS: 685 MPASS(absreg < MAX_XFER_REG); 686 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL, 687 relreg, value); 688 break; 689 case AEREG_SR_WR_ABS: 690 MPASS(absreg < MAX_XFER_REG); 691 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL, 692 relreg, value); 693 break; 694 case AEREG_NEIGH_ABS: 695 MPASS(absreg < MAX_NN_REG); 696 if (absreg >= MAX_NN_REG) 697 return EINVAL; 698 error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL, 699 relreg, value); 700 break; 701 default: 702 panic("Invalid Register Type"); 703 } 704 705 return error; 706 } 707 708 static void 709 qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) 710 { 711 uint32_t ctxen; 712 713 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 714 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 715 716 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 717 ctx_mask &= 0x55; 718 } else { 719 ctx_mask &= 0xff; 720 } 721 722 ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE); 723 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 724 } 725 726 static void 727 qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) 728 { 729 uint32_t ctxen; 730 731 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 732 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 733 ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE)); 734 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 735 } 736 737 static void 738 qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode) 739 { 740 uint32_t val, nval; 741 742 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 743 val &= CTX_ENABLES_IGNORE_W1C_MASK; 744 745 if (mode == 4) 746 nval = val | CTX_ENABLES_INUSE_CONTEXTS; 747 else 748 nval = val & ~CTX_ENABLES_INUSE_CONTEXTS; 749 750 if (val != nval) 751 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 752 } 753 754 static void 755 qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode) 756 { 757 uint32_t val, nval; 758 759 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 760 val &= CTX_ENABLES_IGNORE_W1C_MASK; 761 762 if (mode) 763 nval = val | CTX_ENABLES_NN_MODE; 764 else 765 nval = val & ~CTX_ENABLES_NN_MODE; 766 767 if (val != nval) 768 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 769 } 770 771 static void 772 qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae, 773 enum aereg_type lm, u_char mode) 774 { 775 uint32_t val, nval; 776 uint32_t bit; 777 778 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 779 val &= CTX_ENABLES_IGNORE_W1C_MASK; 780 781 switch (lm) { 782 case AEREG_LMEM0: 783 bit = CTX_ENABLES_LMADDR_0_GLOBAL; 784 break; 785 case AEREG_LMEM1: 786 bit = CTX_ENABLES_LMADDR_1_GLOBAL; 787 break; 788 default: 789 panic("invalid lmem reg type"); 790 break; 791 } 792 793 if (mode) 794 nval = val | bit; 795 else 796 nval = val & ~bit; 797 798 if (val != nval) 799 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 800 } 801 802 static void 803 qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode) 804 { 805 uint32_t val, nval; 806 807 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 808 809 if (mode == 1) 810 nval = val | AE_MISC_CONTROL_SHARE_CS; 811 else 812 nval = val & ~AE_MISC_CONTROL_SHARE_CS; 813 814 if (val != nval) 815 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval); 816 } 817 818 static void 819 qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode) 820 { 821 u_char nae; 822 823 qat_ae_get_shared_ustore_ae(ae, &nae); 824 825 qat_ae_write_shared_cs_mode0(sc, ae, mode); 826 827 if ((sc->sc_ae_mask & (1 << nae))) { 828 qat_ae_write_shared_cs_mode0(sc, nae, mode); 829 } 830 } 831 832 static int 833 qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae, 834 u_int reload_size, int shared_mode, u_int ustore_dram_addr) 835 { 836 uint32_t val, cs_reload; 837 838 switch (reload_size) { 839 case 0: 840 cs_reload = 0x0; 841 break; 842 case QAT_2K: 843 cs_reload = 0x1; 844 break; 845 case QAT_4K: 846 cs_reload = 0x2; 847 break; 848 case QAT_8K: 849 cs_reload = 0x3; 850 break; 851 default: 852 return EINVAL; 853 } 854 855 if (cs_reload) 856 QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr; 857 858 QAT_AE(sc, ae).qae_reload_size = reload_size; 859 860 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 861 val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD | 862 AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS); 863 val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) | 864 __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD); 865 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); 866 867 return 0; 868 } 869 870 static enum qat_ae_status 871 qat_ae_get_status(struct qat_softc *sc, u_char ae) 872 { 873 int error; 874 uint32_t val = 0; 875 876 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 877 if (error || val & CTX_ENABLES_ENABLE) 878 return QAT_AE_ENABLED; 879 880 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); 881 if (val & ACTIVE_CTX_STATUS_ABO) 882 return QAT_AE_ACTIVE; 883 884 return QAT_AE_DISABLED; 885 } 886 887 888 static int 889 qat_ae_is_active(struct qat_softc *sc, u_char ae) 890 { 891 uint32_t val; 892 893 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) 894 return 1; 895 896 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); 897 if (val & ACTIVE_CTX_STATUS_ABO) 898 return 1; 899 else 900 return 0; 901 } 902 903 /* returns 1 if actually waited for specified number of cycles */ 904 static int 905 qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check) 906 { 907 uint32_t cnt, actx; 908 int pcnt, ccnt, elapsed, times; 909 910 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 911 pcnt = cnt & 0xffff; 912 913 times = TIMEOUT_AE_CHECK; 914 do { 915 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 916 ccnt = cnt & 0xffff; 917 918 elapsed = ccnt - pcnt; 919 if (elapsed == 0) { 920 times--; 921 } 922 if (times <= 0) { 923 device_printf(sc->sc_dev, 924 "qat_ae_wait_num_cycles timeout\n"); 925 return -1; 926 } 927 928 if (elapsed < 0) 929 elapsed += 0x10000; 930 931 if (elapsed >= CYCLES_FROM_READY2EXE && check) { 932 if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, 933 &actx) == 0) { 934 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) 935 return 0; 936 } 937 } 938 } while (cycles > elapsed); 939 940 if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) { 941 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) 942 return 0; 943 } 944 945 return 1; 946 } 947 948 int 949 qat_ae_init(struct qat_softc *sc) 950 { 951 int error; 952 uint32_t mask, val = 0; 953 u_char ae; 954 955 /* XXX adf_initSysMemInfo */ 956 957 /* XXX Disable clock gating for some chip if debug mode */ 958 959 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 960 struct qat_ae *qae = &sc->sc_ae[ae]; 961 if (!(mask & 1)) 962 continue; 963 964 qae->qae_ustore_size = USTORE_SIZE; 965 966 qae->qae_free_addr = 0; 967 qae->qae_free_size = USTORE_SIZE; 968 qae->qae_live_ctx_mask = AE_ALL_CTX; 969 qae->qae_ustore_dram_addr = 0; 970 qae->qae_reload_size = 0; 971 } 972 973 /* XXX Enable attention interrupt */ 974 975 error = qat_ae_clear_reset(sc); 976 if (error) 977 return error; 978 979 qat_ae_clear_xfer(sc); 980 981 if (!sc->sc_hw.qhw_fw_auth) { 982 error = qat_ae_clear_gprs(sc); 983 if (error) 984 return error; 985 } 986 987 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ 988 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 989 if (!(mask & 1)) 990 continue; 991 qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val); 992 val |= 0x1; 993 qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val); 994 } 995 996 error = qat_ae_clear_reset(sc); 997 if (error) 998 return error; 999 1000 /* XXX XXX XXX Clean MMP memory if mem scrub is supported */ 1001 /* halMem_ScrubMMPMemory */ 1002 1003 return 0; 1004 } 1005 1006 int 1007 qat_ae_start(struct qat_softc *sc) 1008 { 1009 int error; 1010 u_char ae; 1011 1012 for (ae = 0; ae < sc->sc_ae_num; ae++) { 1013 if ((sc->sc_ae_mask & (1 << ae)) == 0) 1014 continue; 1015 1016 error = qat_aefw_start(sc, ae, 0xff); 1017 if (error) 1018 return error; 1019 } 1020 1021 return 0; 1022 } 1023 1024 void 1025 qat_ae_cluster_intr(void *arg) 1026 { 1027 /* Nothing to implement until we support SRIOV. */ 1028 printf("qat_ae_cluster_intr\n"); 1029 } 1030 1031 static int 1032 qat_ae_clear_reset(struct qat_softc *sc) 1033 { 1034 int error; 1035 uint32_t times, reset, clock, reg, mask; 1036 u_char ae; 1037 1038 reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); 1039 reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK)); 1040 reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)); 1041 times = TIMEOUT_AE_RESET; 1042 do { 1043 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset); 1044 if ((times--) == 0) { 1045 device_printf(sc->sc_dev, "couldn't reset AEs\n"); 1046 return EBUSY; 1047 } 1048 reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); 1049 } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) | 1050 __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)) 1051 & reg); 1052 1053 /* Enable clock for AE and QAT */ 1054 clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN); 1055 clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK); 1056 clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK); 1057 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock); 1058 1059 error = qat_ae_check(sc); 1060 if (error) 1061 return error; 1062 1063 /* 1064 * Set undefined power-up/reset states to reasonable default values... 1065 * just to make sure we're starting from a known point 1066 */ 1067 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1068 if (!(mask & 1)) 1069 continue; 1070 1071 /* init the ctx_enable */ 1072 qat_ae_write_4(sc, ae, CTX_ENABLES, 1073 CTX_ENABLES_INIT); 1074 1075 /* initialize the PCs */ 1076 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1077 CTX_STS_INDIRECT, 1078 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1079 1080 /* init the ctx_arb */ 1081 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, 1082 CTX_ARB_CNTL_INIT); 1083 1084 /* enable cc */ 1085 qat_ae_write_4(sc, ae, CC_ENABLE, 1086 CC_ENABLE_INIT); 1087 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1088 CTX_WAKEUP_EVENTS_INDIRECT, 1089 CTX_WAKEUP_EVENTS_INDIRECT_INIT); 1090 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1091 CTX_SIG_EVENTS_INDIRECT, 1092 CTX_SIG_EVENTS_INDIRECT_INIT); 1093 } 1094 1095 if ((sc->sc_ae_mask != 0) && 1096 sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) { 1097 /* XXX XXX XXX init eSram only when this is boot time */ 1098 } 1099 1100 if ((sc->sc_ae_mask != 0) && 1101 sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) { 1102 /* XXX XXX XXX wait shram to complete initialization */ 1103 } 1104 1105 qat_ae_reset_timestamp(sc); 1106 1107 return 0; 1108 } 1109 1110 static int 1111 qat_ae_check(struct qat_softc *sc) 1112 { 1113 int error, times, ae; 1114 uint32_t cnt, pcnt, mask; 1115 1116 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1117 if (!(mask & 1)) 1118 continue; 1119 1120 times = TIMEOUT_AE_CHECK; 1121 error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 1122 if (error) { 1123 device_printf(sc->sc_dev, 1124 "couldn't access AE %d CSR\n", ae); 1125 return error; 1126 } 1127 pcnt = cnt & 0xffff; 1128 1129 while (1) { 1130 error = qat_ae_read_4(sc, ae, 1131 PROFILE_COUNT, &cnt); 1132 if (error) { 1133 device_printf(sc->sc_dev, 1134 "couldn't access AE %d CSR\n", ae); 1135 return error; 1136 } 1137 cnt &= 0xffff; 1138 if (cnt == pcnt) 1139 times--; 1140 else 1141 break; 1142 if (times <= 0) { 1143 device_printf(sc->sc_dev, 1144 "AE %d CSR is useless\n", ae); 1145 return EFAULT; 1146 } 1147 } 1148 } 1149 1150 return 0; 1151 } 1152 1153 static int 1154 qat_ae_reset_timestamp(struct qat_softc *sc) 1155 { 1156 uint32_t misc, mask; 1157 u_char ae; 1158 1159 /* stop the timestamp timers */ 1160 misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC); 1161 if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) { 1162 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, 1163 misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN)); 1164 } 1165 1166 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1167 if (!(mask & 1)) 1168 continue; 1169 qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0); 1170 qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0); 1171 } 1172 1173 /* start timestamp timers */ 1174 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, 1175 misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN); 1176 1177 return 0; 1178 } 1179 1180 static void 1181 qat_ae_clear_xfer(struct qat_softc *sc) 1182 { 1183 u_int mask, reg; 1184 u_char ae; 1185 1186 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1187 if (!(mask & 1)) 1188 continue; 1189 1190 for (reg = 0; reg < MAX_GPR_REG; reg++) { 1191 qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS, 1192 reg, 0); 1193 qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS, 1194 reg, 0); 1195 } 1196 } 1197 } 1198 1199 static int 1200 qat_ae_clear_gprs(struct qat_softc *sc) 1201 { 1202 uint32_t val; 1203 uint32_t saved_ctx = 0; 1204 int times = TIMEOUT_AE_CHECK, rv; 1205 u_char ae; 1206 u_int mask; 1207 1208 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1209 if (!(mask & 1)) 1210 continue; 1211 1212 /* turn off share control store bit */ 1213 val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 1214 val &= ~AE_MISC_CONTROL_SHARE_CS; 1215 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); 1216 1217 /* turn off ucode parity */ 1218 /* make sure nn_mode is set to self */ 1219 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 1220 val &= CTX_ENABLES_IGNORE_W1C_MASK; 1221 val |= CTX_ENABLES_NN_MODE; 1222 val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE; 1223 qat_ae_write_4(sc, ae, CTX_ENABLES, val); 1224 1225 /* copy instructions to ustore */ 1226 qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst), 1227 ae_clear_gprs_inst); 1228 1229 /* set PC */ 1230 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, 1231 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1232 1233 /* save current context */ 1234 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx); 1235 /* change the active context */ 1236 /* start the context from ctx 0 */ 1237 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0); 1238 1239 /* wakeup-event voluntary */ 1240 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1241 CTX_WAKEUP_EVENTS_INDIRECT, 1242 CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); 1243 /* clean signals */ 1244 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1245 CTX_SIG_EVENTS_INDIRECT, 0); 1246 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1247 1248 qat_ae_enable_ctx(sc, ae, AE_ALL_CTX); 1249 } 1250 1251 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1252 if (!(mask & 1)) 1253 continue; 1254 /* wait for AE to finish */ 1255 do { 1256 rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1); 1257 } while (rv && times--); 1258 if (times <= 0) { 1259 device_printf(sc->sc_dev, 1260 "qat_ae_clear_gprs timeout"); 1261 return ETIMEDOUT; 1262 } 1263 qat_ae_disable_ctx(sc, ae, AE_ALL_CTX); 1264 /* change the active context */ 1265 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1266 saved_ctx & ACTIVE_CTX_STATUS_ACNO); 1267 /* init the ctx_enable */ 1268 qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT); 1269 /* initialize the PCs */ 1270 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1271 CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1272 /* init the ctx_arb */ 1273 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT); 1274 /* enable cc */ 1275 qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT); 1276 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1277 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT); 1278 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, 1279 CTX_SIG_EVENTS_INDIRECT_INIT); 1280 } 1281 1282 return 0; 1283 } 1284 1285 static void 1286 qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae) 1287 { 1288 if (ae & 0x1) 1289 *nae = ae - 1; 1290 else 1291 *nae = ae + 1; 1292 } 1293 1294 static u_int 1295 qat_ae_ucode_parity64(uint64_t ucode) 1296 { 1297 1298 ucode ^= ucode >> 1; 1299 ucode ^= ucode >> 2; 1300 ucode ^= ucode >> 4; 1301 ucode ^= ucode >> 8; 1302 ucode ^= ucode >> 16; 1303 ucode ^= ucode >> 32; 1304 1305 return ((u_int)(ucode & 1)); 1306 } 1307 1308 static uint64_t 1309 qat_ae_ucode_set_ecc(uint64_t ucode) 1310 { 1311 static const uint64_t 1312 bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL, 1313 bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL, 1314 bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL, 1315 bit6mask=0xdaf69a46910ULL; 1316 1317 /* clear the ecc bits */ 1318 ucode &= ~(0x7fULL << USTORE_ECC_BIT_0); 1319 1320 ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) << 1321 USTORE_ECC_BIT_0; 1322 ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) << 1323 USTORE_ECC_BIT_1; 1324 ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) << 1325 USTORE_ECC_BIT_2; 1326 ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) << 1327 USTORE_ECC_BIT_3; 1328 ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) << 1329 USTORE_ECC_BIT_4; 1330 ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) << 1331 USTORE_ECC_BIT_5; 1332 ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) << 1333 USTORE_ECC_BIT_6; 1334 1335 return (ucode); 1336 } 1337 1338 static int 1339 qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, 1340 const uint64_t *ucode) 1341 { 1342 uint64_t tmp; 1343 uint32_t ustore_addr, ulo, uhi; 1344 int i; 1345 1346 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 1347 uaddr |= USTORE_ADDRESS_ECS; 1348 1349 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 1350 for (i = 0; i < ninst; i++) { 1351 tmp = qat_ae_ucode_set_ecc(ucode[i]); 1352 ulo = (uint32_t)(tmp & 0xffffffff); 1353 uhi = (uint32_t)(tmp >> 32); 1354 1355 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); 1356 /* this will auto increment the address */ 1357 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); 1358 } 1359 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 1360 1361 return 0; 1362 } 1363 1364 static int 1365 qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, 1366 uint64_t *ucode) 1367 { 1368 uint32_t misc, ustore_addr, ulo, uhi; 1369 u_int ii; 1370 u_char nae; 1371 1372 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) 1373 return EBUSY; 1374 1375 /* determine whether it neighbour AE runs in shared control store 1376 * status */ 1377 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1378 if (misc & AE_MISC_CONTROL_SHARE_CS) { 1379 qat_ae_get_shared_ustore_ae(ae, &nae); 1380 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) 1381 return EBUSY; 1382 } 1383 1384 /* if reloadable, then get it all from dram-ustore */ 1385 if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD)) 1386 panic("notyet"); /* XXX getReloadUwords */ 1387 1388 /* disable SHARE_CS bit to workaround silicon bug */ 1389 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb); 1390 1391 MPASS(uaddr + ninst <= USTORE_SIZE); 1392 1393 /* save ustore-addr csr */ 1394 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 1395 1396 uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */ 1397 for (ii = 0; ii < ninst; ii++) { 1398 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 1399 1400 uaddr++; 1401 qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo); 1402 qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi); 1403 ucode[ii] = uhi; 1404 ucode[ii] = (ucode[ii] << 32) | ulo; 1405 } 1406 1407 /* restore SHARE_CS bit to workaround silicon bug */ 1408 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 1409 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 1410 1411 return 0; 1412 } 1413 1414 static u_int 1415 qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr, 1416 u_int *value) 1417 { 1418 const uint64_t *inst_arr; 1419 u_int ninst0, curvalue; 1420 int ii, vali, fixup, usize = 0; 1421 1422 if (size == 0) 1423 return 0; 1424 1425 ninst0 = ninst; 1426 vali = 0; 1427 curvalue = value[vali++]; 1428 1429 switch (size) { 1430 case 0x1: 1431 inst_arr = ae_inst_1b; 1432 usize = nitems(ae_inst_1b); 1433 break; 1434 case 0x2: 1435 inst_arr = ae_inst_2b; 1436 usize = nitems(ae_inst_2b); 1437 break; 1438 case 0x3: 1439 inst_arr = ae_inst_3b; 1440 usize = nitems(ae_inst_3b); 1441 break; 1442 default: 1443 inst_arr = ae_inst_4b; 1444 usize = nitems(ae_inst_4b); 1445 break; 1446 } 1447 1448 fixup = ninst; 1449 for (ii = 0; ii < usize; ii++) 1450 ucode[ninst++] = inst_arr[ii]; 1451 1452 INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr)); 1453 fixup++; 1454 INSERT_IMMED_GPRA_CONST(ucode[fixup], 0); 1455 fixup++; 1456 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); 1457 fixup++; 1458 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); 1459 /* XXX fixup++ ? */ 1460 1461 if (size <= 0x4) 1462 return (ninst - ninst0); 1463 1464 size -= sizeof(u_int); 1465 while (size >= sizeof(u_int)) { 1466 curvalue = value[vali++]; 1467 fixup = ninst; 1468 ucode[ninst++] = ae_inst_4b[0x2]; 1469 ucode[ninst++] = ae_inst_4b[0x3]; 1470 ucode[ninst++] = ae_inst_4b[0x8]; 1471 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); 1472 fixup++; 1473 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); 1474 /* XXX fixup++ ? */ 1475 1476 addr += sizeof(u_int); 1477 size -= sizeof(u_int); 1478 } 1479 /* call this function recusive when the left size less than 4 */ 1480 ninst += 1481 qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali); 1482 1483 return (ninst - ninst0); 1484 } 1485 1486 static int 1487 qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx, 1488 uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles, 1489 u_int *endpc) 1490 { 1491 int error = 0, share_cs = 0; 1492 uint64_t savucode[MAX_EXEC_INST]; 1493 uint32_t indr_lm_addr_0, indr_lm_addr_1; 1494 uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1; 1495 uint32_t indr_future_cnt_sig; 1496 uint32_t indr_sig, active_sig; 1497 uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl; 1498 uint32_t misc, nmisc, ctxen; 1499 u_char nae; 1500 1501 MPASS(ninst <= USTORE_SIZE); 1502 1503 if (qat_ae_is_active(sc, ae)) 1504 return EBUSY; 1505 1506 /* save current LM addr */ 1507 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0); 1508 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1); 1509 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, 1510 &indr_lm_addr_byte_0); 1511 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, 1512 &indr_lm_addr_byte_1); 1513 1514 /* backup shared control store bit, and force AE to 1515 none-shared mode before executing ucode snippet */ 1516 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1517 if (misc & AE_MISC_CONTROL_SHARE_CS) { 1518 share_cs = 1; 1519 qat_ae_get_shared_ustore_ae(ae, &nae); 1520 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) 1521 return EBUSY; 1522 } 1523 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 1524 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 1525 1526 /* save current states: */ 1527 if (ninst <= MAX_EXEC_INST) { 1528 error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode); 1529 if (error) { 1530 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 1531 return error; 1532 } 1533 } 1534 1535 /* save wakeup-events */ 1536 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT, 1537 &wakeup_ev); 1538 /* save PC */ 1539 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc); 1540 savpc &= UPC_MASK; 1541 1542 /* save ctx enables */ 1543 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 1544 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 1545 /* save conditional-code */ 1546 qat_ae_read_4(sc, ae, CC_ENABLE, &savcc); 1547 /* save current context */ 1548 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); 1549 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); 1550 1551 /* save indirect csrs */ 1552 qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, 1553 &indr_future_cnt_sig); 1554 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig); 1555 qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig); 1556 1557 /* turn off ucode parity */ 1558 qat_ae_write_4(sc, ae, CTX_ENABLES, 1559 ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE); 1560 1561 /* copy instructions to ustore */ 1562 qat_ae_ucode_write(sc, ae, 0, ninst, ucode); 1563 /* set PC */ 1564 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0); 1565 /* change the active context */ 1566 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1567 ctx & ACTIVE_CTX_STATUS_ACNO); 1568 1569 if (cond_code_off) { 1570 /* disable conditional-code*/ 1571 qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff); 1572 } 1573 1574 /* wakeup-event voluntary */ 1575 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, 1576 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); 1577 1578 /* clean signals */ 1579 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0); 1580 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1581 1582 /* enable context */ 1583 qat_ae_enable_ctx(sc, ae, 1 << ctx); 1584 1585 /* wait for it to finish */ 1586 if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0) 1587 error = ETIMEDOUT; 1588 1589 /* see if we need to get the current PC */ 1590 if (endpc != NULL) { 1591 uint32_t ctx_status; 1592 1593 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, 1594 &ctx_status); 1595 *endpc = ctx_status & UPC_MASK; 1596 } 1597 #if 0 1598 { 1599 uint32_t ctx_status; 1600 1601 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, 1602 &ctx_status); 1603 printf("%s: endpc 0x%08x\n", __func__, 1604 ctx_status & UPC_MASK); 1605 } 1606 #endif 1607 1608 /* retore to previous states: */ 1609 /* disable context */ 1610 qat_ae_disable_ctx(sc, ae, 1 << ctx); 1611 if (ninst <= MAX_EXEC_INST) { 1612 /* instructions */ 1613 qat_ae_ucode_write(sc, ae, 0, ninst, savucode); 1614 } 1615 /* wakeup-events */ 1616 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT, 1617 wakeup_ev); 1618 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc); 1619 1620 /* only restore shared control store bit, 1621 other bit might be changed by AE code snippet */ 1622 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1623 if (share_cs) 1624 nmisc = misc | AE_MISC_CONTROL_SHARE_CS; 1625 else 1626 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 1627 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 1628 /* conditional-code */ 1629 qat_ae_write_4(sc, ae, CC_ENABLE, savcc); 1630 /* change the active context */ 1631 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1632 savctx & ACTIVE_CTX_STATUS_ACNO); 1633 /* restore the nxt ctx to run */ 1634 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 1635 /* restore current LM addr */ 1636 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT, 1637 indr_lm_addr_0); 1638 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT, 1639 indr_lm_addr_1); 1640 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, 1641 indr_lm_addr_byte_0); 1642 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, 1643 indr_lm_addr_byte_1); 1644 1645 /* restore indirect csrs */ 1646 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT, 1647 indr_future_cnt_sig); 1648 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 1649 indr_sig); 1650 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig); 1651 1652 /* ctx-enables */ 1653 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 1654 1655 return error; 1656 } 1657 1658 static int 1659 qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx, 1660 int *first_exec, uint64_t *ucode, u_int ninst, 1661 u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1) 1662 { 1663 1664 if (*first_exec) { 1665 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); 1666 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); 1667 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); 1668 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); 1669 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); 1670 *first_exec = 0; 1671 } 1672 1673 return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL); 1674 } 1675 1676 static int 1677 qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx, 1678 u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1) 1679 { 1680 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); 1681 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); 1682 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); 1683 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); 1684 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); 1685 1686 return 0; 1687 } 1688 1689 static int 1690 qat_ae_get_inst_num(int lmsize) 1691 { 1692 int ninst, left; 1693 1694 if (lmsize == 0) 1695 return 0; 1696 1697 left = lmsize % sizeof(u_int); 1698 1699 if (left) { 1700 ninst = nitems(ae_inst_1b) + 1701 qat_ae_get_inst_num(lmsize - left); 1702 } else { 1703 /* 3 instruction is needed for further code */ 1704 ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b); 1705 } 1706 1707 return (ninst); 1708 } 1709 1710 static int 1711 qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae, 1712 struct qat_ae_batch_init_list *qabi_list, size_t nqabi) 1713 { 1714 struct qat_ae_batch_init *qabi; 1715 size_t alloc_ninst, ninst; 1716 uint64_t *ucode; 1717 u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1; 1718 int insnsz, error = 0, execed = 0, first_exec = 1; 1719 1720 if (STAILQ_FIRST(qabi_list) == NULL) 1721 return 0; 1722 1723 alloc_ninst = min(USTORE_SIZE, nqabi); 1724 ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst); 1725 1726 ninst = 0; 1727 STAILQ_FOREACH(qabi, qabi_list, qabi_next) { 1728 insnsz = qat_ae_get_inst_num(qabi->qabi_size); 1729 if (insnsz + ninst > alloc_ninst) { 1730 /* add ctx_arb[kill] */ 1731 ucode[ninst++] = 0x0E000010000ull; 1732 execed = 1; 1733 1734 error = qat_ae_exec_ucode_init_lm(sc, ae, 0, 1735 &first_exec, ucode, ninst, 1736 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); 1737 if (error) { 1738 qat_ae_restore_init_lm_gprs(sc, ae, 0, 1739 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); 1740 qat_free_mem(ucode); 1741 return error; 1742 } 1743 /* run microExec to execute the microcode */ 1744 ninst = 0; 1745 } 1746 ninst += qat_ae_concat_ucode(ucode, ninst, 1747 qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value); 1748 } 1749 1750 if (ninst > 0) { 1751 ucode[ninst++] = 0x0E000010000ull; 1752 execed = 1; 1753 1754 error = qat_ae_exec_ucode_init_lm(sc, ae, 0, 1755 &first_exec, ucode, ninst, 1756 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); 1757 } 1758 if (execed) { 1759 qat_ae_restore_init_lm_gprs(sc, ae, 0, 1760 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); 1761 } 1762 1763 qat_free_mem(ucode); 1764 1765 return error; 1766 } 1767 1768 static int 1769 qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc) 1770 { 1771 1772 if (qat_ae_is_active(sc, ae)) 1773 return EBUSY; 1774 1775 qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT, 1776 UPC_MASK & upc); 1777 return 0; 1778 } 1779 1780 static inline u_int 1781 qat_aefw_csum_calc(u_int reg, int ch) 1782 { 1783 int i; 1784 u_int topbit = CRC_BITMASK(CRC_WIDTH - 1); 1785 u_int inbyte = (u_int)((reg >> 0x18) ^ ch); 1786 1787 reg ^= inbyte << (CRC_WIDTH - 0x8); 1788 for (i = 0; i < 0x8; i++) { 1789 if (reg & topbit) 1790 reg = (reg << 1) ^ CRC_POLY; 1791 else 1792 reg <<= 1; 1793 } 1794 1795 return (reg & CRC_WIDTHMASK(CRC_WIDTH)); 1796 } 1797 1798 static u_int 1799 qat_aefw_csum(char *buf, int size) 1800 { 1801 u_int csum = 0; 1802 1803 while (size--) { 1804 csum = qat_aefw_csum_calc(csum, *buf++); 1805 } 1806 1807 return csum; 1808 } 1809 1810 static const char * 1811 qat_aefw_uof_string(struct qat_softc *sc, size_t offset) 1812 { 1813 if (offset >= sc->sc_aefw_uof.qafu_str_tab_size) 1814 return NULL; 1815 if (sc->sc_aefw_uof.qafu_str_tab == NULL) 1816 return NULL; 1817 1818 return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset); 1819 } 1820 1821 static struct uof_chunk_hdr * 1822 qat_aefw_uof_find_chunk(struct qat_softc *sc, 1823 const char *id, struct uof_chunk_hdr *cur) 1824 { 1825 struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr; 1826 struct uof_chunk_hdr *uch; 1827 int i; 1828 1829 uch = (struct uof_chunk_hdr *)(uoh + 1); 1830 for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) { 1831 if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size) 1832 return NULL; 1833 1834 if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN)) 1835 return uch; 1836 } 1837 1838 return NULL; 1839 } 1840 1841 static int 1842 qat_aefw_load_mof(struct qat_softc *sc) 1843 { 1844 const struct firmware *fw; 1845 1846 fw = firmware_get(sc->sc_hw.qhw_mof_fwname); 1847 if (fw == NULL) { 1848 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", 1849 sc->sc_hw.qhw_mof_fwname); 1850 return ENXIO; 1851 } 1852 1853 sc->sc_fw_mof = qat_alloc_mem(fw->datasize); 1854 sc->sc_fw_mof_size = fw->datasize; 1855 memcpy(sc->sc_fw_mof, fw->data, fw->datasize); 1856 firmware_put(fw, FIRMWARE_UNLOAD); 1857 return 0; 1858 } 1859 1860 static void 1861 qat_aefw_unload_mof(struct qat_softc *sc) 1862 { 1863 if (sc->sc_fw_mof != NULL) { 1864 qat_free_mem(sc->sc_fw_mof); 1865 sc->sc_fw_mof = NULL; 1866 } 1867 } 1868 1869 static int 1870 qat_aefw_load_mmp(struct qat_softc *sc) 1871 { 1872 const struct firmware *fw; 1873 1874 fw = firmware_get(sc->sc_hw.qhw_mmp_fwname); 1875 if (fw == NULL) { 1876 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", 1877 sc->sc_hw.qhw_mmp_fwname); 1878 return ENXIO; 1879 } 1880 1881 sc->sc_fw_mmp = qat_alloc_mem(fw->datasize); 1882 sc->sc_fw_mmp_size = fw->datasize; 1883 memcpy(sc->sc_fw_mmp, fw->data, fw->datasize); 1884 firmware_put(fw, FIRMWARE_UNLOAD); 1885 return 0; 1886 } 1887 1888 static void 1889 qat_aefw_unload_mmp(struct qat_softc *sc) 1890 { 1891 if (sc->sc_fw_mmp != NULL) { 1892 qat_free_mem(sc->sc_fw_mmp); 1893 sc->sc_fw_mmp = NULL; 1894 } 1895 } 1896 1897 static int 1898 qat_aefw_mof_find_uof0(struct qat_softc *sc, 1899 struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head, 1900 u_int nchunk, size_t size, const char *id, 1901 size_t *fwsize, void **fwptr) 1902 { 1903 int i; 1904 char *uof_name; 1905 1906 for (i = 0; i < nchunk; i++) { 1907 struct mof_uof_chunk_hdr *much = &head[i]; 1908 1909 if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN)) 1910 return EINVAL; 1911 1912 if (much->much_offset + much->much_size > size) 1913 return EINVAL; 1914 1915 if (sc->sc_mof.qmf_sym_size <= much->much_name) 1916 return EINVAL; 1917 1918 uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym + 1919 much->much_name); 1920 1921 if (!strcmp(uof_name, sc->sc_fw_uof_name)) { 1922 *fwptr = (void *)((uintptr_t)muh + 1923 (uintptr_t)much->much_offset); 1924 *fwsize = (size_t)much->much_size; 1925 return 0; 1926 } 1927 } 1928 1929 return ENOENT; 1930 } 1931 1932 static int 1933 qat_aefw_mof_find_uof(struct qat_softc *sc) 1934 { 1935 struct mof_uof_hdr *uof_hdr, *suof_hdr; 1936 u_int nuof_chunks = 0, nsuof_chunks = 0; 1937 int error; 1938 1939 uof_hdr = sc->sc_mof.qmf_uof_objs; 1940 suof_hdr = sc->sc_mof.qmf_suof_objs; 1941 1942 if (uof_hdr != NULL) { 1943 if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) { 1944 return EINVAL; 1945 } 1946 nuof_chunks = uof_hdr->muh_num_chunks; 1947 } 1948 if (suof_hdr != NULL) { 1949 if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks) 1950 return EINVAL; 1951 nsuof_chunks = suof_hdr->muh_num_chunks; 1952 } 1953 1954 if (nuof_chunks + nsuof_chunks == 0) 1955 return EINVAL; 1956 1957 if (uof_hdr != NULL) { 1958 error = qat_aefw_mof_find_uof0(sc, uof_hdr, 1959 (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks, 1960 sc->sc_mof.qmf_uof_objs_size, UOF_IMAG, 1961 &sc->sc_fw_uof_size, &sc->sc_fw_uof); 1962 if (error && error != ENOENT) 1963 return error; 1964 } 1965 1966 if (suof_hdr != NULL) { 1967 error = qat_aefw_mof_find_uof0(sc, suof_hdr, 1968 (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks, 1969 sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG, 1970 &sc->sc_fw_suof_size, &sc->sc_fw_suof); 1971 if (error && error != ENOENT) 1972 return error; 1973 } 1974 1975 if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL) 1976 return ENOENT; 1977 1978 return 0; 1979 } 1980 1981 static int 1982 qat_aefw_mof_parse(struct qat_softc *sc) 1983 { 1984 const struct mof_file_hdr *mfh; 1985 const struct mof_file_chunk_hdr *mfch; 1986 size_t size; 1987 u_int csum; 1988 int error, i; 1989 1990 size = sc->sc_fw_mof_size; 1991 1992 if (size < sizeof(struct mof_file_hdr)) 1993 return EINVAL; 1994 size -= sizeof(struct mof_file_hdr); 1995 1996 mfh = sc->sc_fw_mof; 1997 1998 if (mfh->mfh_fid != MOF_FID) 1999 return EINVAL; 2000 2001 csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof + 2002 offsetof(struct mof_file_hdr, mfh_min_ver)), 2003 sc->sc_fw_mof_size - 2004 offsetof(struct mof_file_hdr, mfh_min_ver)); 2005 if (mfh->mfh_csum != csum) 2006 return EINVAL; 2007 2008 if (mfh->mfh_min_ver != MOF_MIN_VER || 2009 mfh->mfh_maj_ver != MOF_MAJ_VER) 2010 return EINVAL; 2011 2012 if (mfh->mfh_max_chunks < mfh->mfh_num_chunks) 2013 return EINVAL; 2014 2015 if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks) 2016 return EINVAL; 2017 mfch = (const struct mof_file_chunk_hdr *)(mfh + 1); 2018 2019 for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) { 2020 if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size) 2021 return EINVAL; 2022 2023 if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) { 2024 if (sc->sc_mof.qmf_sym != NULL) 2025 return EINVAL; 2026 2027 sc->sc_mof.qmf_sym = 2028 (void *)((uintptr_t)sc->sc_fw_mof + 2029 (uintptr_t)mfch->mfch_offset + sizeof(u_int)); 2030 sc->sc_mof.qmf_sym_size = 2031 *(u_int *)((uintptr_t)sc->sc_fw_mof + 2032 (uintptr_t)mfch->mfch_offset); 2033 2034 if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0) 2035 return EINVAL; 2036 if (mfch->mfch_size != sc->sc_mof.qmf_sym_size + 2037 sizeof(u_int) || mfch->mfch_size == 0) 2038 return EINVAL; 2039 if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym + 2040 sc->sc_mof.qmf_sym_size - 1) != '\0') 2041 return EINVAL; 2042 2043 } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) { 2044 if (sc->sc_mof.qmf_uof_objs != NULL) 2045 return EINVAL; 2046 2047 sc->sc_mof.qmf_uof_objs = 2048 (void *)((uintptr_t)sc->sc_fw_mof + 2049 (uintptr_t)mfch->mfch_offset); 2050 sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size; 2051 2052 } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) { 2053 if (sc->sc_mof.qmf_suof_objs != NULL) 2054 return EINVAL; 2055 2056 sc->sc_mof.qmf_suof_objs = 2057 (void *)((uintptr_t)sc->sc_fw_mof + 2058 (uintptr_t)mfch->mfch_offset); 2059 sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size; 2060 } 2061 } 2062 2063 if (sc->sc_mof.qmf_sym == NULL || 2064 (sc->sc_mof.qmf_uof_objs == NULL && 2065 sc->sc_mof.qmf_suof_objs == NULL)) 2066 return EINVAL; 2067 2068 error = qat_aefw_mof_find_uof(sc); 2069 if (error) 2070 return error; 2071 return 0; 2072 } 2073 2074 static int 2075 qat_aefw_uof_parse_image(struct qat_softc *sc, 2076 struct qat_uof_image *qui, struct uof_chunk_hdr *uch) 2077 { 2078 struct uof_image *image; 2079 struct uof_code_page *page; 2080 uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; 2081 size_t lim = uch->uch_offset + uch->uch_size, size; 2082 int i, p; 2083 2084 size = uch->uch_size; 2085 if (size < sizeof(struct uof_image)) 2086 return EINVAL; 2087 size -= sizeof(struct uof_image); 2088 2089 qui->qui_image = image = 2090 (struct uof_image *)(base + uch->uch_offset); 2091 2092 #define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \ 2093 do { \ 2094 u_int nent; \ 2095 nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\ 2096 if ((lim) < off + sizeof(struct uof_obj_table) + \ 2097 sizeof(type) * nent) \ 2098 return EINVAL; \ 2099 *(np) = nent; \ 2100 if (nent > 0) \ 2101 *(typep) = (type)((struct uof_obj_table *) \ 2102 ((base) + (off)) + 1); \ 2103 else \ 2104 *(typep) = NULL; \ 2105 } while (0) 2106 2107 ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg, 2108 struct uof_ae_reg *, base, image->ui_reg_tab, lim); 2109 ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym, 2110 struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim); 2111 ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak, 2112 struct qui_sbreak *, base, image->ui_sbreak_tab, lim); 2113 2114 if (size < sizeof(struct uof_code_page) * image->ui_num_pages) 2115 return EINVAL; 2116 if (nitems(qui->qui_pages) < image->ui_num_pages) 2117 return EINVAL; 2118 2119 page = (struct uof_code_page *)(image + 1); 2120 2121 for (p = 0; p < image->ui_num_pages; p++, page++) { 2122 struct qat_uof_page *qup = &qui->qui_pages[p]; 2123 struct uof_code_area *uca; 2124 2125 qup->qup_page_num = page->ucp_page_num; 2126 qup->qup_def_page = page->ucp_def_page; 2127 qup->qup_page_region = page->ucp_page_region; 2128 qup->qup_beg_vaddr = page->ucp_beg_vaddr; 2129 qup->qup_beg_paddr = page->ucp_beg_paddr; 2130 2131 ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var, 2132 struct uof_uword_fixup *, base, 2133 page->ucp_uc_var_tab, lim); 2134 ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var, 2135 struct uof_import_var *, base, 2136 page->ucp_imp_var_tab, lim); 2137 ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr, 2138 struct uof_uword_fixup *, base, 2139 page->ucp_imp_expr_tab, lim); 2140 ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg, 2141 struct uof_uword_fixup *, base, 2142 page->ucp_neigh_reg_tab, lim); 2143 2144 if (lim < page->ucp_code_area + sizeof(struct uof_code_area)) 2145 return EINVAL; 2146 2147 uca = (struct uof_code_area *)(base + page->ucp_code_area); 2148 qup->qup_num_micro_words = uca->uca_num_micro_words; 2149 2150 ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks, 2151 struct qat_uof_uword_block *, base, 2152 uca->uca_uword_block_tab, lim); 2153 2154 for (i = 0; i < qup->qup_num_uw_blocks; i++) { 2155 u_int uwordoff = ((struct uof_uword_block *)( 2156 &qup->qup_uw_blocks[i]))->uub_uword_offset; 2157 2158 if (lim < uwordoff) 2159 return EINVAL; 2160 2161 qup->qup_uw_blocks[i].quub_micro_words = 2162 (base + uwordoff); 2163 } 2164 } 2165 2166 #undef ASSIGN_OBJ_TAB 2167 2168 return 0; 2169 } 2170 2171 static int 2172 qat_aefw_uof_parse_images(struct qat_softc *sc) 2173 { 2174 struct uof_chunk_hdr *uch = NULL; 2175 int i, error; 2176 2177 for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) { 2178 uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch); 2179 if (uch == NULL) 2180 break; 2181 2182 if (i >= nitems(sc->sc_aefw_uof.qafu_imgs)) 2183 return ENOENT; 2184 2185 error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch); 2186 if (error) 2187 return error; 2188 2189 sc->sc_aefw_uof.qafu_num_imgs++; 2190 } 2191 2192 return 0; 2193 } 2194 2195 static int 2196 qat_aefw_uof_parse(struct qat_softc *sc) 2197 { 2198 struct uof_file_hdr *ufh; 2199 struct uof_file_chunk_hdr *ufch; 2200 struct uof_obj_hdr *uoh; 2201 struct uof_chunk_hdr *uch; 2202 void *uof = NULL; 2203 size_t size, uof_size, hdr_size; 2204 uintptr_t base; 2205 u_int csum; 2206 int i; 2207 2208 size = sc->sc_fw_uof_size; 2209 if (size < MIN_UOF_SIZE) 2210 return EINVAL; 2211 size -= sizeof(struct uof_file_hdr); 2212 2213 ufh = sc->sc_fw_uof; 2214 2215 if (ufh->ufh_id != UOF_FID) 2216 return EINVAL; 2217 if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER) 2218 return EINVAL; 2219 2220 if (ufh->ufh_max_chunks < ufh->ufh_num_chunks) 2221 return EINVAL; 2222 if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks) 2223 return EINVAL; 2224 ufch = (struct uof_file_chunk_hdr *)(ufh + 1); 2225 2226 uof_size = 0; 2227 for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) { 2228 if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size) 2229 return EINVAL; 2230 2231 if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) { 2232 if (uof != NULL) 2233 return EINVAL; 2234 2235 uof = 2236 (void *)((uintptr_t)sc->sc_fw_uof + 2237 ufch->ufch_offset); 2238 uof_size = ufch->ufch_size; 2239 2240 csum = qat_aefw_csum(uof, uof_size); 2241 if (csum != ufch->ufch_csum) 2242 return EINVAL; 2243 } 2244 } 2245 2246 if (uof == NULL) 2247 return ENOENT; 2248 2249 size = uof_size; 2250 if (size < sizeof(struct uof_obj_hdr)) 2251 return EINVAL; 2252 size -= sizeof(struct uof_obj_hdr); 2253 2254 uoh = uof; 2255 2256 if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks) 2257 return EINVAL; 2258 2259 /* Check if the UOF objects are compatible with the chip */ 2260 if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0) 2261 return ENOTSUP; 2262 2263 if (uoh->uoh_min_cpu_ver > sc->sc_rev || 2264 uoh->uoh_max_cpu_ver < sc->sc_rev) 2265 return ENOTSUP; 2266 2267 sc->sc_aefw_uof.qafu_size = uof_size; 2268 sc->sc_aefw_uof.qafu_obj_hdr = uoh; 2269 2270 base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; 2271 2272 /* map uof string-table */ 2273 uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL); 2274 if (uch != NULL) { 2275 hdr_size = offsetof(struct uof_str_tab, ust_strings); 2276 sc->sc_aefw_uof.qafu_str_tab = 2277 (void *)(base + uch->uch_offset + hdr_size); 2278 sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size; 2279 } 2280 2281 /* get ustore mem inits table -- should be only one */ 2282 uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL); 2283 if (uch != NULL) { 2284 if (uch->uch_size < sizeof(struct uof_obj_table)) 2285 return EINVAL; 2286 sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base + 2287 uch->uch_offset))->uot_nentries; 2288 if (sc->sc_aefw_uof.qafu_num_init_mem) { 2289 sc->sc_aefw_uof.qafu_init_mem = 2290 (struct uof_init_mem *)(base + uch->uch_offset + 2291 sizeof(struct uof_obj_table)); 2292 sc->sc_aefw_uof.qafu_init_mem_size = 2293 uch->uch_size - sizeof(struct uof_obj_table); 2294 } 2295 } 2296 2297 uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL); 2298 if (uch != NULL) { 2299 if (uch->uch_size < sizeof(struct uof_obj_table) + 2300 sizeof(struct uof_var_mem_seg)) 2301 return EINVAL; 2302 sc->sc_aefw_uof.qafu_var_mem_seg = 2303 (struct uof_var_mem_seg *)(base + uch->uch_offset + 2304 sizeof(struct uof_obj_table)); 2305 } 2306 2307 return qat_aefw_uof_parse_images(sc); 2308 } 2309 2310 static int 2311 qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi, 2312 struct suof_chunk_hdr *sch) 2313 { 2314 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; 2315 struct simg_ae_mode *ae_mode; 2316 u_int maj_ver; 2317 2318 qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset + 2319 sizeof(struct suof_obj_hdr); 2320 qsi->qsi_simg_len = 2321 ((struct suof_obj_hdr *) 2322 (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length; 2323 2324 qsi->qsi_css_header = qsi->qsi_simg_buf; 2325 qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr); 2326 qsi->qsi_css_signature = qsi->qsi_css_key + 2327 CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; 2328 qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN; 2329 2330 ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg; 2331 qsi->qsi_ae_mask = ae_mode->sam_ae_mask; 2332 qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name; 2333 qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data; 2334 qsi->qsi_fw_type = ae_mode->sam_fw_type; 2335 2336 if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type) 2337 return EINVAL; 2338 2339 maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff; 2340 if ((maj_ver > ae_mode->sam_devmax_ver) || 2341 (maj_ver < ae_mode->sam_devmin_ver)) { 2342 return EINVAL; 2343 } 2344 2345 return 0; 2346 } 2347 2348 static int 2349 qat_aefw_suof_parse(struct qat_softc *sc) 2350 { 2351 struct suof_file_hdr *sfh; 2352 struct suof_chunk_hdr *sch; 2353 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; 2354 struct qat_suof_image *qsi; 2355 size_t size; 2356 u_int csum; 2357 int ae0_img = MAX_AE; 2358 int i, error; 2359 2360 size = sc->sc_fw_suof_size; 2361 if (size < sizeof(struct suof_file_hdr)) 2362 return EINVAL; 2363 2364 sfh = sc->sc_fw_suof; 2365 2366 if (sfh->sfh_file_id != SUOF_FID) 2367 return EINVAL; 2368 if (sfh->sfh_fw_type != 0) 2369 return EINVAL; 2370 if (sfh->sfh_num_chunks <= 1) 2371 return EINVAL; 2372 if (sfh->sfh_min_ver != SUOF_MIN_VER || 2373 sfh->sfh_maj_ver != SUOF_MAJ_VER) 2374 return EINVAL; 2375 2376 csum = qat_aefw_csum((char *)&sfh->sfh_min_ver, 2377 size - offsetof(struct suof_file_hdr, sfh_min_ver)); 2378 if (csum != sfh->sfh_check_sum) 2379 return EINVAL; 2380 2381 size -= sizeof(struct suof_file_hdr); 2382 2383 qafs->qafs_file_id = SUOF_FID; 2384 qafs->qafs_suof_buf = sc->sc_fw_suof; 2385 qafs->qafs_suof_size = sc->sc_fw_suof_size; 2386 qafs->qafs_check_sum = sfh->sfh_check_sum; 2387 qafs->qafs_min_ver = sfh->sfh_min_ver; 2388 qafs->qafs_maj_ver = sfh->sfh_maj_ver; 2389 qafs->qafs_fw_type = sfh->sfh_fw_type; 2390 2391 if (size < sizeof(struct suof_chunk_hdr)) 2392 return EINVAL; 2393 sch = (struct suof_chunk_hdr *)(sfh + 1); 2394 size -= sizeof(struct suof_chunk_hdr); 2395 2396 if (size < sizeof(struct suof_str_tab)) 2397 return EINVAL; 2398 size -= offsetof(struct suof_str_tab, sst_strings); 2399 2400 qafs->qafs_sym_size = ((struct suof_str_tab *) 2401 (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length; 2402 if (size < qafs->qafs_sym_size) 2403 return EINVAL; 2404 qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset + 2405 offsetof(struct suof_str_tab, sst_strings); 2406 2407 qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1; 2408 if (qafs->qafs_num_simgs == 0) 2409 return EINVAL; 2410 2411 qsi = qat_alloc_mem( 2412 sizeof(struct qat_suof_image) * qafs->qafs_num_simgs); 2413 qafs->qafs_simg = qsi; 2414 2415 for (i = 0; i < qafs->qafs_num_simgs; i++) { 2416 error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]); 2417 if (error) 2418 return error; 2419 if ((qsi[i].qsi_ae_mask & 0x1) != 0) 2420 ae0_img = i; 2421 } 2422 2423 if (ae0_img != qafs->qafs_num_simgs - 1) { 2424 struct qat_suof_image last_qsi; 2425 2426 memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1], 2427 sizeof(struct qat_suof_image)); 2428 memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img], 2429 sizeof(struct qat_suof_image)); 2430 memcpy(&qsi[ae0_img], &last_qsi, 2431 sizeof(struct qat_suof_image)); 2432 } 2433 2434 return 0; 2435 } 2436 2437 static int 2438 qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size, 2439 struct qat_dmamem *dma) 2440 { 2441 struct css_hdr *css = (struct css_hdr *)image; 2442 struct auth_chunk *auth_chunk; 2443 struct fw_auth_desc *auth_desc; 2444 size_t mapsize, simg_offset = sizeof(struct auth_chunk); 2445 bus_size_t bus_addr; 2446 uintptr_t virt_addr; 2447 int error; 2448 2449 if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN) 2450 return EINVAL; 2451 2452 mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ? 2453 CSS_AE_SIMG_LEN + simg_offset : 2454 size + CSS_FWSK_PAD_LEN + simg_offset; 2455 error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE); 2456 if (error) 2457 return error; 2458 2459 memset(dma->qdm_dma_vaddr, 0, mapsize); 2460 2461 auth_chunk = dma->qdm_dma_vaddr; 2462 auth_chunk->ac_chunk_size = mapsize; 2463 auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr; 2464 2465 virt_addr = (uintptr_t)dma->qdm_dma_vaddr; 2466 virt_addr += simg_offset; 2467 bus_addr = auth_chunk->ac_chunk_bus_addr; 2468 bus_addr += simg_offset; 2469 2470 auth_desc = &auth_chunk->ac_fw_auth_desc; 2471 auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32; 2472 auth_desc->fad_css_hdr_low = bus_addr; 2473 2474 memcpy((void *)virt_addr, image, sizeof(struct css_hdr)); 2475 /* pub key */ 2476 virt_addr += sizeof(struct css_hdr); 2477 bus_addr += sizeof(struct css_hdr); 2478 image += sizeof(struct css_hdr); 2479 2480 auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32; 2481 auth_desc->fad_fwsk_pub_low = bus_addr; 2482 2483 memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN); 2484 memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN); 2485 memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN), 2486 image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t)); 2487 2488 virt_addr += CSS_FWSK_PUB_LEN; 2489 bus_addr += CSS_FWSK_PUB_LEN; 2490 image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; 2491 2492 auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32; 2493 auth_desc->fad_signature_low = bus_addr; 2494 2495 memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN); 2496 2497 virt_addr += CSS_SIGNATURE_LEN; 2498 bus_addr += CSS_SIGNATURE_LEN; 2499 image += CSS_SIGNATURE_LEN; 2500 2501 auth_desc->fad_img_high = (uint64_t)bus_addr >> 32; 2502 auth_desc->fad_img_low = bus_addr; 2503 auth_desc->fad_img_len = size - AE_IMG_OFFSET; 2504 2505 memcpy((void *)virt_addr, image, auth_desc->fad_img_len); 2506 2507 if (css->css_fw_type == CSS_AE_FIRMWARE) { 2508 auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high; 2509 auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low; 2510 2511 bus_addr += sizeof(struct simg_ae_mode); 2512 2513 auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32; 2514 auth_desc->fad_img_ae_init_data_low = bus_addr; 2515 2516 bus_addr += SIMG_AE_INIT_SEQ_LEN; 2517 2518 auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32; 2519 auth_desc->fad_img_ae_insts_low = bus_addr; 2520 } else { 2521 auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high; 2522 auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low; 2523 } 2524 2525 bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map, 2526 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2527 2528 return 0; 2529 } 2530 2531 static int 2532 qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma) 2533 { 2534 bus_addr_t addr; 2535 uint32_t fcu, sts; 2536 int retry = 0; 2537 2538 addr = dma->qdm_dma_seg.ds_addr; 2539 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32); 2540 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr); 2541 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH); 2542 2543 do { 2544 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2545 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2546 sts = __SHIFTOUT(fcu, FCU_STATUS_STS); 2547 if (sts == FCU_STATUS_STS_VERI_FAIL) 2548 goto fail; 2549 if (fcu & FCU_STATUS_AUTHFWLD && 2550 sts == FCU_STATUS_STS_VERI_DONE) { 2551 return 0; 2552 } 2553 } while (retry++ < FW_AUTH_MAX_RETRY); 2554 2555 fail: 2556 device_printf(sc->sc_dev, 2557 "firmware authentication error: status 0x%08x retry %d\n", 2558 fcu, retry); 2559 return EINVAL; 2560 } 2561 2562 static int 2563 qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma) 2564 { 2565 struct simg_ae_mode *ae_mode; 2566 uint32_t fcu, sts, loaded; 2567 u_int mask; 2568 u_char ae; 2569 int retry = 0; 2570 2571 ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr + 2572 sizeof(struct auth_chunk) + sizeof(struct css_hdr) + 2573 CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN); 2574 2575 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 2576 if (!(mask & 1)) 2577 continue; 2578 if (!((ae_mode->sam_ae_mask >> ae) & 0x1)) 2579 continue; 2580 if (qat_ae_is_active(sc, ae)) { 2581 device_printf(sc->sc_dev, "AE %d is active\n", ae); 2582 return EINVAL; 2583 } 2584 qat_cap_global_write_4(sc, FCU_CTRL, 2585 FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE)); 2586 do { 2587 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2588 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2589 sts = __SHIFTOUT(fcu, FCU_STATUS_STS); 2590 loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE); 2591 if (sts == FCU_STATUS_STS_LOAD_DONE && 2592 (loaded & (1 << ae))) { 2593 break; 2594 } 2595 } while (retry++ < FW_AUTH_MAX_RETRY); 2596 2597 if (retry > FW_AUTH_MAX_RETRY) { 2598 device_printf(sc->sc_dev, 2599 "firmware load timeout: status %08x\n", fcu); 2600 return EINVAL; 2601 } 2602 } 2603 2604 return 0; 2605 } 2606 2607 static int 2608 qat_aefw_suof_write(struct qat_softc *sc) 2609 { 2610 struct qat_suof_image *qsi; 2611 int i, error = 0; 2612 2613 for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) { 2614 qsi = &sc->sc_aefw_suof.qafs_simg[i]; 2615 error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf, 2616 qsi->qsi_simg_len, &qsi->qsi_dma); 2617 if (error) 2618 return error; 2619 error = qat_aefw_auth(sc, &qsi->qsi_dma); 2620 if (error) { 2621 qat_free_dmamem(sc, &qsi->qsi_dma); 2622 return error; 2623 } 2624 error = qat_aefw_suof_load(sc, &qsi->qsi_dma); 2625 if (error) { 2626 qat_free_dmamem(sc, &qsi->qsi_dma); 2627 return error; 2628 } 2629 qat_free_dmamem(sc, &qsi->qsi_dma); 2630 } 2631 qat_free_mem(sc->sc_aefw_suof.qafs_simg); 2632 2633 return 0; 2634 } 2635 2636 static int 2637 qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae, 2638 struct qat_uof_image *qui) 2639 { 2640 struct qat_ae_slice *slice; 2641 int i, npages, nregions; 2642 2643 if (qae->qae_num_slices >= nitems(qae->qae_slices)) 2644 return ENOENT; 2645 2646 if (qui->qui_image->ui_ae_mode & 2647 (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) { 2648 /* XXX */ 2649 device_printf(sc->sc_dev, 2650 "shared ae mode is not supported yet\n"); 2651 return ENOTSUP; 2652 } 2653 2654 qae->qae_shareable_ustore = 0; /* XXX */ 2655 qae->qae_effect_ustore_size = USTORE_SIZE; 2656 2657 slice = &qae->qae_slices[qae->qae_num_slices]; 2658 2659 slice->qas_image = qui; 2660 slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned; 2661 2662 nregions = qui->qui_image->ui_num_page_regions; 2663 npages = qui->qui_image->ui_num_pages; 2664 2665 if (nregions > nitems(slice->qas_regions)) 2666 return ENOENT; 2667 if (npages > nitems(slice->qas_pages)) 2668 return ENOENT; 2669 2670 for (i = 0; i < nregions; i++) { 2671 STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages); 2672 } 2673 for (i = 0; i < npages; i++) { 2674 struct qat_ae_page *page = &slice->qas_pages[i]; 2675 int region; 2676 2677 page->qap_page = &qui->qui_pages[i]; 2678 region = page->qap_page->qup_page_region; 2679 if (region >= nregions) 2680 return EINVAL; 2681 2682 page->qap_region = &slice->qas_regions[region]; 2683 } 2684 2685 qae->qae_num_slices++; 2686 2687 return 0; 2688 } 2689 2690 static int 2691 qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae) 2692 { 2693 struct uof_image *image; 2694 struct qat_ae *qae = &(QAT_AE(sc, ae)); 2695 int s; 2696 u_char nn_mode; 2697 2698 for (s = 0; s < qae->qae_num_slices; s++) { 2699 if (qae->qae_slices[s].qas_image == NULL) 2700 continue; 2701 2702 image = qae->qae_slices[s].qas_image->qui_image; 2703 qat_ae_write_ctx_mode(sc, ae, 2704 __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE)); 2705 2706 nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE); 2707 if (nn_mode != AE_MODE_NN_MODE_DONTCARE) 2708 qat_ae_write_nn_mode(sc, ae, nn_mode); 2709 2710 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0, 2711 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0)); 2712 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1, 2713 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1)); 2714 2715 qat_ae_write_shared_cs_mode(sc, ae, 2716 __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE)); 2717 qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size, 2718 __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED), 2719 qae->qae_reloc_ustore_dram); 2720 } 2721 2722 return 0; 2723 } 2724 2725 static int 2726 qat_aefw_uof_init(struct qat_softc *sc) 2727 { 2728 int ae, i, error; 2729 uint32_t mask; 2730 2731 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 2732 struct qat_ae *qae; 2733 2734 if (!(mask & 1)) 2735 continue; 2736 2737 qae = &(QAT_AE(sc, ae)); 2738 2739 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 2740 if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned & 2741 (1 << ae)) == 0) 2742 continue; 2743 2744 error = qat_aefw_uof_assign_image(sc, qae, 2745 &sc->sc_aefw_uof.qafu_imgs[i]); 2746 if (error) 2747 return error; 2748 } 2749 2750 /* XXX UcLo_initNumUwordUsed */ 2751 2752 qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */ 2753 2754 error = qat_aefw_uof_init_ae(sc, ae); 2755 if (error) 2756 return error; 2757 } 2758 2759 return 0; 2760 } 2761 2762 int 2763 qat_aefw_load(struct qat_softc *sc) 2764 { 2765 int error; 2766 2767 error = qat_aefw_load_mof(sc); 2768 if (error) 2769 return error; 2770 2771 error = qat_aefw_load_mmp(sc); 2772 if (error) 2773 return error; 2774 2775 error = qat_aefw_mof_parse(sc); 2776 if (error) { 2777 device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error); 2778 return error; 2779 } 2780 2781 if (sc->sc_hw.qhw_fw_auth) { 2782 error = qat_aefw_suof_parse(sc); 2783 if (error) { 2784 device_printf(sc->sc_dev, "couldn't parse suof: %d\n", 2785 error); 2786 return error; 2787 } 2788 2789 error = qat_aefw_suof_write(sc); 2790 if (error) { 2791 device_printf(sc->sc_dev, 2792 "could not write firmware: %d\n", error); 2793 return error; 2794 } 2795 2796 } else { 2797 error = qat_aefw_uof_parse(sc); 2798 if (error) { 2799 device_printf(sc->sc_dev, "couldn't parse uof: %d\n", 2800 error); 2801 return error; 2802 } 2803 2804 error = qat_aefw_uof_init(sc); 2805 if (error) { 2806 device_printf(sc->sc_dev, 2807 "couldn't init for aefw: %d\n", error); 2808 return error; 2809 } 2810 2811 error = qat_aefw_uof_write(sc); 2812 if (error) { 2813 device_printf(sc->sc_dev, 2814 "Could not write firmware: %d\n", error); 2815 return error; 2816 } 2817 } 2818 2819 return 0; 2820 } 2821 2822 void 2823 qat_aefw_unload(struct qat_softc *sc) 2824 { 2825 qat_aefw_unload_mmp(sc); 2826 qat_aefw_unload_mof(sc); 2827 } 2828 2829 int 2830 qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask) 2831 { 2832 uint32_t fcu; 2833 int retry = 0; 2834 2835 if (sc->sc_hw.qhw_fw_auth) { 2836 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START); 2837 do { 2838 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2839 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2840 if (fcu & FCU_STATUS_DONE) 2841 return 0; 2842 } while (retry++ < FW_AUTH_MAX_RETRY); 2843 2844 device_printf(sc->sc_dev, 2845 "firmware start timeout: status %08x\n", fcu); 2846 return EINVAL; 2847 } else { 2848 qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX, 2849 CTX_WAKEUP_EVENTS_INDIRECT, 2850 CTX_WAKEUP_EVENTS_INDIRECT_SLEEP); 2851 qat_ae_enable_ctx(sc, ae, ctx_mask); 2852 } 2853 2854 return 0; 2855 } 2856 2857 static int 2858 qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim) 2859 { 2860 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 2861 struct qat_ae_batch_init_list *qabi_list; 2862 struct uof_mem_val_attr *memattr; 2863 size_t *curinit; 2864 u_long ael; 2865 int i; 2866 const char *sym; 2867 char *ep; 2868 2869 memattr = (struct uof_mem_val_attr *)(uim + 1); 2870 2871 switch (uim->uim_region) { 2872 case LMEM_REGION: 2873 if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) { 2874 device_printf(sc->sc_dev, 2875 "Invalid lmem addr or bytes\n"); 2876 return ENOBUFS; 2877 } 2878 if (uim->uim_scope != UOF_SCOPE_LOCAL) 2879 return EINVAL; 2880 sym = qat_aefw_uof_string(sc, uim->uim_sym_name); 2881 ael = strtoul(sym, &ep, 10); 2882 if (ep == sym || ael > MAX_AE) 2883 return EINVAL; 2884 if ((sc->sc_ae_mask & (1 << ael)) == 0) 2885 return 0; /* ae is fused out */ 2886 2887 curinit = &qafu->qafu_num_lm_init[ael]; 2888 qabi_list = &qafu->qafu_lm_init[ael]; 2889 2890 for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) { 2891 struct qat_ae_batch_init *qabi; 2892 2893 qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init)); 2894 if (*curinit == 0) 2895 STAILQ_INIT(qabi_list); 2896 STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next); 2897 2898 qabi->qabi_ae = (u_int)ael; 2899 qabi->qabi_addr = 2900 uim->uim_addr + memattr->umva_byte_offset; 2901 qabi->qabi_value = &memattr->umva_value; 2902 qabi->qabi_size = 4; 2903 qafu->qafu_num_lm_init_inst[ael] += 2904 qat_ae_get_inst_num(qabi->qabi_size); 2905 (*curinit)++; 2906 if (*curinit >= MAX_LMEM_REG) { 2907 device_printf(sc->sc_dev, 2908 "Invalid lmem val attr\n"); 2909 return ENOBUFS; 2910 } 2911 } 2912 break; 2913 case SRAM_REGION: 2914 case DRAM_REGION: 2915 case DRAM1_REGION: 2916 case SCRATCH_REGION: 2917 case UMEM_REGION: 2918 /* XXX */ 2919 /* fallthrough */ 2920 default: 2921 device_printf(sc->sc_dev, 2922 "unsupported memory region to init: %d\n", 2923 uim->uim_region); 2924 return ENOTSUP; 2925 } 2926 2927 return 0; 2928 } 2929 2930 static void 2931 qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae) 2932 { 2933 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 2934 struct qat_ae_batch_init *qabi; 2935 2936 while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) { 2937 STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next); 2938 qat_free_mem(qabi); 2939 } 2940 2941 qafu->qafu_num_lm_init[ae] = 0; 2942 qafu->qafu_num_lm_init_inst[ae] = 0; 2943 } 2944 2945 static int 2946 qat_aefw_init_ustore(struct qat_softc *sc) 2947 { 2948 uint64_t *fill; 2949 uint32_t dont_init; 2950 int a, i, p; 2951 int error = 0; 2952 int usz, end, start; 2953 u_char ae, nae; 2954 2955 fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t)); 2956 2957 for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) { 2958 struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a]; 2959 struct uof_image *ui = qui->qui_image; 2960 2961 for (i = 0; i < MAX_USTORE; i++) 2962 memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t)); 2963 /* 2964 * Compute do_not_init value as a value that will not be equal 2965 * to fill data when cast to an int 2966 */ 2967 dont_init = 0; 2968 if (dont_init == (uint32_t)fill[0]) 2969 dont_init = 0xffffffff; 2970 2971 for (p = 0; p < ui->ui_num_pages; p++) { 2972 struct qat_uof_page *qup = &qui->qui_pages[p]; 2973 if (!qup->qup_def_page) 2974 continue; 2975 2976 for (i = qup->qup_beg_paddr; 2977 i < qup->qup_beg_paddr + qup->qup_num_micro_words; 2978 i++ ) { 2979 fill[i] = (uint64_t)dont_init; 2980 } 2981 } 2982 2983 for (ae = 0; ae < sc->sc_ae_num; ae++) { 2984 MPASS(ae < UOF_MAX_NUM_OF_AE); 2985 if ((ui->ui_ae_assigned & (1 << ae)) == 0) 2986 continue; 2987 2988 if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) { 2989 qat_ae_get_shared_ustore_ae(ae, &nae); 2990 if (ui->ui_ae_assigned & (1 << ae)) 2991 continue; 2992 } 2993 usz = QAT_AE(sc, ae).qae_effect_ustore_size; 2994 2995 /* initialize the areas not going to be overwritten */ 2996 end = -1; 2997 do { 2998 /* find next uword that needs to be initialized */ 2999 for (start = end + 1; start < usz; start++) { 3000 if ((uint32_t)fill[start] != dont_init) 3001 break; 3002 } 3003 /* see if there are no more such uwords */ 3004 if (start >= usz) 3005 break; 3006 for (end = start + 1; end < usz; end++) { 3007 if ((uint32_t)fill[end] == dont_init) 3008 break; 3009 } 3010 if (QAT_AE(sc, ae).qae_shareable_ustore) { 3011 error = ENOTSUP; /* XXX */ 3012 goto out; 3013 } else { 3014 error = qat_ae_ucode_write(sc, ae, 3015 start, end - start, &fill[start]); 3016 if (error) { 3017 goto out; 3018 } 3019 } 3020 3021 } while (end < usz); 3022 } 3023 } 3024 3025 out: 3026 qat_free_mem(fill); 3027 return error; 3028 } 3029 3030 static int 3031 qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask, 3032 enum aereg_type regtype, u_short regaddr, u_int value) 3033 { 3034 int error = 0; 3035 u_char ctx; 3036 3037 switch (regtype) { 3038 case AEREG_GPA_REL: 3039 case AEREG_GPB_REL: 3040 case AEREG_SR_REL: 3041 case AEREG_SR_RD_REL: 3042 case AEREG_SR_WR_REL: 3043 case AEREG_DR_REL: 3044 case AEREG_DR_RD_REL: 3045 case AEREG_DR_WR_REL: 3046 case AEREG_NEIGH_REL: 3047 /* init for all valid ctx */ 3048 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { 3049 if ((ctx_mask & (1 << ctx)) == 0) 3050 continue; 3051 error = qat_aereg_rel_data_write(sc, ae, ctx, regtype, 3052 regaddr, value); 3053 } 3054 break; 3055 case AEREG_GPA_ABS: 3056 case AEREG_GPB_ABS: 3057 case AEREG_SR_ABS: 3058 case AEREG_SR_RD_ABS: 3059 case AEREG_SR_WR_ABS: 3060 case AEREG_DR_ABS: 3061 case AEREG_DR_RD_ABS: 3062 case AEREG_DR_WR_ABS: 3063 error = qat_aereg_abs_data_write(sc, ae, regtype, 3064 regaddr, value); 3065 break; 3066 default: 3067 error = EINVAL; 3068 break; 3069 } 3070 3071 return error; 3072 } 3073 3074 static int 3075 qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae, 3076 struct qat_uof_image *qui) 3077 { 3078 u_int i, expres; 3079 u_char ctx_mask; 3080 3081 for (i = 0; i < qui->qui_num_init_reg_sym; i++) { 3082 struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i]; 3083 3084 if (uirs->uirs_value_type == EXPR_VAL) { 3085 /* XXX */ 3086 device_printf(sc->sc_dev, 3087 "does not support initializing EXPR_VAL\n"); 3088 return ENOTSUP; 3089 } else { 3090 expres = uirs->uirs_value; 3091 } 3092 3093 switch (uirs->uirs_init_type) { 3094 case INIT_REG: 3095 if (__SHIFTOUT(qui->qui_image->ui_ae_mode, 3096 AE_MODE_CTX_MODE) == MAX_AE_CTX) { 3097 ctx_mask = 0xff; /* 8-ctx mode */ 3098 } else { 3099 ctx_mask = 0x55; /* 4-ctx mode */ 3100 } 3101 qat_aefw_init_reg(sc, ae, ctx_mask, 3102 (enum aereg_type)uirs->uirs_reg_type, 3103 (u_short)uirs->uirs_addr_offset, expres); 3104 break; 3105 case INIT_REG_CTX: 3106 if (__SHIFTOUT(qui->qui_image->ui_ae_mode, 3107 AE_MODE_CTX_MODE) == MAX_AE_CTX) { 3108 ctx_mask = 0xff; /* 8-ctx mode */ 3109 } else { 3110 ctx_mask = 0x55; /* 4-ctx mode */ 3111 } 3112 if (((1 << uirs->uirs_ctx) & ctx_mask) == 0) 3113 return EINVAL; 3114 qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx, 3115 (enum aereg_type)uirs->uirs_reg_type, 3116 (u_short)uirs->uirs_addr_offset, expres); 3117 break; 3118 case INIT_EXPR: 3119 case INIT_EXPR_ENDIAN_SWAP: 3120 default: 3121 device_printf(sc->sc_dev, 3122 "does not support initializing init_type %d\n", 3123 uirs->uirs_init_type); 3124 return ENOTSUP; 3125 } 3126 } 3127 3128 return 0; 3129 } 3130 3131 static int 3132 qat_aefw_init_memory(struct qat_softc *sc) 3133 { 3134 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 3135 size_t uimsz, initmemsz = qafu->qafu_init_mem_size; 3136 struct uof_init_mem *uim; 3137 int error, i; 3138 u_char ae; 3139 3140 uim = qafu->qafu_init_mem; 3141 for (i = 0; i < qafu->qafu_num_init_mem; i++) { 3142 uimsz = sizeof(struct uof_init_mem) + 3143 sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr; 3144 if (uimsz > initmemsz) { 3145 device_printf(sc->sc_dev, 3146 "invalid uof_init_mem or uof_mem_val_attr size\n"); 3147 return EINVAL; 3148 } 3149 3150 if (uim->uim_num_bytes > 0) { 3151 error = qat_aefw_init_memory_one(sc, uim); 3152 if (error) { 3153 device_printf(sc->sc_dev, 3154 "Could not init ae memory: %d\n", error); 3155 return error; 3156 } 3157 } 3158 uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz); 3159 initmemsz -= uimsz; 3160 } 3161 3162 /* run Batch put LM API */ 3163 for (ae = 0; ae < MAX_AE; ae++) { 3164 error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae], 3165 qafu->qafu_num_lm_init_inst[ae]); 3166 if (error) 3167 device_printf(sc->sc_dev, "Could not put lm\n"); 3168 3169 qat_aefw_free_lm_init(sc, ae); 3170 } 3171 3172 error = qat_aefw_init_ustore(sc); 3173 3174 /* XXX run Batch put LM API */ 3175 3176 return error; 3177 } 3178 3179 static int 3180 qat_aefw_init_globals(struct qat_softc *sc) 3181 { 3182 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 3183 int error, i, p, s; 3184 u_char ae; 3185 3186 /* initialize the memory segments */ 3187 if (qafu->qafu_num_init_mem > 0) { 3188 error = qat_aefw_init_memory(sc); 3189 if (error) 3190 return error; 3191 } else { 3192 error = qat_aefw_init_ustore(sc); 3193 if (error) 3194 return error; 3195 } 3196 3197 /* XXX bind import variables with ivd values */ 3198 3199 /* XXX bind the uC global variables 3200 * local variables will done on-the-fly */ 3201 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 3202 for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) { 3203 struct qat_uof_page *qup = 3204 &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p]; 3205 if (qup->qup_num_uw_blocks && 3206 (qup->qup_num_uc_var || qup->qup_num_imp_var)) { 3207 device_printf(sc->sc_dev, 3208 "not support uC global variables\n"); 3209 return ENOTSUP; 3210 } 3211 } 3212 } 3213 3214 for (ae = 0; ae < sc->sc_ae_num; ae++) { 3215 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3216 3217 for (s = 0; s < qae->qae_num_slices; s++) { 3218 struct qat_ae_slice *qas = &qae->qae_slices[s]; 3219 3220 if (qas->qas_image == NULL) 3221 continue; 3222 3223 error = 3224 qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image); 3225 if (error) 3226 return error; 3227 } 3228 } 3229 3230 return 0; 3231 } 3232 3233 static uint64_t 3234 qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup, 3235 u_int addr) 3236 { 3237 uint64_t uinst = 0; 3238 u_int i; 3239 3240 /* find the block */ 3241 for (i = 0; i < qup->qup_num_uw_blocks; i++) { 3242 struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i]; 3243 3244 if ((addr >= quub->quub_start_addr) && 3245 (addr <= (quub->quub_start_addr + 3246 (quub->quub_num_words - 1)))) { 3247 /* unpack n bytes and assigned to the 64-bit uword value. 3248 note: the microwords are stored as packed bytes. 3249 */ 3250 addr -= quub->quub_start_addr; 3251 addr *= AEV2_PACKED_UWORD_BYTES; 3252 memcpy(&uinst, 3253 (void *)((uintptr_t)quub->quub_micro_words + addr), 3254 AEV2_PACKED_UWORD_BYTES); 3255 uinst = uinst & UWORD_MASK; 3256 3257 return uinst; 3258 } 3259 } 3260 3261 return INVLD_UWORD; 3262 } 3263 3264 static int 3265 qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup) 3266 { 3267 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3268 uint64_t fill, *ucode_cpybuf; 3269 u_int error, i, upaddr, ninst, cpylen; 3270 3271 if (qup->qup_num_uc_var || qup->qup_num_neigh_reg || 3272 qup->qup_num_imp_var || qup->qup_num_imp_expr) { 3273 device_printf(sc->sc_dev, 3274 "does not support fixup locals\n"); 3275 return ENOTSUP; 3276 } 3277 3278 ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t)); 3279 3280 /* XXX get fill-pattern from an image -- they are all the same */ 3281 memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern, 3282 sizeof(uint64_t)); 3283 3284 upaddr = qup->qup_beg_paddr; 3285 ninst = qup->qup_num_micro_words; 3286 while (ninst > 0) { 3287 cpylen = min(ninst, UWORD_CPYBUF_SIZE); 3288 3289 /* load the buffer */ 3290 for (i = 0; i < cpylen; i++) { 3291 /* keep below code structure in case there are 3292 * different handling for shared secnarios */ 3293 if (!qae->qae_shareable_ustore) { 3294 /* qat_aefw_get_uof_inst() takes an address that 3295 * is relative to the start of the page. 3296 * So we don't need to add in the physical 3297 * offset of the page. */ 3298 if (qup->qup_page_region != 0) { 3299 /* XXX */ 3300 device_printf(sc->sc_dev, 3301 "region != 0 is not supported\n"); 3302 qat_free_mem(ucode_cpybuf); 3303 return ENOTSUP; 3304 } else { 3305 /* for mixing case, it should take 3306 * physical address */ 3307 ucode_cpybuf[i] = qat_aefw_get_uof_inst( 3308 sc, qup, upaddr + i); 3309 if (ucode_cpybuf[i] == INVLD_UWORD) { 3310 /* fill hole in the uof */ 3311 ucode_cpybuf[i] = fill; 3312 } 3313 } 3314 } else { 3315 /* XXX */ 3316 qat_free_mem(ucode_cpybuf); 3317 return ENOTSUP; 3318 } 3319 } 3320 3321 /* copy the buffer to ustore */ 3322 if (!qae->qae_shareable_ustore) { 3323 error = qat_ae_ucode_write(sc, ae, upaddr, cpylen, 3324 ucode_cpybuf); 3325 if (error) 3326 return error; 3327 } else { 3328 /* XXX */ 3329 qat_free_mem(ucode_cpybuf); 3330 return ENOTSUP; 3331 } 3332 upaddr += cpylen; 3333 ninst -= cpylen; 3334 } 3335 3336 qat_free_mem(ucode_cpybuf); 3337 3338 return 0; 3339 } 3340 3341 static int 3342 qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui) 3343 { 3344 struct uof_image *ui = qui->qui_image; 3345 struct qat_ae_page *qap; 3346 u_int s, p, c; 3347 int error; 3348 u_char ae, ctx_mask; 3349 3350 if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) 3351 ctx_mask = 0xff; /* 8-ctx mode */ 3352 else 3353 ctx_mask = 0x55; /* 4-ctx mode */ 3354 3355 /* load the default page and set assigned CTX PC 3356 * to the entrypoint address */ 3357 for (ae = 0; ae < sc->sc_ae_num; ae++) { 3358 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3359 struct qat_ae_slice *qas; 3360 u_int metadata; 3361 3362 MPASS(ae < UOF_MAX_NUM_OF_AE); 3363 3364 if ((ui->ui_ae_assigned & (1 << ae)) == 0) 3365 continue; 3366 3367 /* find the slice to which this image is assigned */ 3368 for (s = 0; s < qae->qae_num_slices; s++) { 3369 qas = &qae->qae_slices[s]; 3370 if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask) 3371 break; 3372 } 3373 if (s >= qae->qae_num_slices) 3374 continue; 3375 3376 qas = &qae->qae_slices[s]; 3377 3378 for (p = 0; p < ui->ui_num_pages; p++) { 3379 qap = &qas->qas_pages[p]; 3380 3381 /* Only load pages loaded by default */ 3382 if (!qap->qap_page->qup_def_page) 3383 continue; 3384 3385 error = qat_aefw_do_pagein(sc, ae, qap->qap_page); 3386 if (error) 3387 return error; 3388 } 3389 3390 metadata = qas->qas_image->qui_image->ui_app_metadata; 3391 if (metadata != 0xffffffff && bootverbose) { 3392 device_printf(sc->sc_dev, 3393 "loaded firmware: %s\n", 3394 qat_aefw_uof_string(sc, metadata)); 3395 } 3396 3397 /* Assume starting page is page 0 */ 3398 qap = &qas->qas_pages[0]; 3399 for (c = 0; c < MAX_AE_CTX; c++) { 3400 if (ctx_mask & (1 << c)) 3401 qas->qas_cur_pages[c] = qap; 3402 else 3403 qas->qas_cur_pages[c] = NULL; 3404 } 3405 3406 /* set the live context */ 3407 qae->qae_live_ctx_mask = ui->ui_ctx_assigned; 3408 3409 /* set context PC to the image entrypoint address */ 3410 error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned, 3411 ui->ui_entry_address); 3412 if (error) 3413 return error; 3414 } 3415 3416 /* XXX store the checksum for convenience */ 3417 3418 return 0; 3419 } 3420 3421 static int 3422 qat_aefw_uof_write(struct qat_softc *sc) 3423 { 3424 int error = 0; 3425 int i; 3426 3427 error = qat_aefw_init_globals(sc); 3428 if (error) { 3429 device_printf(sc->sc_dev, 3430 "Could not initialize globals\n"); 3431 return error; 3432 } 3433 3434 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 3435 error = qat_aefw_uof_write_one(sc, 3436 &sc->sc_aefw_uof.qafu_imgs[i]); 3437 if (error) 3438 break; 3439 } 3440 3441 /* XXX UcLo_computeFreeUstore */ 3442 3443 return error; 3444 } 3445