1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */ 2 /* $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ 3 4 /* 5 * Copyright (c) 2019 Internet Initiative Japan, Inc. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 /* 31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 37 * * Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * * Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in 41 * the documentation and/or other materials provided with the 42 * distribution. 43 * * Neither the name of Intel Corporation nor the names of its 44 * contributors may be used to endorse or promote products derived 45 * from this software without specific prior written permission. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58 */ 59 60 #include <sys/cdefs.h> 61 __FBSDID("$FreeBSD$"); 62 #if 0 63 __KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); 64 #endif 65 66 #include <sys/param.h> 67 #include <sys/bus.h> 68 #include <sys/firmware.h> 69 #include <sys/limits.h> 70 #include <sys/systm.h> 71 72 #include <machine/bus.h> 73 74 #include <dev/pci/pcireg.h> 75 #include <dev/pci/pcivar.h> 76 77 #include "qatreg.h" 78 #include "qatvar.h" 79 #include "qat_aevar.h" 80 81 static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t, 82 uint32_t); 83 static int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t, 84 uint32_t *); 85 static void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t, 86 bus_size_t, uint32_t); 87 static int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t, 88 bus_size_t, uint32_t *); 89 90 static u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short); 91 static int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char, 92 enum aereg_type, u_short, uint32_t); 93 static int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char, 94 enum aereg_type, u_short, uint32_t *); 95 static int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char, 96 enum aereg_type, u_short, uint32_t); 97 static int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char, 98 enum aereg_type, u_short, uint32_t); 99 static int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char, 100 enum aereg_type, u_short, uint32_t); 101 static int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short, 102 u_short *, u_char *); 103 static int qat_aereg_abs_data_write(struct qat_softc *, u_char, 104 enum aereg_type, u_short, uint32_t); 105 106 static void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int); 107 static void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int); 108 static void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char); 109 static void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char); 110 static void qat_ae_write_lm_mode(struct qat_softc *, u_char, 111 enum aereg_type, u_char); 112 static void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char, 113 u_char); 114 static void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char); 115 static int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int, 116 u_int); 117 118 static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char); 119 static int qat_ae_is_active(struct qat_softc *, u_char); 120 static int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int); 121 122 static int qat_ae_clear_reset(struct qat_softc *); 123 static int qat_ae_check(struct qat_softc *); 124 static int qat_ae_reset_timestamp(struct qat_softc *); 125 static void qat_ae_clear_xfer(struct qat_softc *); 126 static int qat_ae_clear_gprs(struct qat_softc *); 127 128 static void qat_ae_get_shared_ustore_ae(u_char, u_char *); 129 static u_int qat_ae_ucode_parity64(uint64_t); 130 static uint64_t qat_ae_ucode_set_ecc(uint64_t); 131 static int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int, 132 const uint64_t *); 133 static int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int, 134 uint64_t *); 135 static u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *); 136 static int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char, 137 uint64_t *, u_int, int, u_int, u_int *); 138 static int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char, 139 int *, uint64_t *, u_int, 140 u_int *, u_int *, u_int *, u_int *, u_int *); 141 static int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char, 142 u_int, u_int, u_int, u_int, u_int); 143 static int qat_ae_get_inst_num(int); 144 static int qat_ae_batch_put_lm(struct qat_softc *, u_char, 145 struct qat_ae_batch_init_list *, size_t); 146 static int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int); 147 148 static u_int qat_aefw_csum(char *, int); 149 static const char *qat_aefw_uof_string(struct qat_softc *, size_t); 150 static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *, 151 const char *, struct uof_chunk_hdr *); 152 153 static int qat_aefw_load_mof(struct qat_softc *); 154 static void qat_aefw_unload_mof(struct qat_softc *); 155 static int qat_aefw_load_mmp(struct qat_softc *); 156 static void qat_aefw_unload_mmp(struct qat_softc *); 157 158 static int qat_aefw_mof_find_uof0(struct qat_softc *, 159 struct mof_uof_hdr *, struct mof_uof_chunk_hdr *, 160 u_int, size_t, const char *, 161 size_t *, void **); 162 static int qat_aefw_mof_find_uof(struct qat_softc *); 163 static int qat_aefw_mof_parse(struct qat_softc *); 164 165 static int qat_aefw_uof_parse_image(struct qat_softc *, 166 struct qat_uof_image *, struct uof_chunk_hdr *uch); 167 static int qat_aefw_uof_parse_images(struct qat_softc *); 168 static int qat_aefw_uof_parse(struct qat_softc *); 169 170 static int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t, 171 struct qat_dmamem *); 172 static int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *); 173 static int qat_aefw_suof_load(struct qat_softc *sc, 174 struct qat_dmamem *dma); 175 static int qat_aefw_suof_parse_image(struct qat_softc *, 176 struct qat_suof_image *, struct suof_chunk_hdr *); 177 static int qat_aefw_suof_parse(struct qat_softc *); 178 static int qat_aefw_suof_write(struct qat_softc *); 179 180 static int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *, 181 struct qat_uof_image *); 182 static int qat_aefw_uof_init_ae(struct qat_softc *, u_char); 183 static int qat_aefw_uof_init(struct qat_softc *); 184 185 static int qat_aefw_init_memory_one(struct qat_softc *, 186 struct uof_init_mem *); 187 static void qat_aefw_free_lm_init(struct qat_softc *, u_char); 188 static int qat_aefw_init_ustore(struct qat_softc *); 189 static int qat_aefw_init_reg(struct qat_softc *, u_char, u_char, 190 enum aereg_type, u_short, u_int); 191 static int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char, 192 struct qat_uof_image *); 193 static int qat_aefw_init_memory(struct qat_softc *); 194 static int qat_aefw_init_globals(struct qat_softc *); 195 static uint64_t qat_aefw_get_uof_inst(struct qat_softc *, 196 struct qat_uof_page *, u_int); 197 static int qat_aefw_do_pagein(struct qat_softc *, u_char, 198 struct qat_uof_page *); 199 static int qat_aefw_uof_write_one(struct qat_softc *, 200 struct qat_uof_image *); 201 static int qat_aefw_uof_write(struct qat_softc *); 202 203 static int 204 qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, 205 uint32_t value) 206 { 207 int times = TIMEOUT_AE_CSR; 208 209 do { 210 qat_ae_local_write_4(sc, ae, offset, value); 211 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & 212 LOCAL_CSR_STATUS_STATUS) == 0) 213 return 0; 214 215 } while (times--); 216 217 device_printf(sc->sc_dev, 218 "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); 219 return EFAULT; 220 } 221 222 static int 223 qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset, 224 uint32_t *value) 225 { 226 int times = TIMEOUT_AE_CSR; 227 uint32_t v; 228 229 do { 230 v = qat_ae_local_read_4(sc, ae, offset); 231 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & 232 LOCAL_CSR_STATUS_STATUS) == 0) { 233 *value = v; 234 return 0; 235 } 236 } while (times--); 237 238 device_printf(sc->sc_dev, 239 "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); 240 return EFAULT; 241 } 242 243 static void 244 qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask, 245 bus_size_t offset, uint32_t value) 246 { 247 int ctx; 248 uint32_t ctxptr; 249 250 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || 251 offset == FUTURE_COUNT_SIGNAL_INDIRECT || 252 offset == CTX_STS_INDIRECT || 253 offset == CTX_WAKEUP_EVENTS_INDIRECT || 254 offset == CTX_SIG_EVENTS_INDIRECT || 255 offset == LM_ADDR_0_INDIRECT || 256 offset == LM_ADDR_1_INDIRECT || 257 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || 258 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); 259 260 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); 261 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { 262 if ((ctx_mask & (1 << ctx)) == 0) 263 continue; 264 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); 265 qat_ae_write_4(sc, ae, offset, value); 266 } 267 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); 268 } 269 270 static int 271 qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx, 272 bus_size_t offset, uint32_t *value) 273 { 274 int error; 275 uint32_t ctxptr; 276 277 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || 278 offset == FUTURE_COUNT_SIGNAL_INDIRECT || 279 offset == CTX_STS_INDIRECT || 280 offset == CTX_WAKEUP_EVENTS_INDIRECT || 281 offset == CTX_SIG_EVENTS_INDIRECT || 282 offset == LM_ADDR_0_INDIRECT || 283 offset == LM_ADDR_1_INDIRECT || 284 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || 285 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); 286 287 /* save the ctx ptr */ 288 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); 289 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != 290 (ctx & CSR_CTX_POINTER_CONTEXT)) 291 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); 292 293 error = qat_ae_read_4(sc, ae, offset, value); 294 295 /* restore ctx ptr */ 296 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != 297 (ctx & CSR_CTX_POINTER_CONTEXT)) 298 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); 299 300 return error; 301 } 302 303 static u_short 304 qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg) 305 { 306 u_short addr; 307 308 switch (regtype) { 309 case AEREG_GPA_ABS: 310 case AEREG_GPB_ABS: 311 addr = (reg & 0x7f) | 0x80; 312 break; 313 case AEREG_GPA_REL: 314 case AEREG_GPB_REL: 315 addr = reg & 0x1f; 316 break; 317 case AEREG_SR_RD_REL: 318 case AEREG_SR_WR_REL: 319 case AEREG_SR_REL: 320 addr = 0x180 | (reg & 0x1f); 321 break; 322 case AEREG_SR_INDX: 323 addr = 0x140 | ((reg & 0x3) << 1); 324 break; 325 case AEREG_DR_RD_REL: 326 case AEREG_DR_WR_REL: 327 case AEREG_DR_REL: 328 addr = 0x1c0 | (reg & 0x1f); 329 break; 330 case AEREG_DR_INDX: 331 addr = 0x100 | ((reg & 0x3) << 1); 332 break; 333 case AEREG_NEIGH_INDX: 334 addr = 0x241 | ((reg & 0x3) << 1); 335 break; 336 case AEREG_NEIGH_REL: 337 addr = 0x280 | (reg & 0x1f); 338 break; 339 case AEREG_LMEM0: 340 addr = 0x200; 341 break; 342 case AEREG_LMEM1: 343 addr = 0x220; 344 break; 345 case AEREG_NO_DEST: 346 addr = 0x300 | (reg & 0xff); 347 break; 348 default: 349 addr = AEREG_BAD_REGADDR; 350 break; 351 } 352 return (addr); 353 } 354 355 static int 356 qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx, 357 enum aereg_type regtype, u_short relreg, uint32_t value) 358 { 359 uint16_t srchi, srclo, destaddr, data16hi, data16lo; 360 uint64_t inst[] = { 361 0x0F440000000ull, /* immed_w1[reg, val_hi16] */ 362 0x0F040000000ull, /* immed_w0[reg, val_lo16] */ 363 0x0F0000C0300ull, /* nop */ 364 0x0E000010000ull /* ctx_arb[kill] */ 365 }; 366 const int ninst = nitems(inst); 367 const int imm_w1 = 0, imm_w0 = 1; 368 unsigned int ctxen; 369 uint16_t mask; 370 371 /* This logic only works for GPRs and LM index registers, 372 not NN or XFER registers! */ 373 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || 374 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); 375 376 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) { 377 /* determine the context mode */ 378 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 379 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 380 /* 4-ctx mode */ 381 if (ctx & 0x1) 382 return EINVAL; 383 mask = 0x1f; 384 } else { 385 /* 8-ctx mode */ 386 mask = 0x0f; 387 } 388 if (relreg & ~mask) 389 return EINVAL; 390 } 391 if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == 392 AEREG_BAD_REGADDR) { 393 return EINVAL; 394 } 395 396 data16lo = 0xffff & value; 397 data16hi = 0xffff & (value >> 16); 398 srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST, 399 (uint16_t)(0xff & data16hi)); 400 srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST, 401 (uint16_t)(0xff & data16lo)); 402 403 switch (regtype) { 404 case AEREG_GPA_REL: /* A rel source */ 405 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | 406 ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff); 407 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | 408 ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff); 409 break; 410 default: 411 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | 412 ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff); 413 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | 414 ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff); 415 break; 416 } 417 418 return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL); 419 } 420 421 static int 422 qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx, 423 enum aereg_type regtype, u_short relreg, uint32_t *value) 424 { 425 uint64_t inst, savucode; 426 uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi; 427 u_int uaddr, ustore_addr; 428 int error; 429 u_short mask, regaddr; 430 u_char nae; 431 432 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || 433 regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL || 434 regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL || 435 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); 436 437 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) || 438 (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) || 439 (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL)) 440 { 441 /* determine the context mode */ 442 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 443 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 444 /* 4-ctx mode */ 445 if (ctx & 0x1) 446 return EINVAL; 447 mask = 0x1f; 448 } else { 449 /* 8-ctx mode */ 450 mask = 0x0f; 451 } 452 if (relreg & ~mask) 453 return EINVAL; 454 } 455 if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == 456 AEREG_BAD_REGADDR) { 457 return EINVAL; 458 } 459 460 /* instruction -- alu[--, --, B, reg] */ 461 switch (regtype) { 462 case AEREG_GPA_REL: 463 /* A rel source */ 464 inst = 0xA070000000ull | (regaddr & 0x3ff); 465 break; 466 default: 467 inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10)); 468 break; 469 } 470 471 /* backup shared control store bit, and force AE to 472 * none-shared mode before executing ucode snippet */ 473 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 474 if (misc & AE_MISC_CONTROL_SHARE_CS) { 475 qat_ae_get_shared_ustore_ae(ae, &nae); 476 if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae)) 477 return EBUSY; 478 } 479 480 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 481 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 482 483 /* read current context */ 484 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); 485 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); 486 487 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 488 /* prevent clearing the W1C bits: the breakpoint bit, 489 ECC error bit, and Parity error bit */ 490 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 491 492 /* change the context */ 493 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) 494 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 495 ctx & ACTIVE_CTX_STATUS_ACNO); 496 /* save a ustore location */ 497 if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) { 498 /* restore AE_MISC_CONTROL csr */ 499 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 500 501 /* restore the context */ 502 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { 503 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 504 savctx & ACTIVE_CTX_STATUS_ACNO); 505 } 506 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 507 508 return (error); 509 } 510 511 /* turn off ustore parity */ 512 qat_ae_write_4(sc, ae, CTX_ENABLES, 513 ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE)); 514 515 /* save ustore-addr csr */ 516 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 517 518 /* write the ALU instruction to ustore, enable ecs bit */ 519 uaddr = 0 | USTORE_ADDRESS_ECS; 520 521 /* set the uaddress */ 522 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 523 inst = qat_ae_ucode_set_ecc(inst); 524 525 ulo = (uint32_t)(inst & 0xffffffff); 526 uhi = (uint32_t)(inst >> 32); 527 528 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); 529 530 /* this will auto increment the address */ 531 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); 532 533 /* set the uaddress */ 534 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 535 536 /* delay for at least 8 cycles */ 537 qat_ae_wait_num_cycles(sc, ae, 0x8, 0); 538 539 /* read ALU output -- the instruction should have been executed 540 prior to clearing the ECS in putUwords */ 541 qat_ae_read_4(sc, ae, ALU_OUT, value); 542 543 /* restore ustore-addr csr */ 544 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 545 546 /* restore the ustore */ 547 error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode); 548 549 /* restore the context */ 550 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { 551 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 552 savctx & ACTIVE_CTX_STATUS_ACNO); 553 } 554 555 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 556 557 /* restore AE_MISC_CONTROL csr */ 558 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 559 560 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 561 562 return error; 563 } 564 565 static int 566 qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, 567 enum aereg_type regtype, u_short relreg, uint32_t value) 568 { 569 bus_size_t addr; 570 int error; 571 uint32_t ctxen; 572 u_short mask; 573 u_short dr_offset; 574 575 MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL || 576 regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL); 577 578 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 579 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 580 if (ctx & 0x1) { 581 device_printf(sc->sc_dev, 582 "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx); 583 return EINVAL; 584 } 585 mask = 0x1f; 586 dr_offset = 0x20; 587 588 } else { 589 mask = 0x0f; 590 dr_offset = 0x10; 591 } 592 593 if (relreg & ~mask) 594 return EINVAL; 595 596 addr = relreg + (ctx << 0x5); 597 598 switch (regtype) { 599 case AEREG_SR_REL: 600 case AEREG_SR_RD_REL: 601 qat_ae_xfer_write_4(sc, ae, addr, value); 602 break; 603 case AEREG_DR_REL: 604 case AEREG_DR_RD_REL: 605 qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value); 606 break; 607 default: 608 error = EINVAL; 609 } 610 611 return error; 612 } 613 614 static int 615 qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, 616 enum aereg_type regtype, u_short relreg, uint32_t value) 617 { 618 619 panic("notyet"); 620 621 return 0; 622 } 623 624 static int 625 qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx, 626 enum aereg_type regtype, u_short relreg, uint32_t value) 627 { 628 629 panic("notyet"); 630 631 return 0; 632 } 633 634 static int 635 qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae, 636 u_short absreg, u_short *relreg, u_char *ctx) 637 { 638 uint32_t ctxen; 639 640 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 641 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 642 /* 4-ctx mode */ 643 *relreg = absreg & 0x1f; 644 *ctx = (absreg >> 0x4) & 0x6; 645 } else { 646 /* 8-ctx mode */ 647 *relreg = absreg & 0x0f; 648 *ctx = (absreg >> 0x4) & 0x7; 649 } 650 651 return 0; 652 } 653 654 static int 655 qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae, 656 enum aereg_type regtype, u_short absreg, uint32_t value) 657 { 658 int error; 659 u_short relreg; 660 u_char ctx; 661 662 qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx); 663 664 switch (regtype) { 665 case AEREG_GPA_ABS: 666 MPASS(absreg < MAX_GPR_REG); 667 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 668 relreg, value); 669 break; 670 case AEREG_GPB_ABS: 671 MPASS(absreg < MAX_GPR_REG); 672 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 673 relreg, value); 674 break; 675 case AEREG_DR_RD_ABS: 676 MPASS(absreg < MAX_XFER_REG); 677 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL, 678 relreg, value); 679 break; 680 case AEREG_SR_RD_ABS: 681 MPASS(absreg < MAX_XFER_REG); 682 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL, 683 relreg, value); 684 break; 685 case AEREG_DR_WR_ABS: 686 MPASS(absreg < MAX_XFER_REG); 687 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL, 688 relreg, value); 689 break; 690 case AEREG_SR_WR_ABS: 691 MPASS(absreg < MAX_XFER_REG); 692 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL, 693 relreg, value); 694 break; 695 case AEREG_NEIGH_ABS: 696 MPASS(absreg < MAX_NN_REG); 697 if (absreg >= MAX_NN_REG) 698 return EINVAL; 699 error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL, 700 relreg, value); 701 break; 702 default: 703 panic("Invalid Register Type"); 704 } 705 706 return error; 707 } 708 709 static void 710 qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) 711 { 712 uint32_t ctxen; 713 714 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 715 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 716 717 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 718 ctx_mask &= 0x55; 719 } else { 720 ctx_mask &= 0xff; 721 } 722 723 ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE); 724 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 725 } 726 727 static void 728 qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) 729 { 730 uint32_t ctxen; 731 732 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 733 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 734 ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE)); 735 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 736 } 737 738 static void 739 qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode) 740 { 741 uint32_t val, nval; 742 743 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 744 val &= CTX_ENABLES_IGNORE_W1C_MASK; 745 746 if (mode == 4) 747 nval = val | CTX_ENABLES_INUSE_CONTEXTS; 748 else 749 nval = val & ~CTX_ENABLES_INUSE_CONTEXTS; 750 751 if (val != nval) 752 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 753 } 754 755 static void 756 qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode) 757 { 758 uint32_t val, nval; 759 760 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 761 val &= CTX_ENABLES_IGNORE_W1C_MASK; 762 763 if (mode) 764 nval = val | CTX_ENABLES_NN_MODE; 765 else 766 nval = val & ~CTX_ENABLES_NN_MODE; 767 768 if (val != nval) 769 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 770 } 771 772 static void 773 qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae, 774 enum aereg_type lm, u_char mode) 775 { 776 uint32_t val, nval; 777 uint32_t bit; 778 779 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 780 val &= CTX_ENABLES_IGNORE_W1C_MASK; 781 782 switch (lm) { 783 case AEREG_LMEM0: 784 bit = CTX_ENABLES_LMADDR_0_GLOBAL; 785 break; 786 case AEREG_LMEM1: 787 bit = CTX_ENABLES_LMADDR_1_GLOBAL; 788 break; 789 default: 790 panic("invalid lmem reg type"); 791 break; 792 } 793 794 if (mode) 795 nval = val | bit; 796 else 797 nval = val & ~bit; 798 799 if (val != nval) 800 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 801 } 802 803 static void 804 qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode) 805 { 806 uint32_t val, nval; 807 808 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 809 810 if (mode == 1) 811 nval = val | AE_MISC_CONTROL_SHARE_CS; 812 else 813 nval = val & ~AE_MISC_CONTROL_SHARE_CS; 814 815 if (val != nval) 816 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval); 817 } 818 819 static void 820 qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode) 821 { 822 u_char nae; 823 824 qat_ae_get_shared_ustore_ae(ae, &nae); 825 826 qat_ae_write_shared_cs_mode0(sc, ae, mode); 827 828 if ((sc->sc_ae_mask & (1 << nae))) { 829 qat_ae_write_shared_cs_mode0(sc, nae, mode); 830 } 831 } 832 833 static int 834 qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae, 835 u_int reload_size, int shared_mode, u_int ustore_dram_addr) 836 { 837 uint32_t val, cs_reload; 838 839 switch (reload_size) { 840 case 0: 841 cs_reload = 0x0; 842 break; 843 case QAT_2K: 844 cs_reload = 0x1; 845 break; 846 case QAT_4K: 847 cs_reload = 0x2; 848 break; 849 case QAT_8K: 850 cs_reload = 0x3; 851 break; 852 default: 853 return EINVAL; 854 } 855 856 if (cs_reload) 857 QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr; 858 859 QAT_AE(sc, ae).qae_reload_size = reload_size; 860 861 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 862 val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD | 863 AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS); 864 val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) | 865 __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD); 866 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); 867 868 return 0; 869 } 870 871 static enum qat_ae_status 872 qat_ae_get_status(struct qat_softc *sc, u_char ae) 873 { 874 int error; 875 uint32_t val = 0; 876 877 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 878 if (error || val & CTX_ENABLES_ENABLE) 879 return QAT_AE_ENABLED; 880 881 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); 882 if (val & ACTIVE_CTX_STATUS_ABO) 883 return QAT_AE_ACTIVE; 884 885 return QAT_AE_DISABLED; 886 } 887 888 889 static int 890 qat_ae_is_active(struct qat_softc *sc, u_char ae) 891 { 892 uint32_t val; 893 894 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) 895 return 1; 896 897 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); 898 if (val & ACTIVE_CTX_STATUS_ABO) 899 return 1; 900 else 901 return 0; 902 } 903 904 /* returns 1 if actually waited for specified number of cycles */ 905 static int 906 qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check) 907 { 908 uint32_t cnt, actx; 909 int pcnt, ccnt, elapsed, times; 910 911 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 912 pcnt = cnt & 0xffff; 913 914 times = TIMEOUT_AE_CHECK; 915 do { 916 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 917 ccnt = cnt & 0xffff; 918 919 elapsed = ccnt - pcnt; 920 if (elapsed == 0) { 921 times--; 922 } 923 if (times <= 0) { 924 device_printf(sc->sc_dev, 925 "qat_ae_wait_num_cycles timeout\n"); 926 return -1; 927 } 928 929 if (elapsed < 0) 930 elapsed += 0x10000; 931 932 if (elapsed >= CYCLES_FROM_READY2EXE && check) { 933 if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, 934 &actx) == 0) { 935 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) 936 return 0; 937 } 938 } 939 } while (cycles > elapsed); 940 941 if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) { 942 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) 943 return 0; 944 } 945 946 return 1; 947 } 948 949 int 950 qat_ae_init(struct qat_softc *sc) 951 { 952 int error; 953 uint32_t mask, val = 0; 954 u_char ae; 955 956 /* XXX adf_initSysMemInfo */ 957 958 /* XXX Disable clock gating for some chip if debug mode */ 959 960 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 961 struct qat_ae *qae = &sc->sc_ae[ae]; 962 if (!(mask & 1)) 963 continue; 964 965 qae->qae_ustore_size = USTORE_SIZE; 966 967 qae->qae_free_addr = 0; 968 qae->qae_free_size = USTORE_SIZE; 969 qae->qae_live_ctx_mask = AE_ALL_CTX; 970 qae->qae_ustore_dram_addr = 0; 971 qae->qae_reload_size = 0; 972 } 973 974 /* XXX Enable attention interrupt */ 975 976 error = qat_ae_clear_reset(sc); 977 if (error) 978 return error; 979 980 qat_ae_clear_xfer(sc); 981 982 if (!sc->sc_hw.qhw_fw_auth) { 983 error = qat_ae_clear_gprs(sc); 984 if (error) 985 return error; 986 } 987 988 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ 989 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 990 if (!(mask & 1)) 991 continue; 992 qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val); 993 val |= 0x1; 994 qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val); 995 } 996 997 error = qat_ae_clear_reset(sc); 998 if (error) 999 return error; 1000 1001 /* XXX XXX XXX Clean MMP memory if mem scrub is supported */ 1002 /* halMem_ScrubMMPMemory */ 1003 1004 return 0; 1005 } 1006 1007 int 1008 qat_ae_start(struct qat_softc *sc) 1009 { 1010 int error; 1011 u_char ae; 1012 1013 for (ae = 0; ae < sc->sc_ae_num; ae++) { 1014 if ((sc->sc_ae_mask & (1 << ae)) == 0) 1015 continue; 1016 1017 error = qat_aefw_start(sc, ae, 0xff); 1018 if (error) 1019 return error; 1020 } 1021 1022 return 0; 1023 } 1024 1025 void 1026 qat_ae_cluster_intr(void *arg) 1027 { 1028 /* Nothing to implement until we support SRIOV. */ 1029 printf("qat_ae_cluster_intr\n"); 1030 } 1031 1032 static int 1033 qat_ae_clear_reset(struct qat_softc *sc) 1034 { 1035 int error; 1036 uint32_t times, reset, clock, reg, mask; 1037 u_char ae; 1038 1039 reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); 1040 reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK)); 1041 reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)); 1042 times = TIMEOUT_AE_RESET; 1043 do { 1044 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset); 1045 if ((times--) == 0) { 1046 device_printf(sc->sc_dev, "couldn't reset AEs\n"); 1047 return EBUSY; 1048 } 1049 reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); 1050 } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) | 1051 __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)) 1052 & reg); 1053 1054 /* Enable clock for AE and QAT */ 1055 clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN); 1056 clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK); 1057 clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK); 1058 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock); 1059 1060 error = qat_ae_check(sc); 1061 if (error) 1062 return error; 1063 1064 /* 1065 * Set undefined power-up/reset states to reasonable default values... 1066 * just to make sure we're starting from a known point 1067 */ 1068 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1069 if (!(mask & 1)) 1070 continue; 1071 1072 /* init the ctx_enable */ 1073 qat_ae_write_4(sc, ae, CTX_ENABLES, 1074 CTX_ENABLES_INIT); 1075 1076 /* initialize the PCs */ 1077 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1078 CTX_STS_INDIRECT, 1079 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1080 1081 /* init the ctx_arb */ 1082 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, 1083 CTX_ARB_CNTL_INIT); 1084 1085 /* enable cc */ 1086 qat_ae_write_4(sc, ae, CC_ENABLE, 1087 CC_ENABLE_INIT); 1088 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1089 CTX_WAKEUP_EVENTS_INDIRECT, 1090 CTX_WAKEUP_EVENTS_INDIRECT_INIT); 1091 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1092 CTX_SIG_EVENTS_INDIRECT, 1093 CTX_SIG_EVENTS_INDIRECT_INIT); 1094 } 1095 1096 if ((sc->sc_ae_mask != 0) && 1097 sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) { 1098 /* XXX XXX XXX init eSram only when this is boot time */ 1099 } 1100 1101 if ((sc->sc_ae_mask != 0) && 1102 sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) { 1103 /* XXX XXX XXX wait shram to complete initialization */ 1104 } 1105 1106 qat_ae_reset_timestamp(sc); 1107 1108 return 0; 1109 } 1110 1111 static int 1112 qat_ae_check(struct qat_softc *sc) 1113 { 1114 int error, times, ae; 1115 uint32_t cnt, pcnt, mask; 1116 1117 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1118 if (!(mask & 1)) 1119 continue; 1120 1121 times = TIMEOUT_AE_CHECK; 1122 error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 1123 if (error) { 1124 device_printf(sc->sc_dev, 1125 "couldn't access AE %d CSR\n", ae); 1126 return error; 1127 } 1128 pcnt = cnt & 0xffff; 1129 1130 while (1) { 1131 error = qat_ae_read_4(sc, ae, 1132 PROFILE_COUNT, &cnt); 1133 if (error) { 1134 device_printf(sc->sc_dev, 1135 "couldn't access AE %d CSR\n", ae); 1136 return error; 1137 } 1138 cnt &= 0xffff; 1139 if (cnt == pcnt) 1140 times--; 1141 else 1142 break; 1143 if (times <= 0) { 1144 device_printf(sc->sc_dev, 1145 "AE %d CSR is useless\n", ae); 1146 return EFAULT; 1147 } 1148 } 1149 } 1150 1151 return 0; 1152 } 1153 1154 static int 1155 qat_ae_reset_timestamp(struct qat_softc *sc) 1156 { 1157 uint32_t misc, mask; 1158 u_char ae; 1159 1160 /* stop the timestamp timers */ 1161 misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC); 1162 if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) { 1163 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, 1164 misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN)); 1165 } 1166 1167 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1168 if (!(mask & 1)) 1169 continue; 1170 qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0); 1171 qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0); 1172 } 1173 1174 /* start timestamp timers */ 1175 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, 1176 misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN); 1177 1178 return 0; 1179 } 1180 1181 static void 1182 qat_ae_clear_xfer(struct qat_softc *sc) 1183 { 1184 u_int mask, reg; 1185 u_char ae; 1186 1187 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1188 if (!(mask & 1)) 1189 continue; 1190 1191 for (reg = 0; reg < MAX_GPR_REG; reg++) { 1192 qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS, 1193 reg, 0); 1194 qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS, 1195 reg, 0); 1196 } 1197 } 1198 } 1199 1200 static int 1201 qat_ae_clear_gprs(struct qat_softc *sc) 1202 { 1203 uint32_t val; 1204 uint32_t saved_ctx = 0; 1205 int times = TIMEOUT_AE_CHECK, rv; 1206 u_char ae; 1207 u_int mask; 1208 1209 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1210 if (!(mask & 1)) 1211 continue; 1212 1213 /* turn off share control store bit */ 1214 val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 1215 val &= ~AE_MISC_CONTROL_SHARE_CS; 1216 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); 1217 1218 /* turn off ucode parity */ 1219 /* make sure nn_mode is set to self */ 1220 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 1221 val &= CTX_ENABLES_IGNORE_W1C_MASK; 1222 val |= CTX_ENABLES_NN_MODE; 1223 val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE; 1224 qat_ae_write_4(sc, ae, CTX_ENABLES, val); 1225 1226 /* copy instructions to ustore */ 1227 qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst), 1228 ae_clear_gprs_inst); 1229 1230 /* set PC */ 1231 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, 1232 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1233 1234 /* save current context */ 1235 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx); 1236 /* change the active context */ 1237 /* start the context from ctx 0 */ 1238 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0); 1239 1240 /* wakeup-event voluntary */ 1241 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1242 CTX_WAKEUP_EVENTS_INDIRECT, 1243 CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); 1244 /* clean signals */ 1245 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1246 CTX_SIG_EVENTS_INDIRECT, 0); 1247 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1248 1249 qat_ae_enable_ctx(sc, ae, AE_ALL_CTX); 1250 } 1251 1252 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1253 if (!(mask & 1)) 1254 continue; 1255 /* wait for AE to finish */ 1256 do { 1257 rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1); 1258 } while (rv && times--); 1259 if (times <= 0) { 1260 device_printf(sc->sc_dev, 1261 "qat_ae_clear_gprs timeout"); 1262 return ETIMEDOUT; 1263 } 1264 qat_ae_disable_ctx(sc, ae, AE_ALL_CTX); 1265 /* change the active context */ 1266 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1267 saved_ctx & ACTIVE_CTX_STATUS_ACNO); 1268 /* init the ctx_enable */ 1269 qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT); 1270 /* initialize the PCs */ 1271 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1272 CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1273 /* init the ctx_arb */ 1274 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT); 1275 /* enable cc */ 1276 qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT); 1277 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1278 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT); 1279 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, 1280 CTX_SIG_EVENTS_INDIRECT_INIT); 1281 } 1282 1283 return 0; 1284 } 1285 1286 static void 1287 qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae) 1288 { 1289 if (ae & 0x1) 1290 *nae = ae - 1; 1291 else 1292 *nae = ae + 1; 1293 } 1294 1295 static u_int 1296 qat_ae_ucode_parity64(uint64_t ucode) 1297 { 1298 1299 ucode ^= ucode >> 1; 1300 ucode ^= ucode >> 2; 1301 ucode ^= ucode >> 4; 1302 ucode ^= ucode >> 8; 1303 ucode ^= ucode >> 16; 1304 ucode ^= ucode >> 32; 1305 1306 return ((u_int)(ucode & 1)); 1307 } 1308 1309 static uint64_t 1310 qat_ae_ucode_set_ecc(uint64_t ucode) 1311 { 1312 static const uint64_t 1313 bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL, 1314 bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL, 1315 bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL, 1316 bit6mask=0xdaf69a46910ULL; 1317 1318 /* clear the ecc bits */ 1319 ucode &= ~(0x7fULL << USTORE_ECC_BIT_0); 1320 1321 ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) << 1322 USTORE_ECC_BIT_0; 1323 ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) << 1324 USTORE_ECC_BIT_1; 1325 ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) << 1326 USTORE_ECC_BIT_2; 1327 ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) << 1328 USTORE_ECC_BIT_3; 1329 ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) << 1330 USTORE_ECC_BIT_4; 1331 ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) << 1332 USTORE_ECC_BIT_5; 1333 ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) << 1334 USTORE_ECC_BIT_6; 1335 1336 return (ucode); 1337 } 1338 1339 static int 1340 qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, 1341 const uint64_t *ucode) 1342 { 1343 uint64_t tmp; 1344 uint32_t ustore_addr, ulo, uhi; 1345 int i; 1346 1347 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 1348 uaddr |= USTORE_ADDRESS_ECS; 1349 1350 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 1351 for (i = 0; i < ninst; i++) { 1352 tmp = qat_ae_ucode_set_ecc(ucode[i]); 1353 ulo = (uint32_t)(tmp & 0xffffffff); 1354 uhi = (uint32_t)(tmp >> 32); 1355 1356 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); 1357 /* this will auto increment the address */ 1358 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); 1359 } 1360 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 1361 1362 return 0; 1363 } 1364 1365 static int 1366 qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, 1367 uint64_t *ucode) 1368 { 1369 uint32_t misc, ustore_addr, ulo, uhi; 1370 u_int ii; 1371 u_char nae; 1372 1373 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) 1374 return EBUSY; 1375 1376 /* determine whether it neighbour AE runs in shared control store 1377 * status */ 1378 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1379 if (misc & AE_MISC_CONTROL_SHARE_CS) { 1380 qat_ae_get_shared_ustore_ae(ae, &nae); 1381 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) 1382 return EBUSY; 1383 } 1384 1385 /* if reloadable, then get it all from dram-ustore */ 1386 if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD)) 1387 panic("notyet"); /* XXX getReloadUwords */ 1388 1389 /* disable SHARE_CS bit to workaround silicon bug */ 1390 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb); 1391 1392 MPASS(uaddr + ninst <= USTORE_SIZE); 1393 1394 /* save ustore-addr csr */ 1395 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 1396 1397 uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */ 1398 for (ii = 0; ii < ninst; ii++) { 1399 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 1400 1401 uaddr++; 1402 qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo); 1403 qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi); 1404 ucode[ii] = uhi; 1405 ucode[ii] = (ucode[ii] << 32) | ulo; 1406 } 1407 1408 /* restore SHARE_CS bit to workaround silicon bug */ 1409 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 1410 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 1411 1412 return 0; 1413 } 1414 1415 static u_int 1416 qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr, 1417 u_int *value) 1418 { 1419 const uint64_t *inst_arr; 1420 u_int ninst0, curvalue; 1421 int ii, vali, fixup, usize = 0; 1422 1423 if (size == 0) 1424 return 0; 1425 1426 ninst0 = ninst; 1427 vali = 0; 1428 curvalue = value[vali++]; 1429 1430 switch (size) { 1431 case 0x1: 1432 inst_arr = ae_inst_1b; 1433 usize = nitems(ae_inst_1b); 1434 break; 1435 case 0x2: 1436 inst_arr = ae_inst_2b; 1437 usize = nitems(ae_inst_2b); 1438 break; 1439 case 0x3: 1440 inst_arr = ae_inst_3b; 1441 usize = nitems(ae_inst_3b); 1442 break; 1443 default: 1444 inst_arr = ae_inst_4b; 1445 usize = nitems(ae_inst_4b); 1446 break; 1447 } 1448 1449 fixup = ninst; 1450 for (ii = 0; ii < usize; ii++) 1451 ucode[ninst++] = inst_arr[ii]; 1452 1453 INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr)); 1454 fixup++; 1455 INSERT_IMMED_GPRA_CONST(ucode[fixup], 0); 1456 fixup++; 1457 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); 1458 fixup++; 1459 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); 1460 /* XXX fixup++ ? */ 1461 1462 if (size <= 0x4) 1463 return (ninst - ninst0); 1464 1465 size -= sizeof(u_int); 1466 while (size >= sizeof(u_int)) { 1467 curvalue = value[vali++]; 1468 fixup = ninst; 1469 ucode[ninst++] = ae_inst_4b[0x2]; 1470 ucode[ninst++] = ae_inst_4b[0x3]; 1471 ucode[ninst++] = ae_inst_4b[0x8]; 1472 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); 1473 fixup++; 1474 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); 1475 /* XXX fixup++ ? */ 1476 1477 addr += sizeof(u_int); 1478 size -= sizeof(u_int); 1479 } 1480 /* call this function recusive when the left size less than 4 */ 1481 ninst += 1482 qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali); 1483 1484 return (ninst - ninst0); 1485 } 1486 1487 static int 1488 qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx, 1489 uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles, 1490 u_int *endpc) 1491 { 1492 int error = 0, share_cs = 0; 1493 uint64_t savucode[MAX_EXEC_INST]; 1494 uint32_t indr_lm_addr_0, indr_lm_addr_1; 1495 uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1; 1496 uint32_t indr_future_cnt_sig; 1497 uint32_t indr_sig, active_sig; 1498 uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl; 1499 uint32_t misc, nmisc, ctxen; 1500 u_char nae; 1501 1502 MPASS(ninst <= USTORE_SIZE); 1503 1504 if (qat_ae_is_active(sc, ae)) 1505 return EBUSY; 1506 1507 /* save current LM addr */ 1508 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0); 1509 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1); 1510 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, 1511 &indr_lm_addr_byte_0); 1512 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, 1513 &indr_lm_addr_byte_1); 1514 1515 /* backup shared control store bit, and force AE to 1516 none-shared mode before executing ucode snippet */ 1517 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1518 if (misc & AE_MISC_CONTROL_SHARE_CS) { 1519 share_cs = 1; 1520 qat_ae_get_shared_ustore_ae(ae, &nae); 1521 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) 1522 return EBUSY; 1523 } 1524 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 1525 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 1526 1527 /* save current states: */ 1528 if (ninst <= MAX_EXEC_INST) { 1529 error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode); 1530 if (error) { 1531 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 1532 return error; 1533 } 1534 } 1535 1536 /* save wakeup-events */ 1537 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT, 1538 &wakeup_ev); 1539 /* save PC */ 1540 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc); 1541 savpc &= UPC_MASK; 1542 1543 /* save ctx enables */ 1544 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 1545 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 1546 /* save conditional-code */ 1547 qat_ae_read_4(sc, ae, CC_ENABLE, &savcc); 1548 /* save current context */ 1549 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); 1550 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); 1551 1552 /* save indirect csrs */ 1553 qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, 1554 &indr_future_cnt_sig); 1555 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig); 1556 qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig); 1557 1558 /* turn off ucode parity */ 1559 qat_ae_write_4(sc, ae, CTX_ENABLES, 1560 ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE); 1561 1562 /* copy instructions to ustore */ 1563 qat_ae_ucode_write(sc, ae, 0, ninst, ucode); 1564 /* set PC */ 1565 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0); 1566 /* change the active context */ 1567 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1568 ctx & ACTIVE_CTX_STATUS_ACNO); 1569 1570 if (cond_code_off) { 1571 /* disable conditional-code*/ 1572 qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff); 1573 } 1574 1575 /* wakeup-event voluntary */ 1576 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, 1577 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); 1578 1579 /* clean signals */ 1580 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0); 1581 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1582 1583 /* enable context */ 1584 qat_ae_enable_ctx(sc, ae, 1 << ctx); 1585 1586 /* wait for it to finish */ 1587 if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0) 1588 error = ETIMEDOUT; 1589 1590 /* see if we need to get the current PC */ 1591 if (endpc != NULL) { 1592 uint32_t ctx_status; 1593 1594 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, 1595 &ctx_status); 1596 *endpc = ctx_status & UPC_MASK; 1597 } 1598 #if 0 1599 { 1600 uint32_t ctx_status; 1601 1602 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, 1603 &ctx_status); 1604 printf("%s: endpc 0x%08x\n", __func__, 1605 ctx_status & UPC_MASK); 1606 } 1607 #endif 1608 1609 /* retore to previous states: */ 1610 /* disable context */ 1611 qat_ae_disable_ctx(sc, ae, 1 << ctx); 1612 if (ninst <= MAX_EXEC_INST) { 1613 /* instructions */ 1614 qat_ae_ucode_write(sc, ae, 0, ninst, savucode); 1615 } 1616 /* wakeup-events */ 1617 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT, 1618 wakeup_ev); 1619 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc); 1620 1621 /* only restore shared control store bit, 1622 other bit might be changed by AE code snippet */ 1623 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1624 if (share_cs) 1625 nmisc = misc | AE_MISC_CONTROL_SHARE_CS; 1626 else 1627 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 1628 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 1629 /* conditional-code */ 1630 qat_ae_write_4(sc, ae, CC_ENABLE, savcc); 1631 /* change the active context */ 1632 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1633 savctx & ACTIVE_CTX_STATUS_ACNO); 1634 /* restore the nxt ctx to run */ 1635 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 1636 /* restore current LM addr */ 1637 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT, 1638 indr_lm_addr_0); 1639 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT, 1640 indr_lm_addr_1); 1641 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, 1642 indr_lm_addr_byte_0); 1643 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, 1644 indr_lm_addr_byte_1); 1645 1646 /* restore indirect csrs */ 1647 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT, 1648 indr_future_cnt_sig); 1649 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 1650 indr_sig); 1651 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig); 1652 1653 /* ctx-enables */ 1654 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 1655 1656 return error; 1657 } 1658 1659 static int 1660 qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx, 1661 int *first_exec, uint64_t *ucode, u_int ninst, 1662 u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1) 1663 { 1664 1665 if (*first_exec) { 1666 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); 1667 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); 1668 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); 1669 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); 1670 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); 1671 *first_exec = 0; 1672 } 1673 1674 return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL); 1675 } 1676 1677 static int 1678 qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx, 1679 u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1) 1680 { 1681 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); 1682 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); 1683 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); 1684 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); 1685 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); 1686 1687 return 0; 1688 } 1689 1690 static int 1691 qat_ae_get_inst_num(int lmsize) 1692 { 1693 int ninst, left; 1694 1695 if (lmsize == 0) 1696 return 0; 1697 1698 left = lmsize % sizeof(u_int); 1699 1700 if (left) { 1701 ninst = nitems(ae_inst_1b) + 1702 qat_ae_get_inst_num(lmsize - left); 1703 } else { 1704 /* 3 instruction is needed for further code */ 1705 ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b); 1706 } 1707 1708 return (ninst); 1709 } 1710 1711 static int 1712 qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae, 1713 struct qat_ae_batch_init_list *qabi_list, size_t nqabi) 1714 { 1715 struct qat_ae_batch_init *qabi; 1716 size_t alloc_ninst, ninst; 1717 uint64_t *ucode; 1718 u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1; 1719 int insnsz, error = 0, execed = 0, first_exec = 1; 1720 1721 if (STAILQ_FIRST(qabi_list) == NULL) 1722 return 0; 1723 1724 alloc_ninst = min(USTORE_SIZE, nqabi); 1725 ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst); 1726 1727 ninst = 0; 1728 STAILQ_FOREACH(qabi, qabi_list, qabi_next) { 1729 insnsz = qat_ae_get_inst_num(qabi->qabi_size); 1730 if (insnsz + ninst > alloc_ninst) { 1731 /* add ctx_arb[kill] */ 1732 ucode[ninst++] = 0x0E000010000ull; 1733 execed = 1; 1734 1735 error = qat_ae_exec_ucode_init_lm(sc, ae, 0, 1736 &first_exec, ucode, ninst, 1737 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); 1738 if (error) { 1739 qat_ae_restore_init_lm_gprs(sc, ae, 0, 1740 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); 1741 qat_free_mem(ucode); 1742 return error; 1743 } 1744 /* run microExec to execute the microcode */ 1745 ninst = 0; 1746 } 1747 ninst += qat_ae_concat_ucode(ucode, ninst, 1748 qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value); 1749 } 1750 1751 if (ninst > 0) { 1752 ucode[ninst++] = 0x0E000010000ull; 1753 execed = 1; 1754 1755 error = qat_ae_exec_ucode_init_lm(sc, ae, 0, 1756 &first_exec, ucode, ninst, 1757 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); 1758 } 1759 if (execed) { 1760 qat_ae_restore_init_lm_gprs(sc, ae, 0, 1761 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); 1762 } 1763 1764 qat_free_mem(ucode); 1765 1766 return error; 1767 } 1768 1769 static int 1770 qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc) 1771 { 1772 1773 if (qat_ae_is_active(sc, ae)) 1774 return EBUSY; 1775 1776 qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT, 1777 UPC_MASK & upc); 1778 return 0; 1779 } 1780 1781 static inline u_int 1782 qat_aefw_csum_calc(u_int reg, int ch) 1783 { 1784 int i; 1785 u_int topbit = CRC_BITMASK(CRC_WIDTH - 1); 1786 u_int inbyte = (u_int)((reg >> 0x18) ^ ch); 1787 1788 reg ^= inbyte << (CRC_WIDTH - 0x8); 1789 for (i = 0; i < 0x8; i++) { 1790 if (reg & topbit) 1791 reg = (reg << 1) ^ CRC_POLY; 1792 else 1793 reg <<= 1; 1794 } 1795 1796 return (reg & CRC_WIDTHMASK(CRC_WIDTH)); 1797 } 1798 1799 static u_int 1800 qat_aefw_csum(char *buf, int size) 1801 { 1802 u_int csum = 0; 1803 1804 while (size--) { 1805 csum = qat_aefw_csum_calc(csum, *buf++); 1806 } 1807 1808 return csum; 1809 } 1810 1811 static const char * 1812 qat_aefw_uof_string(struct qat_softc *sc, size_t offset) 1813 { 1814 if (offset >= sc->sc_aefw_uof.qafu_str_tab_size) 1815 return NULL; 1816 if (sc->sc_aefw_uof.qafu_str_tab == NULL) 1817 return NULL; 1818 1819 return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset); 1820 } 1821 1822 static struct uof_chunk_hdr * 1823 qat_aefw_uof_find_chunk(struct qat_softc *sc, 1824 const char *id, struct uof_chunk_hdr *cur) 1825 { 1826 struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr; 1827 struct uof_chunk_hdr *uch; 1828 int i; 1829 1830 uch = (struct uof_chunk_hdr *)(uoh + 1); 1831 for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) { 1832 if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size) 1833 return NULL; 1834 1835 if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN)) 1836 return uch; 1837 } 1838 1839 return NULL; 1840 } 1841 1842 static int 1843 qat_aefw_load_mof(struct qat_softc *sc) 1844 { 1845 const struct firmware *fw; 1846 1847 fw = firmware_get(sc->sc_hw.qhw_mof_fwname); 1848 if (fw == NULL) { 1849 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", 1850 sc->sc_hw.qhw_mof_fwname); 1851 return ENXIO; 1852 } 1853 1854 sc->sc_fw_mof = qat_alloc_mem(fw->datasize); 1855 sc->sc_fw_mof_size = fw->datasize; 1856 memcpy(sc->sc_fw_mof, fw->data, fw->datasize); 1857 firmware_put(fw, FIRMWARE_UNLOAD); 1858 return 0; 1859 } 1860 1861 static void 1862 qat_aefw_unload_mof(struct qat_softc *sc) 1863 { 1864 if (sc->sc_fw_mof != NULL) { 1865 qat_free_mem(sc->sc_fw_mof); 1866 sc->sc_fw_mof = NULL; 1867 } 1868 } 1869 1870 static int 1871 qat_aefw_load_mmp(struct qat_softc *sc) 1872 { 1873 const struct firmware *fw; 1874 1875 fw = firmware_get(sc->sc_hw.qhw_mmp_fwname); 1876 if (fw == NULL) { 1877 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", 1878 sc->sc_hw.qhw_mmp_fwname); 1879 return ENXIO; 1880 } 1881 1882 sc->sc_fw_mmp = qat_alloc_mem(fw->datasize); 1883 sc->sc_fw_mmp_size = fw->datasize; 1884 memcpy(sc->sc_fw_mmp, fw->data, fw->datasize); 1885 firmware_put(fw, FIRMWARE_UNLOAD); 1886 return 0; 1887 } 1888 1889 static void 1890 qat_aefw_unload_mmp(struct qat_softc *sc) 1891 { 1892 if (sc->sc_fw_mmp != NULL) { 1893 qat_free_mem(sc->sc_fw_mmp); 1894 sc->sc_fw_mmp = NULL; 1895 } 1896 } 1897 1898 static int 1899 qat_aefw_mof_find_uof0(struct qat_softc *sc, 1900 struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head, 1901 u_int nchunk, size_t size, const char *id, 1902 size_t *fwsize, void **fwptr) 1903 { 1904 int i; 1905 char *uof_name; 1906 1907 for (i = 0; i < nchunk; i++) { 1908 struct mof_uof_chunk_hdr *much = &head[i]; 1909 1910 if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN)) 1911 return EINVAL; 1912 1913 if (much->much_offset + much->much_size > size) 1914 return EINVAL; 1915 1916 if (sc->sc_mof.qmf_sym_size <= much->much_name) 1917 return EINVAL; 1918 1919 uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym + 1920 much->much_name); 1921 1922 if (!strcmp(uof_name, sc->sc_fw_uof_name)) { 1923 *fwptr = (void *)((uintptr_t)muh + 1924 (uintptr_t)much->much_offset); 1925 *fwsize = (size_t)much->much_size; 1926 return 0; 1927 } 1928 } 1929 1930 return ENOENT; 1931 } 1932 1933 static int 1934 qat_aefw_mof_find_uof(struct qat_softc *sc) 1935 { 1936 struct mof_uof_hdr *uof_hdr, *suof_hdr; 1937 u_int nuof_chunks = 0, nsuof_chunks = 0; 1938 int error; 1939 1940 uof_hdr = sc->sc_mof.qmf_uof_objs; 1941 suof_hdr = sc->sc_mof.qmf_suof_objs; 1942 1943 if (uof_hdr != NULL) { 1944 if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) { 1945 return EINVAL; 1946 } 1947 nuof_chunks = uof_hdr->muh_num_chunks; 1948 } 1949 if (suof_hdr != NULL) { 1950 if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks) 1951 return EINVAL; 1952 nsuof_chunks = suof_hdr->muh_num_chunks; 1953 } 1954 1955 if (nuof_chunks + nsuof_chunks == 0) 1956 return EINVAL; 1957 1958 if (uof_hdr != NULL) { 1959 error = qat_aefw_mof_find_uof0(sc, uof_hdr, 1960 (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks, 1961 sc->sc_mof.qmf_uof_objs_size, UOF_IMAG, 1962 &sc->sc_fw_uof_size, &sc->sc_fw_uof); 1963 if (error && error != ENOENT) 1964 return error; 1965 } 1966 1967 if (suof_hdr != NULL) { 1968 error = qat_aefw_mof_find_uof0(sc, suof_hdr, 1969 (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks, 1970 sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG, 1971 &sc->sc_fw_suof_size, &sc->sc_fw_suof); 1972 if (error && error != ENOENT) 1973 return error; 1974 } 1975 1976 if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL) 1977 return ENOENT; 1978 1979 return 0; 1980 } 1981 1982 static int 1983 qat_aefw_mof_parse(struct qat_softc *sc) 1984 { 1985 const struct mof_file_hdr *mfh; 1986 const struct mof_file_chunk_hdr *mfch; 1987 size_t size; 1988 u_int csum; 1989 int error, i; 1990 1991 size = sc->sc_fw_mof_size; 1992 1993 if (size < sizeof(struct mof_file_hdr)) 1994 return EINVAL; 1995 size -= sizeof(struct mof_file_hdr); 1996 1997 mfh = sc->sc_fw_mof; 1998 1999 if (mfh->mfh_fid != MOF_FID) 2000 return EINVAL; 2001 2002 csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof + 2003 offsetof(struct mof_file_hdr, mfh_min_ver)), 2004 sc->sc_fw_mof_size - 2005 offsetof(struct mof_file_hdr, mfh_min_ver)); 2006 if (mfh->mfh_csum != csum) 2007 return EINVAL; 2008 2009 if (mfh->mfh_min_ver != MOF_MIN_VER || 2010 mfh->mfh_maj_ver != MOF_MAJ_VER) 2011 return EINVAL; 2012 2013 if (mfh->mfh_max_chunks < mfh->mfh_num_chunks) 2014 return EINVAL; 2015 2016 if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks) 2017 return EINVAL; 2018 mfch = (const struct mof_file_chunk_hdr *)(mfh + 1); 2019 2020 for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) { 2021 if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size) 2022 return EINVAL; 2023 2024 if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) { 2025 if (sc->sc_mof.qmf_sym != NULL) 2026 return EINVAL; 2027 2028 sc->sc_mof.qmf_sym = 2029 (void *)((uintptr_t)sc->sc_fw_mof + 2030 (uintptr_t)mfch->mfch_offset + sizeof(u_int)); 2031 sc->sc_mof.qmf_sym_size = 2032 *(u_int *)((uintptr_t)sc->sc_fw_mof + 2033 (uintptr_t)mfch->mfch_offset); 2034 2035 if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0) 2036 return EINVAL; 2037 if (mfch->mfch_size != sc->sc_mof.qmf_sym_size + 2038 sizeof(u_int) || mfch->mfch_size == 0) 2039 return EINVAL; 2040 if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym + 2041 sc->sc_mof.qmf_sym_size - 1) != '\0') 2042 return EINVAL; 2043 2044 } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) { 2045 if (sc->sc_mof.qmf_uof_objs != NULL) 2046 return EINVAL; 2047 2048 sc->sc_mof.qmf_uof_objs = 2049 (void *)((uintptr_t)sc->sc_fw_mof + 2050 (uintptr_t)mfch->mfch_offset); 2051 sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size; 2052 2053 } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) { 2054 if (sc->sc_mof.qmf_suof_objs != NULL) 2055 return EINVAL; 2056 2057 sc->sc_mof.qmf_suof_objs = 2058 (void *)((uintptr_t)sc->sc_fw_mof + 2059 (uintptr_t)mfch->mfch_offset); 2060 sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size; 2061 } 2062 } 2063 2064 if (sc->sc_mof.qmf_sym == NULL || 2065 (sc->sc_mof.qmf_uof_objs == NULL && 2066 sc->sc_mof.qmf_suof_objs == NULL)) 2067 return EINVAL; 2068 2069 error = qat_aefw_mof_find_uof(sc); 2070 if (error) 2071 return error; 2072 return 0; 2073 } 2074 2075 static int 2076 qat_aefw_uof_parse_image(struct qat_softc *sc, 2077 struct qat_uof_image *qui, struct uof_chunk_hdr *uch) 2078 { 2079 struct uof_image *image; 2080 struct uof_code_page *page; 2081 uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; 2082 size_t lim = uch->uch_offset + uch->uch_size, size; 2083 int i, p; 2084 2085 size = uch->uch_size; 2086 if (size < sizeof(struct uof_image)) 2087 return EINVAL; 2088 size -= sizeof(struct uof_image); 2089 2090 qui->qui_image = image = 2091 (struct uof_image *)(base + uch->uch_offset); 2092 2093 #define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \ 2094 do { \ 2095 u_int nent; \ 2096 nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\ 2097 if ((lim) < off + sizeof(struct uof_obj_table) + \ 2098 sizeof(type) * nent) \ 2099 return EINVAL; \ 2100 *(np) = nent; \ 2101 if (nent > 0) \ 2102 *(typep) = (type)((struct uof_obj_table *) \ 2103 ((base) + (off)) + 1); \ 2104 else \ 2105 *(typep) = NULL; \ 2106 } while (0) 2107 2108 ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg, 2109 struct uof_ae_reg *, base, image->ui_reg_tab, lim); 2110 ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym, 2111 struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim); 2112 ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak, 2113 struct qui_sbreak *, base, image->ui_sbreak_tab, lim); 2114 2115 if (size < sizeof(struct uof_code_page) * image->ui_num_pages) 2116 return EINVAL; 2117 if (nitems(qui->qui_pages) < image->ui_num_pages) 2118 return EINVAL; 2119 2120 page = (struct uof_code_page *)(image + 1); 2121 2122 for (p = 0; p < image->ui_num_pages; p++, page++) { 2123 struct qat_uof_page *qup = &qui->qui_pages[p]; 2124 struct uof_code_area *uca; 2125 2126 qup->qup_page_num = page->ucp_page_num; 2127 qup->qup_def_page = page->ucp_def_page; 2128 qup->qup_page_region = page->ucp_page_region; 2129 qup->qup_beg_vaddr = page->ucp_beg_vaddr; 2130 qup->qup_beg_paddr = page->ucp_beg_paddr; 2131 2132 ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var, 2133 struct uof_uword_fixup *, base, 2134 page->ucp_uc_var_tab, lim); 2135 ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var, 2136 struct uof_import_var *, base, 2137 page->ucp_imp_var_tab, lim); 2138 ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr, 2139 struct uof_uword_fixup *, base, 2140 page->ucp_imp_expr_tab, lim); 2141 ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg, 2142 struct uof_uword_fixup *, base, 2143 page->ucp_neigh_reg_tab, lim); 2144 2145 if (lim < page->ucp_code_area + sizeof(struct uof_code_area)) 2146 return EINVAL; 2147 2148 uca = (struct uof_code_area *)(base + page->ucp_code_area); 2149 qup->qup_num_micro_words = uca->uca_num_micro_words; 2150 2151 ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks, 2152 struct qat_uof_uword_block *, base, 2153 uca->uca_uword_block_tab, lim); 2154 2155 for (i = 0; i < qup->qup_num_uw_blocks; i++) { 2156 u_int uwordoff = ((struct uof_uword_block *)( 2157 &qup->qup_uw_blocks[i]))->uub_uword_offset; 2158 2159 if (lim < uwordoff) 2160 return EINVAL; 2161 2162 qup->qup_uw_blocks[i].quub_micro_words = 2163 (base + uwordoff); 2164 } 2165 } 2166 2167 #undef ASSIGN_OBJ_TAB 2168 2169 return 0; 2170 } 2171 2172 static int 2173 qat_aefw_uof_parse_images(struct qat_softc *sc) 2174 { 2175 struct uof_chunk_hdr *uch = NULL; 2176 int i, error; 2177 2178 for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) { 2179 uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch); 2180 if (uch == NULL) 2181 break; 2182 2183 if (i >= nitems(sc->sc_aefw_uof.qafu_imgs)) 2184 return ENOENT; 2185 2186 error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch); 2187 if (error) 2188 return error; 2189 2190 sc->sc_aefw_uof.qafu_num_imgs++; 2191 } 2192 2193 return 0; 2194 } 2195 2196 static int 2197 qat_aefw_uof_parse(struct qat_softc *sc) 2198 { 2199 struct uof_file_hdr *ufh; 2200 struct uof_file_chunk_hdr *ufch; 2201 struct uof_obj_hdr *uoh; 2202 struct uof_chunk_hdr *uch; 2203 void *uof = NULL; 2204 size_t size, uof_size, hdr_size; 2205 uintptr_t base; 2206 u_int csum; 2207 int i; 2208 2209 size = sc->sc_fw_uof_size; 2210 if (size < MIN_UOF_SIZE) 2211 return EINVAL; 2212 size -= sizeof(struct uof_file_hdr); 2213 2214 ufh = sc->sc_fw_uof; 2215 2216 if (ufh->ufh_id != UOF_FID) 2217 return EINVAL; 2218 if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER) 2219 return EINVAL; 2220 2221 if (ufh->ufh_max_chunks < ufh->ufh_num_chunks) 2222 return EINVAL; 2223 if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks) 2224 return EINVAL; 2225 ufch = (struct uof_file_chunk_hdr *)(ufh + 1); 2226 2227 uof_size = 0; 2228 for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) { 2229 if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size) 2230 return EINVAL; 2231 2232 if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) { 2233 if (uof != NULL) 2234 return EINVAL; 2235 2236 uof = 2237 (void *)((uintptr_t)sc->sc_fw_uof + 2238 ufch->ufch_offset); 2239 uof_size = ufch->ufch_size; 2240 2241 csum = qat_aefw_csum(uof, uof_size); 2242 if (csum != ufch->ufch_csum) 2243 return EINVAL; 2244 } 2245 } 2246 2247 if (uof == NULL) 2248 return ENOENT; 2249 2250 size = uof_size; 2251 if (size < sizeof(struct uof_obj_hdr)) 2252 return EINVAL; 2253 size -= sizeof(struct uof_obj_hdr); 2254 2255 uoh = uof; 2256 2257 if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks) 2258 return EINVAL; 2259 2260 /* Check if the UOF objects are compatible with the chip */ 2261 if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0) 2262 return ENOTSUP; 2263 2264 if (uoh->uoh_min_cpu_ver > sc->sc_rev || 2265 uoh->uoh_max_cpu_ver < sc->sc_rev) 2266 return ENOTSUP; 2267 2268 sc->sc_aefw_uof.qafu_size = uof_size; 2269 sc->sc_aefw_uof.qafu_obj_hdr = uoh; 2270 2271 base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; 2272 2273 /* map uof string-table */ 2274 uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL); 2275 if (uch != NULL) { 2276 hdr_size = offsetof(struct uof_str_tab, ust_strings); 2277 sc->sc_aefw_uof.qafu_str_tab = 2278 (void *)(base + uch->uch_offset + hdr_size); 2279 sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size; 2280 } 2281 2282 /* get ustore mem inits table -- should be only one */ 2283 uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL); 2284 if (uch != NULL) { 2285 if (uch->uch_size < sizeof(struct uof_obj_table)) 2286 return EINVAL; 2287 sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base + 2288 uch->uch_offset))->uot_nentries; 2289 if (sc->sc_aefw_uof.qafu_num_init_mem) { 2290 sc->sc_aefw_uof.qafu_init_mem = 2291 (struct uof_init_mem *)(base + uch->uch_offset + 2292 sizeof(struct uof_obj_table)); 2293 sc->sc_aefw_uof.qafu_init_mem_size = 2294 uch->uch_size - sizeof(struct uof_obj_table); 2295 } 2296 } 2297 2298 uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL); 2299 if (uch != NULL) { 2300 if (uch->uch_size < sizeof(struct uof_obj_table) + 2301 sizeof(struct uof_var_mem_seg)) 2302 return EINVAL; 2303 sc->sc_aefw_uof.qafu_var_mem_seg = 2304 (struct uof_var_mem_seg *)(base + uch->uch_offset + 2305 sizeof(struct uof_obj_table)); 2306 } 2307 2308 return qat_aefw_uof_parse_images(sc); 2309 } 2310 2311 static int 2312 qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi, 2313 struct suof_chunk_hdr *sch) 2314 { 2315 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; 2316 struct simg_ae_mode *ae_mode; 2317 u_int maj_ver; 2318 2319 qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset + 2320 sizeof(struct suof_obj_hdr); 2321 qsi->qsi_simg_len = 2322 ((struct suof_obj_hdr *) 2323 (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length; 2324 2325 qsi->qsi_css_header = qsi->qsi_simg_buf; 2326 qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr); 2327 qsi->qsi_css_signature = qsi->qsi_css_key + 2328 CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; 2329 qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN; 2330 2331 ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg; 2332 qsi->qsi_ae_mask = ae_mode->sam_ae_mask; 2333 qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name; 2334 qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data; 2335 qsi->qsi_fw_type = ae_mode->sam_fw_type; 2336 2337 if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type) 2338 return EINVAL; 2339 2340 maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff; 2341 if ((maj_ver > ae_mode->sam_devmax_ver) || 2342 (maj_ver < ae_mode->sam_devmin_ver)) { 2343 return EINVAL; 2344 } 2345 2346 return 0; 2347 } 2348 2349 static int 2350 qat_aefw_suof_parse(struct qat_softc *sc) 2351 { 2352 struct suof_file_hdr *sfh; 2353 struct suof_chunk_hdr *sch; 2354 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; 2355 struct qat_suof_image *qsi; 2356 size_t size; 2357 u_int csum; 2358 int ae0_img = MAX_AE; 2359 int i, error; 2360 2361 size = sc->sc_fw_suof_size; 2362 if (size < sizeof(struct suof_file_hdr)) 2363 return EINVAL; 2364 2365 sfh = sc->sc_fw_suof; 2366 2367 if (sfh->sfh_file_id != SUOF_FID) 2368 return EINVAL; 2369 if (sfh->sfh_fw_type != 0) 2370 return EINVAL; 2371 if (sfh->sfh_num_chunks <= 1) 2372 return EINVAL; 2373 if (sfh->sfh_min_ver != SUOF_MIN_VER || 2374 sfh->sfh_maj_ver != SUOF_MAJ_VER) 2375 return EINVAL; 2376 2377 csum = qat_aefw_csum((char *)&sfh->sfh_min_ver, 2378 size - offsetof(struct suof_file_hdr, sfh_min_ver)); 2379 if (csum != sfh->sfh_check_sum) 2380 return EINVAL; 2381 2382 size -= sizeof(struct suof_file_hdr); 2383 2384 qafs->qafs_file_id = SUOF_FID; 2385 qafs->qafs_suof_buf = sc->sc_fw_suof; 2386 qafs->qafs_suof_size = sc->sc_fw_suof_size; 2387 qafs->qafs_check_sum = sfh->sfh_check_sum; 2388 qafs->qafs_min_ver = sfh->sfh_min_ver; 2389 qafs->qafs_maj_ver = sfh->sfh_maj_ver; 2390 qafs->qafs_fw_type = sfh->sfh_fw_type; 2391 2392 if (size < sizeof(struct suof_chunk_hdr)) 2393 return EINVAL; 2394 sch = (struct suof_chunk_hdr *)(sfh + 1); 2395 size -= sizeof(struct suof_chunk_hdr); 2396 2397 if (size < sizeof(struct suof_str_tab)) 2398 return EINVAL; 2399 size -= offsetof(struct suof_str_tab, sst_strings); 2400 2401 qafs->qafs_sym_size = ((struct suof_str_tab *) 2402 (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length; 2403 if (size < qafs->qafs_sym_size) 2404 return EINVAL; 2405 qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset + 2406 offsetof(struct suof_str_tab, sst_strings); 2407 2408 qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1; 2409 if (qafs->qafs_num_simgs == 0) 2410 return EINVAL; 2411 2412 qsi = qat_alloc_mem( 2413 sizeof(struct qat_suof_image) * qafs->qafs_num_simgs); 2414 qafs->qafs_simg = qsi; 2415 2416 for (i = 0; i < qafs->qafs_num_simgs; i++) { 2417 error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]); 2418 if (error) 2419 return error; 2420 if ((qsi[i].qsi_ae_mask & 0x1) != 0) 2421 ae0_img = i; 2422 } 2423 2424 if (ae0_img != qafs->qafs_num_simgs - 1) { 2425 struct qat_suof_image last_qsi; 2426 2427 memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1], 2428 sizeof(struct qat_suof_image)); 2429 memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img], 2430 sizeof(struct qat_suof_image)); 2431 memcpy(&qsi[ae0_img], &last_qsi, 2432 sizeof(struct qat_suof_image)); 2433 } 2434 2435 return 0; 2436 } 2437 2438 static int 2439 qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size, 2440 struct qat_dmamem *dma) 2441 { 2442 struct css_hdr *css = (struct css_hdr *)image; 2443 struct auth_chunk *auth_chunk; 2444 struct fw_auth_desc *auth_desc; 2445 size_t mapsize, simg_offset = sizeof(struct auth_chunk); 2446 bus_size_t bus_addr; 2447 uintptr_t virt_addr; 2448 int error; 2449 2450 if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN) 2451 return EINVAL; 2452 2453 mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ? 2454 CSS_AE_SIMG_LEN + simg_offset : 2455 size + CSS_FWSK_PAD_LEN + simg_offset; 2456 error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE); 2457 if (error) 2458 return error; 2459 2460 memset(dma->qdm_dma_vaddr, 0, mapsize); 2461 2462 auth_chunk = dma->qdm_dma_vaddr; 2463 auth_chunk->ac_chunk_size = mapsize; 2464 auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr; 2465 2466 virt_addr = (uintptr_t)dma->qdm_dma_vaddr; 2467 virt_addr += simg_offset; 2468 bus_addr = auth_chunk->ac_chunk_bus_addr; 2469 bus_addr += simg_offset; 2470 2471 auth_desc = &auth_chunk->ac_fw_auth_desc; 2472 auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32; 2473 auth_desc->fad_css_hdr_low = bus_addr; 2474 2475 memcpy((void *)virt_addr, image, sizeof(struct css_hdr)); 2476 /* pub key */ 2477 virt_addr += sizeof(struct css_hdr); 2478 bus_addr += sizeof(struct css_hdr); 2479 image += sizeof(struct css_hdr); 2480 2481 auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32; 2482 auth_desc->fad_fwsk_pub_low = bus_addr; 2483 2484 memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN); 2485 memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN); 2486 memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN), 2487 image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t)); 2488 2489 virt_addr += CSS_FWSK_PUB_LEN; 2490 bus_addr += CSS_FWSK_PUB_LEN; 2491 image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; 2492 2493 auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32; 2494 auth_desc->fad_signature_low = bus_addr; 2495 2496 memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN); 2497 2498 virt_addr += CSS_SIGNATURE_LEN; 2499 bus_addr += CSS_SIGNATURE_LEN; 2500 image += CSS_SIGNATURE_LEN; 2501 2502 auth_desc->fad_img_high = (uint64_t)bus_addr >> 32; 2503 auth_desc->fad_img_low = bus_addr; 2504 auth_desc->fad_img_len = size - AE_IMG_OFFSET; 2505 2506 memcpy((void *)virt_addr, image, auth_desc->fad_img_len); 2507 2508 if (css->css_fw_type == CSS_AE_FIRMWARE) { 2509 auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high; 2510 auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low; 2511 2512 bus_addr += sizeof(struct simg_ae_mode); 2513 2514 auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32; 2515 auth_desc->fad_img_ae_init_data_low = bus_addr; 2516 2517 bus_addr += SIMG_AE_INIT_SEQ_LEN; 2518 2519 auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32; 2520 auth_desc->fad_img_ae_insts_low = bus_addr; 2521 } else { 2522 auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high; 2523 auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low; 2524 } 2525 2526 bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map, 2527 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2528 2529 return 0; 2530 } 2531 2532 static int 2533 qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma) 2534 { 2535 bus_addr_t addr; 2536 uint32_t fcu, sts; 2537 int retry = 0; 2538 2539 addr = dma->qdm_dma_seg.ds_addr; 2540 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32); 2541 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr); 2542 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH); 2543 2544 do { 2545 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2546 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2547 sts = __SHIFTOUT(fcu, FCU_STATUS_STS); 2548 if (sts == FCU_STATUS_STS_VERI_FAIL) 2549 goto fail; 2550 if (fcu & FCU_STATUS_AUTHFWLD && 2551 sts == FCU_STATUS_STS_VERI_DONE) { 2552 return 0; 2553 } 2554 } while (retry++ < FW_AUTH_MAX_RETRY); 2555 2556 fail: 2557 device_printf(sc->sc_dev, 2558 "firmware authentication error: status 0x%08x retry %d\n", 2559 fcu, retry); 2560 return EINVAL; 2561 } 2562 2563 static int 2564 qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma) 2565 { 2566 struct simg_ae_mode *ae_mode; 2567 uint32_t fcu, sts, loaded; 2568 u_int mask; 2569 u_char ae; 2570 int retry = 0; 2571 2572 ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr + 2573 sizeof(struct auth_chunk) + sizeof(struct css_hdr) + 2574 CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN); 2575 2576 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 2577 if (!(mask & 1)) 2578 continue; 2579 if (!((ae_mode->sam_ae_mask >> ae) & 0x1)) 2580 continue; 2581 if (qat_ae_is_active(sc, ae)) { 2582 device_printf(sc->sc_dev, "AE %d is active\n", ae); 2583 return EINVAL; 2584 } 2585 qat_cap_global_write_4(sc, FCU_CTRL, 2586 FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE)); 2587 do { 2588 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2589 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2590 sts = __SHIFTOUT(fcu, FCU_STATUS_STS); 2591 loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE); 2592 if (sts == FCU_STATUS_STS_LOAD_DONE && 2593 (loaded & (1 << ae))) { 2594 break; 2595 } 2596 } while (retry++ < FW_AUTH_MAX_RETRY); 2597 2598 if (retry > FW_AUTH_MAX_RETRY) { 2599 device_printf(sc->sc_dev, 2600 "firmware load timeout: status %08x\n", fcu); 2601 return EINVAL; 2602 } 2603 } 2604 2605 return 0; 2606 } 2607 2608 static int 2609 qat_aefw_suof_write(struct qat_softc *sc) 2610 { 2611 struct qat_suof_image *qsi; 2612 int i, error = 0; 2613 2614 for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) { 2615 qsi = &sc->sc_aefw_suof.qafs_simg[i]; 2616 error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf, 2617 qsi->qsi_simg_len, &qsi->qsi_dma); 2618 if (error) 2619 return error; 2620 error = qat_aefw_auth(sc, &qsi->qsi_dma); 2621 if (error) { 2622 qat_free_dmamem(sc, &qsi->qsi_dma); 2623 return error; 2624 } 2625 error = qat_aefw_suof_load(sc, &qsi->qsi_dma); 2626 if (error) { 2627 qat_free_dmamem(sc, &qsi->qsi_dma); 2628 return error; 2629 } 2630 qat_free_dmamem(sc, &qsi->qsi_dma); 2631 } 2632 qat_free_mem(sc->sc_aefw_suof.qafs_simg); 2633 2634 return 0; 2635 } 2636 2637 static int 2638 qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae, 2639 struct qat_uof_image *qui) 2640 { 2641 struct qat_ae_slice *slice; 2642 int i, npages, nregions; 2643 2644 if (qae->qae_num_slices >= nitems(qae->qae_slices)) 2645 return ENOENT; 2646 2647 if (qui->qui_image->ui_ae_mode & 2648 (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) { 2649 /* XXX */ 2650 device_printf(sc->sc_dev, 2651 "shared ae mode is not supported yet\n"); 2652 return ENOTSUP; 2653 } 2654 2655 qae->qae_shareable_ustore = 0; /* XXX */ 2656 qae->qae_effect_ustore_size = USTORE_SIZE; 2657 2658 slice = &qae->qae_slices[qae->qae_num_slices]; 2659 2660 slice->qas_image = qui; 2661 slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned; 2662 2663 nregions = qui->qui_image->ui_num_page_regions; 2664 npages = qui->qui_image->ui_num_pages; 2665 2666 if (nregions > nitems(slice->qas_regions)) 2667 return ENOENT; 2668 if (npages > nitems(slice->qas_pages)) 2669 return ENOENT; 2670 2671 for (i = 0; i < nregions; i++) { 2672 STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages); 2673 } 2674 for (i = 0; i < npages; i++) { 2675 struct qat_ae_page *page = &slice->qas_pages[i]; 2676 int region; 2677 2678 page->qap_page = &qui->qui_pages[i]; 2679 region = page->qap_page->qup_page_region; 2680 if (region >= nregions) 2681 return EINVAL; 2682 2683 page->qap_region = &slice->qas_regions[region]; 2684 } 2685 2686 qae->qae_num_slices++; 2687 2688 return 0; 2689 } 2690 2691 static int 2692 qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae) 2693 { 2694 struct uof_image *image; 2695 struct qat_ae *qae = &(QAT_AE(sc, ae)); 2696 int s; 2697 u_char nn_mode; 2698 2699 for (s = 0; s < qae->qae_num_slices; s++) { 2700 if (qae->qae_slices[s].qas_image == NULL) 2701 continue; 2702 2703 image = qae->qae_slices[s].qas_image->qui_image; 2704 qat_ae_write_ctx_mode(sc, ae, 2705 __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE)); 2706 2707 nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE); 2708 if (nn_mode != AE_MODE_NN_MODE_DONTCARE) 2709 qat_ae_write_nn_mode(sc, ae, nn_mode); 2710 2711 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0, 2712 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0)); 2713 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1, 2714 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1)); 2715 2716 qat_ae_write_shared_cs_mode(sc, ae, 2717 __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE)); 2718 qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size, 2719 __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED), 2720 qae->qae_reloc_ustore_dram); 2721 } 2722 2723 return 0; 2724 } 2725 2726 static int 2727 qat_aefw_uof_init(struct qat_softc *sc) 2728 { 2729 int ae, i, error; 2730 uint32_t mask; 2731 2732 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 2733 struct qat_ae *qae; 2734 2735 if (!(mask & 1)) 2736 continue; 2737 2738 qae = &(QAT_AE(sc, ae)); 2739 2740 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 2741 if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned & 2742 (1 << ae)) == 0) 2743 continue; 2744 2745 error = qat_aefw_uof_assign_image(sc, qae, 2746 &sc->sc_aefw_uof.qafu_imgs[i]); 2747 if (error) 2748 return error; 2749 } 2750 2751 /* XXX UcLo_initNumUwordUsed */ 2752 2753 qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */ 2754 2755 error = qat_aefw_uof_init_ae(sc, ae); 2756 if (error) 2757 return error; 2758 } 2759 2760 return 0; 2761 } 2762 2763 int 2764 qat_aefw_load(struct qat_softc *sc) 2765 { 2766 int error; 2767 2768 error = qat_aefw_load_mof(sc); 2769 if (error) 2770 return error; 2771 2772 error = qat_aefw_load_mmp(sc); 2773 if (error) 2774 return error; 2775 2776 error = qat_aefw_mof_parse(sc); 2777 if (error) { 2778 device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error); 2779 return error; 2780 } 2781 2782 if (sc->sc_hw.qhw_fw_auth) { 2783 error = qat_aefw_suof_parse(sc); 2784 if (error) { 2785 device_printf(sc->sc_dev, "couldn't parse suof: %d\n", 2786 error); 2787 return error; 2788 } 2789 2790 error = qat_aefw_suof_write(sc); 2791 if (error) { 2792 device_printf(sc->sc_dev, 2793 "could not write firmware: %d\n", error); 2794 return error; 2795 } 2796 2797 } else { 2798 error = qat_aefw_uof_parse(sc); 2799 if (error) { 2800 device_printf(sc->sc_dev, "couldn't parse uof: %d\n", 2801 error); 2802 return error; 2803 } 2804 2805 error = qat_aefw_uof_init(sc); 2806 if (error) { 2807 device_printf(sc->sc_dev, 2808 "couldn't init for aefw: %d\n", error); 2809 return error; 2810 } 2811 2812 error = qat_aefw_uof_write(sc); 2813 if (error) { 2814 device_printf(sc->sc_dev, 2815 "Could not write firmware: %d\n", error); 2816 return error; 2817 } 2818 } 2819 2820 return 0; 2821 } 2822 2823 void 2824 qat_aefw_unload(struct qat_softc *sc) 2825 { 2826 qat_aefw_unload_mmp(sc); 2827 qat_aefw_unload_mof(sc); 2828 } 2829 2830 int 2831 qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask) 2832 { 2833 uint32_t fcu; 2834 int retry = 0; 2835 2836 if (sc->sc_hw.qhw_fw_auth) { 2837 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START); 2838 do { 2839 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2840 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2841 if (fcu & FCU_STATUS_DONE) 2842 return 0; 2843 } while (retry++ < FW_AUTH_MAX_RETRY); 2844 2845 device_printf(sc->sc_dev, 2846 "firmware start timeout: status %08x\n", fcu); 2847 return EINVAL; 2848 } else { 2849 qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX, 2850 CTX_WAKEUP_EVENTS_INDIRECT, 2851 CTX_WAKEUP_EVENTS_INDIRECT_SLEEP); 2852 qat_ae_enable_ctx(sc, ae, ctx_mask); 2853 } 2854 2855 return 0; 2856 } 2857 2858 static int 2859 qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim) 2860 { 2861 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 2862 struct qat_ae_batch_init_list *qabi_list; 2863 struct uof_mem_val_attr *memattr; 2864 size_t *curinit; 2865 u_long ael; 2866 int i; 2867 const char *sym; 2868 char *ep; 2869 2870 memattr = (struct uof_mem_val_attr *)(uim + 1); 2871 2872 switch (uim->uim_region) { 2873 case LMEM_REGION: 2874 if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) { 2875 device_printf(sc->sc_dev, 2876 "Invalid lmem addr or bytes\n"); 2877 return ENOBUFS; 2878 } 2879 if (uim->uim_scope != UOF_SCOPE_LOCAL) 2880 return EINVAL; 2881 sym = qat_aefw_uof_string(sc, uim->uim_sym_name); 2882 ael = strtoul(sym, &ep, 10); 2883 if (ep == sym || ael > MAX_AE) 2884 return EINVAL; 2885 if ((sc->sc_ae_mask & (1 << ael)) == 0) 2886 return 0; /* ae is fused out */ 2887 2888 curinit = &qafu->qafu_num_lm_init[ael]; 2889 qabi_list = &qafu->qafu_lm_init[ael]; 2890 2891 for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) { 2892 struct qat_ae_batch_init *qabi; 2893 2894 qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init)); 2895 if (*curinit == 0) 2896 STAILQ_INIT(qabi_list); 2897 STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next); 2898 2899 qabi->qabi_ae = (u_int)ael; 2900 qabi->qabi_addr = 2901 uim->uim_addr + memattr->umva_byte_offset; 2902 qabi->qabi_value = &memattr->umva_value; 2903 qabi->qabi_size = 4; 2904 qafu->qafu_num_lm_init_inst[ael] += 2905 qat_ae_get_inst_num(qabi->qabi_size); 2906 (*curinit)++; 2907 if (*curinit >= MAX_LMEM_REG) { 2908 device_printf(sc->sc_dev, 2909 "Invalid lmem val attr\n"); 2910 return ENOBUFS; 2911 } 2912 } 2913 break; 2914 case SRAM_REGION: 2915 case DRAM_REGION: 2916 case DRAM1_REGION: 2917 case SCRATCH_REGION: 2918 case UMEM_REGION: 2919 /* XXX */ 2920 /* fallthrough */ 2921 default: 2922 device_printf(sc->sc_dev, 2923 "unsupported memory region to init: %d\n", 2924 uim->uim_region); 2925 return ENOTSUP; 2926 } 2927 2928 return 0; 2929 } 2930 2931 static void 2932 qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae) 2933 { 2934 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 2935 struct qat_ae_batch_init *qabi; 2936 2937 while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) { 2938 STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next); 2939 qat_free_mem(qabi); 2940 } 2941 2942 qafu->qafu_num_lm_init[ae] = 0; 2943 qafu->qafu_num_lm_init_inst[ae] = 0; 2944 } 2945 2946 static int 2947 qat_aefw_init_ustore(struct qat_softc *sc) 2948 { 2949 uint64_t *fill; 2950 uint32_t dont_init; 2951 int a, i, p; 2952 int error = 0; 2953 int usz, end, start; 2954 u_char ae, nae; 2955 2956 fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t)); 2957 2958 for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) { 2959 struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a]; 2960 struct uof_image *ui = qui->qui_image; 2961 2962 for (i = 0; i < MAX_USTORE; i++) 2963 memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t)); 2964 /* 2965 * Compute do_not_init value as a value that will not be equal 2966 * to fill data when cast to an int 2967 */ 2968 dont_init = 0; 2969 if (dont_init == (uint32_t)fill[0]) 2970 dont_init = 0xffffffff; 2971 2972 for (p = 0; p < ui->ui_num_pages; p++) { 2973 struct qat_uof_page *qup = &qui->qui_pages[p]; 2974 if (!qup->qup_def_page) 2975 continue; 2976 2977 for (i = qup->qup_beg_paddr; 2978 i < qup->qup_beg_paddr + qup->qup_num_micro_words; 2979 i++ ) { 2980 fill[i] = (uint64_t)dont_init; 2981 } 2982 } 2983 2984 for (ae = 0; ae < sc->sc_ae_num; ae++) { 2985 MPASS(ae < UOF_MAX_NUM_OF_AE); 2986 if ((ui->ui_ae_assigned & (1 << ae)) == 0) 2987 continue; 2988 2989 if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) { 2990 qat_ae_get_shared_ustore_ae(ae, &nae); 2991 if (ui->ui_ae_assigned & (1 << ae)) 2992 continue; 2993 } 2994 usz = QAT_AE(sc, ae).qae_effect_ustore_size; 2995 2996 /* initialize the areas not going to be overwritten */ 2997 end = -1; 2998 do { 2999 /* find next uword that needs to be initialized */ 3000 for (start = end + 1; start < usz; start++) { 3001 if ((uint32_t)fill[start] != dont_init) 3002 break; 3003 } 3004 /* see if there are no more such uwords */ 3005 if (start >= usz) 3006 break; 3007 for (end = start + 1; end < usz; end++) { 3008 if ((uint32_t)fill[end] == dont_init) 3009 break; 3010 } 3011 if (QAT_AE(sc, ae).qae_shareable_ustore) { 3012 error = ENOTSUP; /* XXX */ 3013 goto out; 3014 } else { 3015 error = qat_ae_ucode_write(sc, ae, 3016 start, end - start, &fill[start]); 3017 if (error) { 3018 goto out; 3019 } 3020 } 3021 3022 } while (end < usz); 3023 } 3024 } 3025 3026 out: 3027 qat_free_mem(fill); 3028 return error; 3029 } 3030 3031 static int 3032 qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask, 3033 enum aereg_type regtype, u_short regaddr, u_int value) 3034 { 3035 int error = 0; 3036 u_char ctx; 3037 3038 switch (regtype) { 3039 case AEREG_GPA_REL: 3040 case AEREG_GPB_REL: 3041 case AEREG_SR_REL: 3042 case AEREG_SR_RD_REL: 3043 case AEREG_SR_WR_REL: 3044 case AEREG_DR_REL: 3045 case AEREG_DR_RD_REL: 3046 case AEREG_DR_WR_REL: 3047 case AEREG_NEIGH_REL: 3048 /* init for all valid ctx */ 3049 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { 3050 if ((ctx_mask & (1 << ctx)) == 0) 3051 continue; 3052 error = qat_aereg_rel_data_write(sc, ae, ctx, regtype, 3053 regaddr, value); 3054 } 3055 break; 3056 case AEREG_GPA_ABS: 3057 case AEREG_GPB_ABS: 3058 case AEREG_SR_ABS: 3059 case AEREG_SR_RD_ABS: 3060 case AEREG_SR_WR_ABS: 3061 case AEREG_DR_ABS: 3062 case AEREG_DR_RD_ABS: 3063 case AEREG_DR_WR_ABS: 3064 error = qat_aereg_abs_data_write(sc, ae, regtype, 3065 regaddr, value); 3066 break; 3067 default: 3068 error = EINVAL; 3069 break; 3070 } 3071 3072 return error; 3073 } 3074 3075 static int 3076 qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae, 3077 struct qat_uof_image *qui) 3078 { 3079 u_int i, expres; 3080 u_char ctx_mask; 3081 3082 for (i = 0; i < qui->qui_num_init_reg_sym; i++) { 3083 struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i]; 3084 3085 if (uirs->uirs_value_type == EXPR_VAL) { 3086 /* XXX */ 3087 device_printf(sc->sc_dev, 3088 "does not support initializing EXPR_VAL\n"); 3089 return ENOTSUP; 3090 } else { 3091 expres = uirs->uirs_value; 3092 } 3093 3094 switch (uirs->uirs_init_type) { 3095 case INIT_REG: 3096 if (__SHIFTOUT(qui->qui_image->ui_ae_mode, 3097 AE_MODE_CTX_MODE) == MAX_AE_CTX) { 3098 ctx_mask = 0xff; /* 8-ctx mode */ 3099 } else { 3100 ctx_mask = 0x55; /* 4-ctx mode */ 3101 } 3102 qat_aefw_init_reg(sc, ae, ctx_mask, 3103 (enum aereg_type)uirs->uirs_reg_type, 3104 (u_short)uirs->uirs_addr_offset, expres); 3105 break; 3106 case INIT_REG_CTX: 3107 if (__SHIFTOUT(qui->qui_image->ui_ae_mode, 3108 AE_MODE_CTX_MODE) == MAX_AE_CTX) { 3109 ctx_mask = 0xff; /* 8-ctx mode */ 3110 } else { 3111 ctx_mask = 0x55; /* 4-ctx mode */ 3112 } 3113 if (((1 << uirs->uirs_ctx) & ctx_mask) == 0) 3114 return EINVAL; 3115 qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx, 3116 (enum aereg_type)uirs->uirs_reg_type, 3117 (u_short)uirs->uirs_addr_offset, expres); 3118 break; 3119 case INIT_EXPR: 3120 case INIT_EXPR_ENDIAN_SWAP: 3121 default: 3122 device_printf(sc->sc_dev, 3123 "does not support initializing init_type %d\n", 3124 uirs->uirs_init_type); 3125 return ENOTSUP; 3126 } 3127 } 3128 3129 return 0; 3130 } 3131 3132 static int 3133 qat_aefw_init_memory(struct qat_softc *sc) 3134 { 3135 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 3136 size_t uimsz, initmemsz = qafu->qafu_init_mem_size; 3137 struct uof_init_mem *uim; 3138 int error, i; 3139 u_char ae; 3140 3141 uim = qafu->qafu_init_mem; 3142 for (i = 0; i < qafu->qafu_num_init_mem; i++) { 3143 uimsz = sizeof(struct uof_init_mem) + 3144 sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr; 3145 if (uimsz > initmemsz) { 3146 device_printf(sc->sc_dev, 3147 "invalid uof_init_mem or uof_mem_val_attr size\n"); 3148 return EINVAL; 3149 } 3150 3151 if (uim->uim_num_bytes > 0) { 3152 error = qat_aefw_init_memory_one(sc, uim); 3153 if (error) { 3154 device_printf(sc->sc_dev, 3155 "Could not init ae memory: %d\n", error); 3156 return error; 3157 } 3158 } 3159 uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz); 3160 initmemsz -= uimsz; 3161 } 3162 3163 /* run Batch put LM API */ 3164 for (ae = 0; ae < MAX_AE; ae++) { 3165 error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae], 3166 qafu->qafu_num_lm_init_inst[ae]); 3167 if (error) 3168 device_printf(sc->sc_dev, "Could not put lm\n"); 3169 3170 qat_aefw_free_lm_init(sc, ae); 3171 } 3172 3173 error = qat_aefw_init_ustore(sc); 3174 3175 /* XXX run Batch put LM API */ 3176 3177 return error; 3178 } 3179 3180 static int 3181 qat_aefw_init_globals(struct qat_softc *sc) 3182 { 3183 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 3184 int error, i, p, s; 3185 u_char ae; 3186 3187 /* initialize the memory segments */ 3188 if (qafu->qafu_num_init_mem > 0) { 3189 error = qat_aefw_init_memory(sc); 3190 if (error) 3191 return error; 3192 } else { 3193 error = qat_aefw_init_ustore(sc); 3194 if (error) 3195 return error; 3196 } 3197 3198 /* XXX bind import variables with ivd values */ 3199 3200 /* XXX bind the uC global variables 3201 * local variables will done on-the-fly */ 3202 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 3203 for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) { 3204 struct qat_uof_page *qup = 3205 &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p]; 3206 if (qup->qup_num_uw_blocks && 3207 (qup->qup_num_uc_var || qup->qup_num_imp_var)) { 3208 device_printf(sc->sc_dev, 3209 "not support uC global variables\n"); 3210 return ENOTSUP; 3211 } 3212 } 3213 } 3214 3215 for (ae = 0; ae < sc->sc_ae_num; ae++) { 3216 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3217 3218 for (s = 0; s < qae->qae_num_slices; s++) { 3219 struct qat_ae_slice *qas = &qae->qae_slices[s]; 3220 3221 if (qas->qas_image == NULL) 3222 continue; 3223 3224 error = 3225 qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image); 3226 if (error) 3227 return error; 3228 } 3229 } 3230 3231 return 0; 3232 } 3233 3234 static uint64_t 3235 qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup, 3236 u_int addr) 3237 { 3238 uint64_t uinst = 0; 3239 u_int i; 3240 3241 /* find the block */ 3242 for (i = 0; i < qup->qup_num_uw_blocks; i++) { 3243 struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i]; 3244 3245 if ((addr >= quub->quub_start_addr) && 3246 (addr <= (quub->quub_start_addr + 3247 (quub->quub_num_words - 1)))) { 3248 /* unpack n bytes and assigned to the 64-bit uword value. 3249 note: the microwords are stored as packed bytes. 3250 */ 3251 addr -= quub->quub_start_addr; 3252 addr *= AEV2_PACKED_UWORD_BYTES; 3253 memcpy(&uinst, 3254 (void *)((uintptr_t)quub->quub_micro_words + addr), 3255 AEV2_PACKED_UWORD_BYTES); 3256 uinst = uinst & UWORD_MASK; 3257 3258 return uinst; 3259 } 3260 } 3261 3262 return INVLD_UWORD; 3263 } 3264 3265 static int 3266 qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup) 3267 { 3268 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3269 uint64_t fill, *ucode_cpybuf; 3270 u_int error, i, upaddr, ninst, cpylen; 3271 3272 if (qup->qup_num_uc_var || qup->qup_num_neigh_reg || 3273 qup->qup_num_imp_var || qup->qup_num_imp_expr) { 3274 device_printf(sc->sc_dev, 3275 "does not support fixup locals\n"); 3276 return ENOTSUP; 3277 } 3278 3279 ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t)); 3280 3281 /* XXX get fill-pattern from an image -- they are all the same */ 3282 memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern, 3283 sizeof(uint64_t)); 3284 3285 upaddr = qup->qup_beg_paddr; 3286 ninst = qup->qup_num_micro_words; 3287 while (ninst > 0) { 3288 cpylen = min(ninst, UWORD_CPYBUF_SIZE); 3289 3290 /* load the buffer */ 3291 for (i = 0; i < cpylen; i++) { 3292 /* keep below code structure in case there are 3293 * different handling for shared secnarios */ 3294 if (!qae->qae_shareable_ustore) { 3295 /* qat_aefw_get_uof_inst() takes an address that 3296 * is relative to the start of the page. 3297 * So we don't need to add in the physical 3298 * offset of the page. */ 3299 if (qup->qup_page_region != 0) { 3300 /* XXX */ 3301 device_printf(sc->sc_dev, 3302 "region != 0 is not supported\n"); 3303 qat_free_mem(ucode_cpybuf); 3304 return ENOTSUP; 3305 } else { 3306 /* for mixing case, it should take 3307 * physical address */ 3308 ucode_cpybuf[i] = qat_aefw_get_uof_inst( 3309 sc, qup, upaddr + i); 3310 if (ucode_cpybuf[i] == INVLD_UWORD) { 3311 /* fill hole in the uof */ 3312 ucode_cpybuf[i] = fill; 3313 } 3314 } 3315 } else { 3316 /* XXX */ 3317 qat_free_mem(ucode_cpybuf); 3318 return ENOTSUP; 3319 } 3320 } 3321 3322 /* copy the buffer to ustore */ 3323 if (!qae->qae_shareable_ustore) { 3324 error = qat_ae_ucode_write(sc, ae, upaddr, cpylen, 3325 ucode_cpybuf); 3326 if (error) 3327 return error; 3328 } else { 3329 /* XXX */ 3330 qat_free_mem(ucode_cpybuf); 3331 return ENOTSUP; 3332 } 3333 upaddr += cpylen; 3334 ninst -= cpylen; 3335 } 3336 3337 qat_free_mem(ucode_cpybuf); 3338 3339 return 0; 3340 } 3341 3342 static int 3343 qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui) 3344 { 3345 struct uof_image *ui = qui->qui_image; 3346 struct qat_ae_page *qap; 3347 u_int s, p, c; 3348 int error; 3349 u_char ae, ctx_mask; 3350 3351 if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) 3352 ctx_mask = 0xff; /* 8-ctx mode */ 3353 else 3354 ctx_mask = 0x55; /* 4-ctx mode */ 3355 3356 /* load the default page and set assigned CTX PC 3357 * to the entrypoint address */ 3358 for (ae = 0; ae < sc->sc_ae_num; ae++) { 3359 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3360 struct qat_ae_slice *qas; 3361 u_int metadata; 3362 3363 MPASS(ae < UOF_MAX_NUM_OF_AE); 3364 3365 if ((ui->ui_ae_assigned & (1 << ae)) == 0) 3366 continue; 3367 3368 /* find the slice to which this image is assigned */ 3369 for (s = 0; s < qae->qae_num_slices; s++) { 3370 qas = &qae->qae_slices[s]; 3371 if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask) 3372 break; 3373 } 3374 if (s >= qae->qae_num_slices) 3375 continue; 3376 3377 qas = &qae->qae_slices[s]; 3378 3379 for (p = 0; p < ui->ui_num_pages; p++) { 3380 qap = &qas->qas_pages[p]; 3381 3382 /* Only load pages loaded by default */ 3383 if (!qap->qap_page->qup_def_page) 3384 continue; 3385 3386 error = qat_aefw_do_pagein(sc, ae, qap->qap_page); 3387 if (error) 3388 return error; 3389 } 3390 3391 metadata = qas->qas_image->qui_image->ui_app_metadata; 3392 if (metadata != 0xffffffff && bootverbose) { 3393 device_printf(sc->sc_dev, 3394 "loaded firmware: %s\n", 3395 qat_aefw_uof_string(sc, metadata)); 3396 } 3397 3398 /* Assume starting page is page 0 */ 3399 qap = &qas->qas_pages[0]; 3400 for (c = 0; c < MAX_AE_CTX; c++) { 3401 if (ctx_mask & (1 << c)) 3402 qas->qas_cur_pages[c] = qap; 3403 else 3404 qas->qas_cur_pages[c] = NULL; 3405 } 3406 3407 /* set the live context */ 3408 qae->qae_live_ctx_mask = ui->ui_ctx_assigned; 3409 3410 /* set context PC to the image entrypoint address */ 3411 error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned, 3412 ui->ui_entry_address); 3413 if (error) 3414 return error; 3415 } 3416 3417 /* XXX store the checksum for convenience */ 3418 3419 return 0; 3420 } 3421 3422 static int 3423 qat_aefw_uof_write(struct qat_softc *sc) 3424 { 3425 int error = 0; 3426 int i; 3427 3428 error = qat_aefw_init_globals(sc); 3429 if (error) { 3430 device_printf(sc->sc_dev, 3431 "Could not initialize globals\n"); 3432 return error; 3433 } 3434 3435 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 3436 error = qat_aefw_uof_write_one(sc, 3437 &sc->sc_aefw_uof.qafu_imgs[i]); 3438 if (error) 3439 break; 3440 } 3441 3442 /* XXX UcLo_computeFreeUstore */ 3443 3444 return error; 3445 } 3446