/* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */ /* $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); #endif #include #include #include #include #include #include #include #include #include "qatreg.h" #include "qatvar.h" #include "qat_aevar.h" static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t, uint32_t); static int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t, uint32_t *); static void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t, bus_size_t, uint32_t); static int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t, bus_size_t, uint32_t *); static u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short); static int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char, enum aereg_type, u_short, uint32_t); static int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char, enum aereg_type, u_short, uint32_t *); static int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char, enum aereg_type, u_short, uint32_t); static int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char, enum aereg_type, u_short, uint32_t); static int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char, enum aereg_type, u_short, uint32_t); static int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short, u_short *, u_char *); static int qat_aereg_abs_data_write(struct qat_softc *, u_char, enum aereg_type, u_short, uint32_t); static void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int); static void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int); static void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char); static void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char); static void qat_ae_write_lm_mode(struct qat_softc *, u_char, enum aereg_type, u_char); static void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char, u_char); static void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char); static int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int, u_int); static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char); static int qat_ae_is_active(struct qat_softc *, u_char); static int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int); static int qat_ae_clear_reset(struct qat_softc *); static int qat_ae_check(struct qat_softc *); static int qat_ae_reset_timestamp(struct qat_softc *); static void qat_ae_clear_xfer(struct qat_softc *); static int qat_ae_clear_gprs(struct qat_softc *); static void qat_ae_get_shared_ustore_ae(u_char, u_char *); static u_int qat_ae_ucode_parity64(uint64_t); static uint64_t qat_ae_ucode_set_ecc(uint64_t); static int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int, const uint64_t *); static int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int, uint64_t *); static u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *); static int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char, uint64_t *, u_int, int, u_int, u_int *); static int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char, int *, uint64_t *, u_int, u_int *, u_int *, u_int *, u_int *, u_int *); static int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char, u_int, u_int, u_int, u_int, u_int); static int qat_ae_get_inst_num(int); static int qat_ae_batch_put_lm(struct qat_softc *, u_char, struct qat_ae_batch_init_list *, size_t); static int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int); static u_int qat_aefw_csum(char *, int); static const char *qat_aefw_uof_string(struct qat_softc *, size_t); static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *, const char *, struct uof_chunk_hdr *); static int qat_aefw_load_mof(struct qat_softc *); static void qat_aefw_unload_mof(struct qat_softc *); static int qat_aefw_load_mmp(struct qat_softc *); static void qat_aefw_unload_mmp(struct qat_softc *); static int qat_aefw_mof_find_uof0(struct qat_softc *, struct mof_uof_hdr *, struct mof_uof_chunk_hdr *, u_int, size_t, const char *, size_t *, void **); static int qat_aefw_mof_find_uof(struct qat_softc *); static int qat_aefw_mof_parse(struct qat_softc *); static int qat_aefw_uof_parse_image(struct qat_softc *, struct qat_uof_image *, struct uof_chunk_hdr *uch); static int qat_aefw_uof_parse_images(struct qat_softc *); static int qat_aefw_uof_parse(struct qat_softc *); static int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t, struct qat_dmamem *); static int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *); static int qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma); static int qat_aefw_suof_parse_image(struct qat_softc *, struct qat_suof_image *, struct suof_chunk_hdr *); static int qat_aefw_suof_parse(struct qat_softc *); static int qat_aefw_suof_write(struct qat_softc *); static int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *, struct qat_uof_image *); static int qat_aefw_uof_init_ae(struct qat_softc *, u_char); static int qat_aefw_uof_init(struct qat_softc *); static int qat_aefw_init_memory_one(struct qat_softc *, struct uof_init_mem *); static void qat_aefw_free_lm_init(struct qat_softc *, u_char); static int qat_aefw_init_ustore(struct qat_softc *); static int qat_aefw_init_reg(struct qat_softc *, u_char, u_char, enum aereg_type, u_short, u_int); static int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char, struct qat_uof_image *); static int qat_aefw_init_memory(struct qat_softc *); static int qat_aefw_init_globals(struct qat_softc *); static uint64_t qat_aefw_get_uof_inst(struct qat_softc *, struct qat_uof_page *, u_int); static int qat_aefw_do_pagein(struct qat_softc *, u_char, struct qat_uof_page *); static int qat_aefw_uof_write_one(struct qat_softc *, struct qat_uof_image *); static int qat_aefw_uof_write(struct qat_softc *); static int qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, uint32_t value) { int times = TIMEOUT_AE_CSR; do { qat_ae_local_write_4(sc, ae, offset, value); if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & LOCAL_CSR_STATUS_STATUS) == 0) return 0; } while (times--); device_printf(sc->sc_dev, "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); return EFAULT; } static int qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset, uint32_t *value) { int times = TIMEOUT_AE_CSR; uint32_t v; do { v = qat_ae_local_read_4(sc, ae, offset); if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & LOCAL_CSR_STATUS_STATUS) == 0) { *value = v; return 0; } } while (times--); device_printf(sc->sc_dev, "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); return EFAULT; } static void qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask, bus_size_t offset, uint32_t value) { int ctx; uint32_t ctxptr; MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || offset == FUTURE_COUNT_SIGNAL_INDIRECT || offset == CTX_STS_INDIRECT || offset == CTX_WAKEUP_EVENTS_INDIRECT || offset == CTX_SIG_EVENTS_INDIRECT || offset == LM_ADDR_0_INDIRECT || offset == LM_ADDR_1_INDIRECT || offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { if ((ctx_mask & (1 << ctx)) == 0) continue; qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); qat_ae_write_4(sc, ae, offset, value); } qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); } static int qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx, bus_size_t offset, uint32_t *value) { int error; uint32_t ctxptr; MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || offset == FUTURE_COUNT_SIGNAL_INDIRECT || offset == CTX_STS_INDIRECT || offset == CTX_WAKEUP_EVENTS_INDIRECT || offset == CTX_SIG_EVENTS_INDIRECT || offset == LM_ADDR_0_INDIRECT || offset == LM_ADDR_1_INDIRECT || offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); /* save the ctx ptr */ qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != (ctx & CSR_CTX_POINTER_CONTEXT)) qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); error = qat_ae_read_4(sc, ae, offset, value); /* restore ctx ptr */ if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != (ctx & CSR_CTX_POINTER_CONTEXT)) qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); return error; } static u_short qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg) { u_short addr; switch (regtype) { case AEREG_GPA_ABS: case AEREG_GPB_ABS: addr = (reg & 0x7f) | 0x80; break; case AEREG_GPA_REL: case AEREG_GPB_REL: addr = reg & 0x1f; break; case AEREG_SR_RD_REL: case AEREG_SR_WR_REL: case AEREG_SR_REL: addr = 0x180 | (reg & 0x1f); break; case AEREG_SR_INDX: addr = 0x140 | ((reg & 0x3) << 1); break; case AEREG_DR_RD_REL: case AEREG_DR_WR_REL: case AEREG_DR_REL: addr = 0x1c0 | (reg & 0x1f); break; case AEREG_DR_INDX: addr = 0x100 | ((reg & 0x3) << 1); break; case AEREG_NEIGH_INDX: addr = 0x241 | ((reg & 0x3) << 1); break; case AEREG_NEIGH_REL: addr = 0x280 | (reg & 0x1f); break; case AEREG_LMEM0: addr = 0x200; break; case AEREG_LMEM1: addr = 0x220; break; case AEREG_NO_DEST: addr = 0x300 | (reg & 0xff); break; default: addr = AEREG_BAD_REGADDR; break; } return (addr); } static int qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx, enum aereg_type regtype, u_short relreg, uint32_t value) { uint16_t srchi, srclo, destaddr, data16hi, data16lo; uint64_t inst[] = { 0x0F440000000ull, /* immed_w1[reg, val_hi16] */ 0x0F040000000ull, /* immed_w0[reg, val_lo16] */ 0x0F0000C0300ull, /* nop */ 0x0E000010000ull /* ctx_arb[kill] */ }; const int ninst = nitems(inst); const int imm_w1 = 0, imm_w0 = 1; unsigned int ctxen; uint16_t mask; /* This logic only works for GPRs and LM index registers, not NN or XFER registers! */ MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) { /* determine the context mode */ qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { /* 4-ctx mode */ if (ctx & 0x1) return EINVAL; mask = 0x1f; } else { /* 8-ctx mode */ mask = 0x0f; } if (relreg & ~mask) return EINVAL; } if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == AEREG_BAD_REGADDR) { return EINVAL; } data16lo = 0xffff & value; data16hi = 0xffff & (value >> 16); srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST, (uint16_t)(0xff & data16hi)); srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST, (uint16_t)(0xff & data16lo)); switch (regtype) { case AEREG_GPA_REL: /* A rel source */ inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff); inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff); break; default: inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff); inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff); break; } return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL); } static int qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx, enum aereg_type regtype, u_short relreg, uint32_t *value) { uint64_t inst, savucode; uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi; u_int uaddr, ustore_addr; int error; u_short mask, regaddr; u_char nae; MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL || regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) || (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) || (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL)) { /* determine the context mode */ qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { /* 4-ctx mode */ if (ctx & 0x1) return EINVAL; mask = 0x1f; } else { /* 8-ctx mode */ mask = 0x0f; } if (relreg & ~mask) return EINVAL; } if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == AEREG_BAD_REGADDR) { return EINVAL; } /* instruction -- alu[--, --, B, reg] */ switch (regtype) { case AEREG_GPA_REL: /* A rel source */ inst = 0xA070000000ull | (regaddr & 0x3ff); break; default: inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10)); break; } /* backup shared control store bit, and force AE to * none-shared mode before executing ucode snippet */ qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); if (misc & AE_MISC_CONTROL_SHARE_CS) { qat_ae_get_shared_ustore_ae(ae, &nae); if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae)) return EBUSY; } nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); /* read current context */ qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); /* prevent clearing the W1C bits: the breakpoint bit, ECC error bit, and Parity error bit */ ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; /* change the context */ if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, ctx & ACTIVE_CTX_STATUS_ACNO); /* save a ustore location */ if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) { /* restore AE_MISC_CONTROL csr */ qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); /* restore the context */ if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, savctx & ACTIVE_CTX_STATUS_ACNO); } qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); return (error); } /* turn off ustore parity */ qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE)); /* save ustore-addr csr */ qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); /* write the ALU instruction to ustore, enable ecs bit */ uaddr = 0 | USTORE_ADDRESS_ECS; /* set the uaddress */ qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); inst = qat_ae_ucode_set_ecc(inst); ulo = (uint32_t)(inst & 0xffffffff); uhi = (uint32_t)(inst >> 32); qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); /* this will auto increment the address */ qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); /* set the uaddress */ qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); /* delay for at least 8 cycles */ qat_ae_wait_num_cycles(sc, ae, 0x8, 0); /* read ALU output -- the instruction should have been executed prior to clearing the ECS in putUwords */ qat_ae_read_4(sc, ae, ALU_OUT, value); /* restore ustore-addr csr */ qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); /* restore the ustore */ error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode); /* restore the context */ if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, savctx & ACTIVE_CTX_STATUS_ACNO); } qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); /* restore AE_MISC_CONTROL csr */ qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); return error; } static int qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, enum aereg_type regtype, u_short relreg, uint32_t value) { bus_size_t addr; int error; uint32_t ctxen; u_short mask; u_short dr_offset; MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL || regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL); error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { if (ctx & 0x1) { device_printf(sc->sc_dev, "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx); return EINVAL; } mask = 0x1f; dr_offset = 0x20; } else { mask = 0x0f; dr_offset = 0x10; } if (relreg & ~mask) return EINVAL; addr = relreg + (ctx << 0x5); switch (regtype) { case AEREG_SR_REL: case AEREG_SR_RD_REL: qat_ae_xfer_write_4(sc, ae, addr, value); break; case AEREG_DR_REL: case AEREG_DR_RD_REL: qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value); break; default: error = EINVAL; } return error; } static int qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, enum aereg_type regtype, u_short relreg, uint32_t value) { panic("notyet"); return 0; } static int qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx, enum aereg_type regtype, u_short relreg, uint32_t value) { panic("notyet"); return 0; } static int qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae, u_short absreg, u_short *relreg, u_char *ctx) { uint32_t ctxen; qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { /* 4-ctx mode */ *relreg = absreg & 0x1f; *ctx = (absreg >> 0x4) & 0x6; } else { /* 8-ctx mode */ *relreg = absreg & 0x0f; *ctx = (absreg >> 0x4) & 0x7; } return 0; } static int qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae, enum aereg_type regtype, u_short absreg, uint32_t value) { int error; u_short relreg; u_char ctx; qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx); switch (regtype) { case AEREG_GPA_ABS: MPASS(absreg < MAX_GPR_REG); error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, relreg, value); break; case AEREG_GPB_ABS: MPASS(absreg < MAX_GPR_REG); error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, relreg, value); break; case AEREG_DR_RD_ABS: MPASS(absreg < MAX_XFER_REG); error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL, relreg, value); break; case AEREG_SR_RD_ABS: MPASS(absreg < MAX_XFER_REG); error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL, relreg, value); break; case AEREG_DR_WR_ABS: MPASS(absreg < MAX_XFER_REG); error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL, relreg, value); break; case AEREG_SR_WR_ABS: MPASS(absreg < MAX_XFER_REG); error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL, relreg, value); break; case AEREG_NEIGH_ABS: MPASS(absreg < MAX_NN_REG); if (absreg >= MAX_NN_REG) return EINVAL; error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL, relreg, value); break; default: panic("Invalid Register Type"); } return error; } static void qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) { uint32_t ctxen; qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { ctx_mask &= 0x55; } else { ctx_mask &= 0xff; } ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE); qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); } static void qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) { uint32_t ctxen; qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE)); qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); } static void qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode) { uint32_t val, nval; qat_ae_read_4(sc, ae, CTX_ENABLES, &val); val &= CTX_ENABLES_IGNORE_W1C_MASK; if (mode == 4) nval = val | CTX_ENABLES_INUSE_CONTEXTS; else nval = val & ~CTX_ENABLES_INUSE_CONTEXTS; if (val != nval) qat_ae_write_4(sc, ae, CTX_ENABLES, nval); } static void qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode) { uint32_t val, nval; qat_ae_read_4(sc, ae, CTX_ENABLES, &val); val &= CTX_ENABLES_IGNORE_W1C_MASK; if (mode) nval = val | CTX_ENABLES_NN_MODE; else nval = val & ~CTX_ENABLES_NN_MODE; if (val != nval) qat_ae_write_4(sc, ae, CTX_ENABLES, nval); } static void qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae, enum aereg_type lm, u_char mode) { uint32_t val, nval; uint32_t bit; qat_ae_read_4(sc, ae, CTX_ENABLES, &val); val &= CTX_ENABLES_IGNORE_W1C_MASK; switch (lm) { case AEREG_LMEM0: bit = CTX_ENABLES_LMADDR_0_GLOBAL; break; case AEREG_LMEM1: bit = CTX_ENABLES_LMADDR_1_GLOBAL; break; default: panic("invalid lmem reg type"); break; } if (mode) nval = val | bit; else nval = val & ~bit; if (val != nval) qat_ae_write_4(sc, ae, CTX_ENABLES, nval); } static void qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode) { uint32_t val, nval; qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); if (mode == 1) nval = val | AE_MISC_CONTROL_SHARE_CS; else nval = val & ~AE_MISC_CONTROL_SHARE_CS; if (val != nval) qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval); } static void qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode) { u_char nae; qat_ae_get_shared_ustore_ae(ae, &nae); qat_ae_write_shared_cs_mode0(sc, ae, mode); if ((sc->sc_ae_mask & (1 << nae))) { qat_ae_write_shared_cs_mode0(sc, nae, mode); } } static int qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae, u_int reload_size, int shared_mode, u_int ustore_dram_addr) { uint32_t val, cs_reload; switch (reload_size) { case 0: cs_reload = 0x0; break; case QAT_2K: cs_reload = 0x1; break; case QAT_4K: cs_reload = 0x2; break; case QAT_8K: cs_reload = 0x3; break; default: return EINVAL; } if (cs_reload) QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr; QAT_AE(sc, ae).qae_reload_size = reload_size; qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD | AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS); val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) | __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD); qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); return 0; } static enum qat_ae_status qat_ae_get_status(struct qat_softc *sc, u_char ae) { int error; uint32_t val = 0; error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val); if (error || val & CTX_ENABLES_ENABLE) return QAT_AE_ENABLED; qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); if (val & ACTIVE_CTX_STATUS_ABO) return QAT_AE_ACTIVE; return QAT_AE_DISABLED; } static int qat_ae_is_active(struct qat_softc *sc, u_char ae) { uint32_t val; if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) return 1; qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); if (val & ACTIVE_CTX_STATUS_ABO) return 1; else return 0; } /* returns 1 if actually waited for specified number of cycles */ static int qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check) { uint32_t cnt, actx; int pcnt, ccnt, elapsed, times; qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); pcnt = cnt & 0xffff; times = TIMEOUT_AE_CHECK; do { qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); ccnt = cnt & 0xffff; elapsed = ccnt - pcnt; if (elapsed == 0) { times--; } if (times <= 0) { device_printf(sc->sc_dev, "qat_ae_wait_num_cycles timeout\n"); return -1; } if (elapsed < 0) elapsed += 0x10000; if (elapsed >= CYCLES_FROM_READY2EXE && check) { if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) { if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) return 0; } } } while (cycles > elapsed); if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) { if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) return 0; } return 1; } int qat_ae_init(struct qat_softc *sc) { int error; uint32_t mask, val = 0; u_char ae; /* XXX adf_initSysMemInfo */ /* XXX Disable clock gating for some chip if debug mode */ for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { struct qat_ae *qae = &sc->sc_ae[ae]; if (!(mask & 1)) continue; qae->qae_ustore_size = USTORE_SIZE; qae->qae_free_addr = 0; qae->qae_free_size = USTORE_SIZE; qae->qae_live_ctx_mask = AE_ALL_CTX; qae->qae_ustore_dram_addr = 0; qae->qae_reload_size = 0; } /* XXX Enable attention interrupt */ error = qat_ae_clear_reset(sc); if (error) return error; qat_ae_clear_xfer(sc); if (!sc->sc_hw.qhw_fw_auth) { error = qat_ae_clear_gprs(sc); if (error) return error; } /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val); val |= 0x1; qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val); } error = qat_ae_clear_reset(sc); if (error) return error; /* XXX XXX XXX Clean MMP memory if mem scrub is supported */ /* halMem_ScrubMMPMemory */ return 0; } int qat_ae_start(struct qat_softc *sc) { int error; u_char ae; for (ae = 0; ae < sc->sc_ae_num; ae++) { if ((sc->sc_ae_mask & (1 << ae)) == 0) continue; error = qat_aefw_start(sc, ae, 0xff); if (error) return error; } return 0; } void qat_ae_cluster_intr(void *arg) { /* Nothing to implement until we support SRIOV. */ printf("qat_ae_cluster_intr\n"); } static int qat_ae_clear_reset(struct qat_softc *sc) { int error; uint32_t times, reset, clock, reg, mask; u_char ae; reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK)); reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)); times = TIMEOUT_AE_RESET; do { qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset); if ((times--) == 0) { device_printf(sc->sc_dev, "couldn't reset AEs\n"); return EBUSY; } reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) | __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)) & reg); /* Enable clock for AE and QAT */ clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN); clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK); clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK); qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock); error = qat_ae_check(sc); if (error) return error; /* * Set undefined power-up/reset states to reasonable default values... * just to make sure we're starting from a known point */ for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; /* init the ctx_enable */ qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT); /* initialize the PCs */ qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); /* init the ctx_arb */ qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT); /* enable cc */ qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT); qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT); qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, CTX_SIG_EVENTS_INDIRECT_INIT); } if ((sc->sc_ae_mask != 0) && sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) { /* XXX XXX XXX init eSram only when this is boot time */ } if ((sc->sc_ae_mask != 0) && sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) { /* XXX XXX XXX wait shram to complete initialization */ } qat_ae_reset_timestamp(sc); return 0; } static int qat_ae_check(struct qat_softc *sc) { int error, times, ae; uint32_t cnt, pcnt, mask; for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; times = TIMEOUT_AE_CHECK; error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); if (error) { device_printf(sc->sc_dev, "couldn't access AE %d CSR\n", ae); return error; } pcnt = cnt & 0xffff; while (1) { error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); if (error) { device_printf(sc->sc_dev, "couldn't access AE %d CSR\n", ae); return error; } cnt &= 0xffff; if (cnt == pcnt) times--; else break; if (times <= 0) { device_printf(sc->sc_dev, "AE %d CSR is useless\n", ae); return EFAULT; } } } return 0; } static int qat_ae_reset_timestamp(struct qat_softc *sc) { uint32_t misc, mask; u_char ae; /* stop the timestamp timers */ misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC); if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) { qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN)); } for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0); qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0); } /* start timestamp timers */ qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN); return 0; } static void qat_ae_clear_xfer(struct qat_softc *sc) { u_int mask, reg; u_char ae; for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; for (reg = 0; reg < MAX_GPR_REG; reg++) { qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS, reg, 0); qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS, reg, 0); } } } static int qat_ae_clear_gprs(struct qat_softc *sc) { uint32_t val; uint32_t saved_ctx = 0; int times = TIMEOUT_AE_CHECK, rv; u_char ae; u_int mask; for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; /* turn off share control store bit */ val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); val &= ~AE_MISC_CONTROL_SHARE_CS; qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); /* turn off ucode parity */ /* make sure nn_mode is set to self */ qat_ae_read_4(sc, ae, CTX_ENABLES, &val); val &= CTX_ENABLES_IGNORE_W1C_MASK; val |= CTX_ENABLES_NN_MODE; val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE; qat_ae_write_4(sc, ae, CTX_ENABLES, val); /* copy instructions to ustore */ qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst), ae_clear_gprs_inst); /* set PC */ qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); /* save current context */ qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx); /* change the active context */ /* start the context from ctx 0 */ qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0); /* wakeup-event voluntary */ qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); /* clean signals */ qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, 0); qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); qat_ae_enable_ctx(sc, ae, AE_ALL_CTX); } for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; /* wait for AE to finish */ do { rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1); } while (rv && times--); if (times <= 0) { device_printf(sc->sc_dev, "qat_ae_clear_gprs timeout"); return ETIMEDOUT; } qat_ae_disable_ctx(sc, ae, AE_ALL_CTX); /* change the active context */ qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, saved_ctx & ACTIVE_CTX_STATUS_ACNO); /* init the ctx_enable */ qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT); /* initialize the PCs */ qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); /* init the ctx_arb */ qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT); /* enable cc */ qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT); qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT); qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, CTX_SIG_EVENTS_INDIRECT_INIT); } return 0; } static void qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae) { if (ae & 0x1) *nae = ae - 1; else *nae = ae + 1; } static u_int qat_ae_ucode_parity64(uint64_t ucode) { ucode ^= ucode >> 1; ucode ^= ucode >> 2; ucode ^= ucode >> 4; ucode ^= ucode >> 8; ucode ^= ucode >> 16; ucode ^= ucode >> 32; return ((u_int)(ucode & 1)); } static uint64_t qat_ae_ucode_set_ecc(uint64_t ucode) { static const uint64_t bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL, bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL, bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL, bit6mask=0xdaf69a46910ULL; /* clear the ecc bits */ ucode &= ~(0x7fULL << USTORE_ECC_BIT_0); ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) << USTORE_ECC_BIT_0; ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) << USTORE_ECC_BIT_1; ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) << USTORE_ECC_BIT_2; ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) << USTORE_ECC_BIT_3; ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) << USTORE_ECC_BIT_4; ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) << USTORE_ECC_BIT_5; ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) << USTORE_ECC_BIT_6; return (ucode); } static int qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, const uint64_t *ucode) { uint64_t tmp; uint32_t ustore_addr, ulo, uhi; int i; qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); uaddr |= USTORE_ADDRESS_ECS; qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); for (i = 0; i < ninst; i++) { tmp = qat_ae_ucode_set_ecc(ucode[i]); ulo = (uint32_t)(tmp & 0xffffffff); uhi = (uint32_t)(tmp >> 32); qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); /* this will auto increment the address */ qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); } qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); return 0; } static int qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, uint64_t *ucode) { uint32_t misc, ustore_addr, ulo, uhi; u_int ii; u_char nae; if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) return EBUSY; /* determine whether it neighbour AE runs in shared control store * status */ qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); if (misc & AE_MISC_CONTROL_SHARE_CS) { qat_ae_get_shared_ustore_ae(ae, &nae); if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) return EBUSY; } /* if reloadable, then get it all from dram-ustore */ if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD)) panic("notyet"); /* XXX getReloadUwords */ /* disable SHARE_CS bit to workaround silicon bug */ qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb); MPASS(uaddr + ninst <= USTORE_SIZE); /* save ustore-addr csr */ qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */ for (ii = 0; ii < ninst; ii++) { qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); uaddr++; qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo); qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi); ucode[ii] = uhi; ucode[ii] = (ucode[ii] << 32) | ulo; } /* restore SHARE_CS bit to workaround silicon bug */ qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); return 0; } static u_int qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr, u_int *value) { const uint64_t *inst_arr; u_int ninst0, curvalue; int ii, vali, fixup, usize = 0; if (size == 0) return 0; ninst0 = ninst; vali = 0; curvalue = value[vali++]; switch (size) { case 0x1: inst_arr = ae_inst_1b; usize = nitems(ae_inst_1b); break; case 0x2: inst_arr = ae_inst_2b; usize = nitems(ae_inst_2b); break; case 0x3: inst_arr = ae_inst_3b; usize = nitems(ae_inst_3b); break; default: inst_arr = ae_inst_4b; usize = nitems(ae_inst_4b); break; } fixup = ninst; for (ii = 0; ii < usize; ii++) ucode[ninst++] = inst_arr[ii]; INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr)); fixup++; INSERT_IMMED_GPRA_CONST(ucode[fixup], 0); fixup++; INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); fixup++; INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); /* XXX fixup++ ? */ if (size <= 0x4) return (ninst - ninst0); size -= sizeof(u_int); while (size >= sizeof(u_int)) { curvalue = value[vali++]; fixup = ninst; ucode[ninst++] = ae_inst_4b[0x2]; ucode[ninst++] = ae_inst_4b[0x3]; ucode[ninst++] = ae_inst_4b[0x8]; INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); fixup++; INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); /* XXX fixup++ ? */ addr += sizeof(u_int); size -= sizeof(u_int); } /* call this function recusive when the left size less than 4 */ ninst += qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali); return (ninst - ninst0); } static int qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx, uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles, u_int *endpc) { int error = 0, share_cs = 0; uint64_t savucode[MAX_EXEC_INST]; uint32_t indr_lm_addr_0, indr_lm_addr_1; uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1; uint32_t indr_future_cnt_sig; uint32_t indr_sig, active_sig; uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl; uint32_t misc, nmisc, ctxen; u_char nae; MPASS(ninst <= USTORE_SIZE); if (qat_ae_is_active(sc, ae)) return EBUSY; /* save current LM addr */ qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0); qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1); qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &indr_lm_addr_byte_0); qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &indr_lm_addr_byte_1); /* backup shared control store bit, and force AE to none-shared mode before executing ucode snippet */ qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); if (misc & AE_MISC_CONTROL_SHARE_CS) { share_cs = 1; qat_ae_get_shared_ustore_ae(ae, &nae); if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) return EBUSY; } nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); /* save current states: */ if (ninst <= MAX_EXEC_INST) { error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode); if (error) { qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); return error; } } /* save wakeup-events */ qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT, &wakeup_ev); /* save PC */ qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc); savpc &= UPC_MASK; /* save ctx enables */ qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; /* save conditional-code */ qat_ae_read_4(sc, ae, CC_ENABLE, &savcc); /* save current context */ qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); /* save indirect csrs */ qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, &indr_future_cnt_sig); qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig); qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig); /* turn off ucode parity */ qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE); /* copy instructions to ustore */ qat_ae_ucode_write(sc, ae, 0, ninst, ucode); /* set PC */ qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0); /* change the active context */ qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, ctx & ACTIVE_CTX_STATUS_ACNO); if (cond_code_off) { /* disable conditional-code*/ qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff); } /* wakeup-event voluntary */ qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); /* clean signals */ qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0); qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); /* enable context */ qat_ae_enable_ctx(sc, ae, 1 << ctx); /* wait for it to finish */ if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0) error = ETIMEDOUT; /* see if we need to get the current PC */ if (endpc != NULL) { uint32_t ctx_status; qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &ctx_status); *endpc = ctx_status & UPC_MASK; } #if 0 { uint32_t ctx_status; qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &ctx_status); printf("%s: endpc 0x%08x\n", __func__, ctx_status & UPC_MASK); } #endif /* retore to previous states: */ /* disable context */ qat_ae_disable_ctx(sc, ae, 1 << ctx); if (ninst <= MAX_EXEC_INST) { /* instructions */ qat_ae_ucode_write(sc, ae, 0, ninst, savucode); } /* wakeup-events */ qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT, wakeup_ev); qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc); /* only restore shared control store bit, other bit might be changed by AE code snippet */ qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); if (share_cs) nmisc = misc | AE_MISC_CONTROL_SHARE_CS; else nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); /* conditional-code */ qat_ae_write_4(sc, ae, CC_ENABLE, savcc); /* change the active context */ qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, savctx & ACTIVE_CTX_STATUS_ACNO); /* restore the nxt ctx to run */ qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); /* restore current LM addr */ qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT, indr_lm_addr_0); qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT, indr_lm_addr_1); qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, indr_lm_addr_byte_0); qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, indr_lm_addr_byte_1); /* restore indirect csrs */ qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT, indr_future_cnt_sig); qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, indr_sig); qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig); /* ctx-enables */ qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); return error; } static int qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx, int *first_exec, uint64_t *ucode, u_int ninst, u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1) { if (*first_exec) { qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); *first_exec = 0; } return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL); } static int qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx, u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1) { qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); return 0; } static int qat_ae_get_inst_num(int lmsize) { int ninst, left; if (lmsize == 0) return 0; left = lmsize % sizeof(u_int); if (left) { ninst = nitems(ae_inst_1b) + qat_ae_get_inst_num(lmsize - left); } else { /* 3 instruction is needed for further code */ ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b); } return (ninst); } static int qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae, struct qat_ae_batch_init_list *qabi_list, size_t nqabi) { struct qat_ae_batch_init *qabi; size_t alloc_ninst, ninst; uint64_t *ucode; u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1; int insnsz, error = 0, execed = 0, first_exec = 1; if (STAILQ_FIRST(qabi_list) == NULL) return 0; alloc_ninst = min(USTORE_SIZE, nqabi); ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst); ninst = 0; STAILQ_FOREACH(qabi, qabi_list, qabi_next) { insnsz = qat_ae_get_inst_num(qabi->qabi_size); if (insnsz + ninst > alloc_ninst) { /* add ctx_arb[kill] */ ucode[ninst++] = 0x0E000010000ull; execed = 1; error = qat_ae_exec_ucode_init_lm(sc, ae, 0, &first_exec, ucode, ninst, &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); if (error) { qat_ae_restore_init_lm_gprs(sc, ae, 0, gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); qat_free_mem(ucode); return error; } /* run microExec to execute the microcode */ ninst = 0; } ninst += qat_ae_concat_ucode(ucode, ninst, qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value); } if (ninst > 0) { ucode[ninst++] = 0x0E000010000ull; execed = 1; error = qat_ae_exec_ucode_init_lm(sc, ae, 0, &first_exec, ucode, ninst, &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); } if (execed) { qat_ae_restore_init_lm_gprs(sc, ae, 0, gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); } qat_free_mem(ucode); return error; } static int qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc) { if (qat_ae_is_active(sc, ae)) return EBUSY; qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT, UPC_MASK & upc); return 0; } static inline u_int qat_aefw_csum_calc(u_int reg, int ch) { int i; u_int topbit = CRC_BITMASK(CRC_WIDTH - 1); u_int inbyte = (u_int)((reg >> 0x18) ^ ch); reg ^= inbyte << (CRC_WIDTH - 0x8); for (i = 0; i < 0x8; i++) { if (reg & topbit) reg = (reg << 1) ^ CRC_POLY; else reg <<= 1; } return (reg & CRC_WIDTHMASK(CRC_WIDTH)); } static u_int qat_aefw_csum(char *buf, int size) { u_int csum = 0; while (size--) { csum = qat_aefw_csum_calc(csum, *buf++); } return csum; } static const char * qat_aefw_uof_string(struct qat_softc *sc, size_t offset) { if (offset >= sc->sc_aefw_uof.qafu_str_tab_size) return NULL; if (sc->sc_aefw_uof.qafu_str_tab == NULL) return NULL; return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset); } static struct uof_chunk_hdr * qat_aefw_uof_find_chunk(struct qat_softc *sc, const char *id, struct uof_chunk_hdr *cur) { struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr; struct uof_chunk_hdr *uch; int i; uch = (struct uof_chunk_hdr *)(uoh + 1); for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) { if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size) return NULL; if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN)) return uch; } return NULL; } static int qat_aefw_load_mof(struct qat_softc *sc) { const struct firmware *fw; fw = firmware_get(sc->sc_hw.qhw_mof_fwname); if (fw == NULL) { device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", sc->sc_hw.qhw_mof_fwname); return ENXIO; } sc->sc_fw_mof = qat_alloc_mem(fw->datasize); sc->sc_fw_mof_size = fw->datasize; memcpy(sc->sc_fw_mof, fw->data, fw->datasize); firmware_put(fw, FIRMWARE_UNLOAD); return 0; } static void qat_aefw_unload_mof(struct qat_softc *sc) { if (sc->sc_fw_mof != NULL) { qat_free_mem(sc->sc_fw_mof); sc->sc_fw_mof = NULL; } } static int qat_aefw_load_mmp(struct qat_softc *sc) { const struct firmware *fw; fw = firmware_get(sc->sc_hw.qhw_mmp_fwname); if (fw == NULL) { device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", sc->sc_hw.qhw_mmp_fwname); return ENXIO; } sc->sc_fw_mmp = qat_alloc_mem(fw->datasize); sc->sc_fw_mmp_size = fw->datasize; memcpy(sc->sc_fw_mmp, fw->data, fw->datasize); firmware_put(fw, FIRMWARE_UNLOAD); return 0; } static void qat_aefw_unload_mmp(struct qat_softc *sc) { if (sc->sc_fw_mmp != NULL) { qat_free_mem(sc->sc_fw_mmp); sc->sc_fw_mmp = NULL; } } static int qat_aefw_mof_find_uof0(struct qat_softc *sc, struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head, u_int nchunk, size_t size, const char *id, size_t *fwsize, void **fwptr) { int i; char *uof_name; for (i = 0; i < nchunk; i++) { struct mof_uof_chunk_hdr *much = &head[i]; if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN)) return EINVAL; if (much->much_offset + much->much_size > size) return EINVAL; if (sc->sc_mof.qmf_sym_size <= much->much_name) return EINVAL; uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym + much->much_name); if (!strcmp(uof_name, sc->sc_fw_uof_name)) { *fwptr = (void *)((uintptr_t)muh + (uintptr_t)much->much_offset); *fwsize = (size_t)much->much_size; return 0; } } return ENOENT; } static int qat_aefw_mof_find_uof(struct qat_softc *sc) { struct mof_uof_hdr *uof_hdr, *suof_hdr; u_int nuof_chunks = 0, nsuof_chunks = 0; int error; uof_hdr = sc->sc_mof.qmf_uof_objs; suof_hdr = sc->sc_mof.qmf_suof_objs; if (uof_hdr != NULL) { if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) { return EINVAL; } nuof_chunks = uof_hdr->muh_num_chunks; } if (suof_hdr != NULL) { if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks) return EINVAL; nsuof_chunks = suof_hdr->muh_num_chunks; } if (nuof_chunks + nsuof_chunks == 0) return EINVAL; if (uof_hdr != NULL) { error = qat_aefw_mof_find_uof0(sc, uof_hdr, (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks, sc->sc_mof.qmf_uof_objs_size, UOF_IMAG, &sc->sc_fw_uof_size, &sc->sc_fw_uof); if (error && error != ENOENT) return error; } if (suof_hdr != NULL) { error = qat_aefw_mof_find_uof0(sc, suof_hdr, (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks, sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG, &sc->sc_fw_suof_size, &sc->sc_fw_suof); if (error && error != ENOENT) return error; } if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL) return ENOENT; return 0; } static int qat_aefw_mof_parse(struct qat_softc *sc) { const struct mof_file_hdr *mfh; const struct mof_file_chunk_hdr *mfch; size_t size; u_int csum; int error, i; size = sc->sc_fw_mof_size; if (size < sizeof(struct mof_file_hdr)) return EINVAL; size -= sizeof(struct mof_file_hdr); mfh = sc->sc_fw_mof; if (mfh->mfh_fid != MOF_FID) return EINVAL; csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof + offsetof(struct mof_file_hdr, mfh_min_ver)), sc->sc_fw_mof_size - offsetof(struct mof_file_hdr, mfh_min_ver)); if (mfh->mfh_csum != csum) return EINVAL; if (mfh->mfh_min_ver != MOF_MIN_VER || mfh->mfh_maj_ver != MOF_MAJ_VER) return EINVAL; if (mfh->mfh_max_chunks < mfh->mfh_num_chunks) return EINVAL; if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks) return EINVAL; mfch = (const struct mof_file_chunk_hdr *)(mfh + 1); for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) { if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size) return EINVAL; if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) { if (sc->sc_mof.qmf_sym != NULL) return EINVAL; sc->sc_mof.qmf_sym = (void *)((uintptr_t)sc->sc_fw_mof + (uintptr_t)mfch->mfch_offset + sizeof(u_int)); sc->sc_mof.qmf_sym_size = *(u_int *)((uintptr_t)sc->sc_fw_mof + (uintptr_t)mfch->mfch_offset); if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0) return EINVAL; if (mfch->mfch_size != sc->sc_mof.qmf_sym_size + sizeof(u_int) || mfch->mfch_size == 0) return EINVAL; if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym + sc->sc_mof.qmf_sym_size - 1) != '\0') return EINVAL; } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) { if (sc->sc_mof.qmf_uof_objs != NULL) return EINVAL; sc->sc_mof.qmf_uof_objs = (void *)((uintptr_t)sc->sc_fw_mof + (uintptr_t)mfch->mfch_offset); sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size; } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) { if (sc->sc_mof.qmf_suof_objs != NULL) return EINVAL; sc->sc_mof.qmf_suof_objs = (void *)((uintptr_t)sc->sc_fw_mof + (uintptr_t)mfch->mfch_offset); sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size; } } if (sc->sc_mof.qmf_sym == NULL || (sc->sc_mof.qmf_uof_objs == NULL && sc->sc_mof.qmf_suof_objs == NULL)) return EINVAL; error = qat_aefw_mof_find_uof(sc); if (error) return error; return 0; } static int qat_aefw_uof_parse_image(struct qat_softc *sc, struct qat_uof_image *qui, struct uof_chunk_hdr *uch) { struct uof_image *image; struct uof_code_page *page; uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; size_t lim = uch->uch_offset + uch->uch_size, size; int i, p; size = uch->uch_size; if (size < sizeof(struct uof_image)) return EINVAL; size -= sizeof(struct uof_image); qui->qui_image = image = (struct uof_image *)(base + uch->uch_offset); #define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \ do { \ u_int nent; \ nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\ if ((lim) < off + sizeof(struct uof_obj_table) + \ sizeof(type) * nent) \ return EINVAL; \ *(np) = nent; \ if (nent > 0) \ *(typep) = (type)((struct uof_obj_table *) \ ((base) + (off)) + 1); \ else \ *(typep) = NULL; \ } while (0) ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg, struct uof_ae_reg *, base, image->ui_reg_tab, lim); ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym, struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim); ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak, struct qui_sbreak *, base, image->ui_sbreak_tab, lim); if (size < sizeof(struct uof_code_page) * image->ui_num_pages) return EINVAL; if (nitems(qui->qui_pages) < image->ui_num_pages) return EINVAL; page = (struct uof_code_page *)(image + 1); for (p = 0; p < image->ui_num_pages; p++, page++) { struct qat_uof_page *qup = &qui->qui_pages[p]; struct uof_code_area *uca; qup->qup_page_num = page->ucp_page_num; qup->qup_def_page = page->ucp_def_page; qup->qup_page_region = page->ucp_page_region; qup->qup_beg_vaddr = page->ucp_beg_vaddr; qup->qup_beg_paddr = page->ucp_beg_paddr; ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var, struct uof_uword_fixup *, base, page->ucp_uc_var_tab, lim); ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var, struct uof_import_var *, base, page->ucp_imp_var_tab, lim); ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr, struct uof_uword_fixup *, base, page->ucp_imp_expr_tab, lim); ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg, struct uof_uword_fixup *, base, page->ucp_neigh_reg_tab, lim); if (lim < page->ucp_code_area + sizeof(struct uof_code_area)) return EINVAL; uca = (struct uof_code_area *)(base + page->ucp_code_area); qup->qup_num_micro_words = uca->uca_num_micro_words; ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks, struct qat_uof_uword_block *, base, uca->uca_uword_block_tab, lim); for (i = 0; i < qup->qup_num_uw_blocks; i++) { u_int uwordoff = ((struct uof_uword_block *)( &qup->qup_uw_blocks[i]))->uub_uword_offset; if (lim < uwordoff) return EINVAL; qup->qup_uw_blocks[i].quub_micro_words = (base + uwordoff); } } #undef ASSIGN_OBJ_TAB return 0; } static int qat_aefw_uof_parse_images(struct qat_softc *sc) { struct uof_chunk_hdr *uch = NULL; int i, error; for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) { uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch); if (uch == NULL) break; if (i >= nitems(sc->sc_aefw_uof.qafu_imgs)) return ENOENT; error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch); if (error) return error; sc->sc_aefw_uof.qafu_num_imgs++; } return 0; } static int qat_aefw_uof_parse(struct qat_softc *sc) { struct uof_file_hdr *ufh; struct uof_file_chunk_hdr *ufch; struct uof_obj_hdr *uoh; struct uof_chunk_hdr *uch; void *uof = NULL; size_t size, uof_size, hdr_size; uintptr_t base; u_int csum; int i; size = sc->sc_fw_uof_size; if (size < MIN_UOF_SIZE) return EINVAL; size -= sizeof(struct uof_file_hdr); ufh = sc->sc_fw_uof; if (ufh->ufh_id != UOF_FID) return EINVAL; if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER) return EINVAL; if (ufh->ufh_max_chunks < ufh->ufh_num_chunks) return EINVAL; if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks) return EINVAL; ufch = (struct uof_file_chunk_hdr *)(ufh + 1); uof_size = 0; for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) { if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size) return EINVAL; if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) { if (uof != NULL) return EINVAL; uof = (void *)((uintptr_t)sc->sc_fw_uof + ufch->ufch_offset); uof_size = ufch->ufch_size; csum = qat_aefw_csum(uof, uof_size); if (csum != ufch->ufch_csum) return EINVAL; } } if (uof == NULL) return ENOENT; size = uof_size; if (size < sizeof(struct uof_obj_hdr)) return EINVAL; size -= sizeof(struct uof_obj_hdr); uoh = uof; if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks) return EINVAL; /* Check if the UOF objects are compatible with the chip */ if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0) return ENOTSUP; if (uoh->uoh_min_cpu_ver > sc->sc_rev || uoh->uoh_max_cpu_ver < sc->sc_rev) return ENOTSUP; sc->sc_aefw_uof.qafu_size = uof_size; sc->sc_aefw_uof.qafu_obj_hdr = uoh; base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; /* map uof string-table */ uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL); if (uch != NULL) { hdr_size = offsetof(struct uof_str_tab, ust_strings); sc->sc_aefw_uof.qafu_str_tab = (void *)(base + uch->uch_offset + hdr_size); sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size; } /* get ustore mem inits table -- should be only one */ uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL); if (uch != NULL) { if (uch->uch_size < sizeof(struct uof_obj_table)) return EINVAL; sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base + uch->uch_offset))->uot_nentries; if (sc->sc_aefw_uof.qafu_num_init_mem) { sc->sc_aefw_uof.qafu_init_mem = (struct uof_init_mem *)(base + uch->uch_offset + sizeof(struct uof_obj_table)); sc->sc_aefw_uof.qafu_init_mem_size = uch->uch_size - sizeof(struct uof_obj_table); } } uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL); if (uch != NULL) { if (uch->uch_size < sizeof(struct uof_obj_table) + sizeof(struct uof_var_mem_seg)) return EINVAL; sc->sc_aefw_uof.qafu_var_mem_seg = (struct uof_var_mem_seg *)(base + uch->uch_offset + sizeof(struct uof_obj_table)); } return qat_aefw_uof_parse_images(sc); } static int qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi, struct suof_chunk_hdr *sch) { struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; struct simg_ae_mode *ae_mode; u_int maj_ver; qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset + sizeof(struct suof_obj_hdr); qsi->qsi_simg_len = ((struct suof_obj_hdr *) (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length; qsi->qsi_css_header = qsi->qsi_simg_buf; qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr); qsi->qsi_css_signature = qsi->qsi_css_key + CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN; ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg; qsi->qsi_ae_mask = ae_mode->sam_ae_mask; qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name; qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data; qsi->qsi_fw_type = ae_mode->sam_fw_type; if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type) return EINVAL; maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff; if ((maj_ver > ae_mode->sam_devmax_ver) || (maj_ver < ae_mode->sam_devmin_ver)) { return EINVAL; } return 0; } static int qat_aefw_suof_parse(struct qat_softc *sc) { struct suof_file_hdr *sfh; struct suof_chunk_hdr *sch; struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; struct qat_suof_image *qsi; size_t size; u_int csum; int ae0_img = MAX_AE; int i, error; size = sc->sc_fw_suof_size; if (size < sizeof(struct suof_file_hdr)) return EINVAL; sfh = sc->sc_fw_suof; if (sfh->sfh_file_id != SUOF_FID) return EINVAL; if (sfh->sfh_fw_type != 0) return EINVAL; if (sfh->sfh_num_chunks <= 1) return EINVAL; if (sfh->sfh_min_ver != SUOF_MIN_VER || sfh->sfh_maj_ver != SUOF_MAJ_VER) return EINVAL; csum = qat_aefw_csum((char *)&sfh->sfh_min_ver, size - offsetof(struct suof_file_hdr, sfh_min_ver)); if (csum != sfh->sfh_check_sum) return EINVAL; size -= sizeof(struct suof_file_hdr); qafs->qafs_file_id = SUOF_FID; qafs->qafs_suof_buf = sc->sc_fw_suof; qafs->qafs_suof_size = sc->sc_fw_suof_size; qafs->qafs_check_sum = sfh->sfh_check_sum; qafs->qafs_min_ver = sfh->sfh_min_ver; qafs->qafs_maj_ver = sfh->sfh_maj_ver; qafs->qafs_fw_type = sfh->sfh_fw_type; if (size < sizeof(struct suof_chunk_hdr)) return EINVAL; sch = (struct suof_chunk_hdr *)(sfh + 1); size -= sizeof(struct suof_chunk_hdr); if (size < sizeof(struct suof_str_tab)) return EINVAL; size -= offsetof(struct suof_str_tab, sst_strings); qafs->qafs_sym_size = ((struct suof_str_tab *) (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length; if (size < qafs->qafs_sym_size) return EINVAL; qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset + offsetof(struct suof_str_tab, sst_strings); qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1; if (qafs->qafs_num_simgs == 0) return EINVAL; qsi = qat_alloc_mem( sizeof(struct qat_suof_image) * qafs->qafs_num_simgs); qafs->qafs_simg = qsi; for (i = 0; i < qafs->qafs_num_simgs; i++) { error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]); if (error) return error; if ((qsi[i].qsi_ae_mask & 0x1) != 0) ae0_img = i; } if (ae0_img != qafs->qafs_num_simgs - 1) { struct qat_suof_image last_qsi; memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1], sizeof(struct qat_suof_image)); memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img], sizeof(struct qat_suof_image)); memcpy(&qsi[ae0_img], &last_qsi, sizeof(struct qat_suof_image)); } return 0; } static int qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size, struct qat_dmamem *dma) { struct css_hdr *css = (struct css_hdr *)image; struct auth_chunk *auth_chunk; struct fw_auth_desc *auth_desc; size_t mapsize, simg_offset = sizeof(struct auth_chunk); bus_size_t bus_addr; uintptr_t virt_addr; int error; if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN) return EINVAL; mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ? CSS_AE_SIMG_LEN + simg_offset : size + CSS_FWSK_PAD_LEN + simg_offset; error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE); if (error) return error; memset(dma->qdm_dma_vaddr, 0, mapsize); auth_chunk = dma->qdm_dma_vaddr; auth_chunk->ac_chunk_size = mapsize; auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr; virt_addr = (uintptr_t)dma->qdm_dma_vaddr; virt_addr += simg_offset; bus_addr = auth_chunk->ac_chunk_bus_addr; bus_addr += simg_offset; auth_desc = &auth_chunk->ac_fw_auth_desc; auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32; auth_desc->fad_css_hdr_low = bus_addr; memcpy((void *)virt_addr, image, sizeof(struct css_hdr)); /* pub key */ virt_addr += sizeof(struct css_hdr); bus_addr += sizeof(struct css_hdr); image += sizeof(struct css_hdr); auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32; auth_desc->fad_fwsk_pub_low = bus_addr; memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN); memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN); memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN), image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t)); virt_addr += CSS_FWSK_PUB_LEN; bus_addr += CSS_FWSK_PUB_LEN; image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32; auth_desc->fad_signature_low = bus_addr; memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN); virt_addr += CSS_SIGNATURE_LEN; bus_addr += CSS_SIGNATURE_LEN; image += CSS_SIGNATURE_LEN; auth_desc->fad_img_high = (uint64_t)bus_addr >> 32; auth_desc->fad_img_low = bus_addr; auth_desc->fad_img_len = size - AE_IMG_OFFSET; memcpy((void *)virt_addr, image, auth_desc->fad_img_len); if (css->css_fw_type == CSS_AE_FIRMWARE) { auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high; auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low; bus_addr += sizeof(struct simg_ae_mode); auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32; auth_desc->fad_img_ae_init_data_low = bus_addr; bus_addr += SIMG_AE_INIT_SEQ_LEN; auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32; auth_desc->fad_img_ae_insts_low = bus_addr; } else { auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high; auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low; } bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); return 0; } static int qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma) { bus_addr_t addr; uint32_t fcu, sts; int retry = 0; addr = dma->qdm_dma_seg.ds_addr; qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32); qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr); qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH); do { DELAY(FW_AUTH_WAIT_PERIOD * 1000); fcu = qat_cap_global_read_4(sc, FCU_STATUS); sts = __SHIFTOUT(fcu, FCU_STATUS_STS); if (sts == FCU_STATUS_STS_VERI_FAIL) goto fail; if (fcu & FCU_STATUS_AUTHFWLD && sts == FCU_STATUS_STS_VERI_DONE) { return 0; } } while (retry++ < FW_AUTH_MAX_RETRY); fail: device_printf(sc->sc_dev, "firmware authentication error: status 0x%08x retry %d\n", fcu, retry); return EINVAL; } static int qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma) { struct simg_ae_mode *ae_mode; uint32_t fcu, sts, loaded; u_int mask; u_char ae; int retry = 0; ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr + sizeof(struct auth_chunk) + sizeof(struct css_hdr) + CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN); for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; if (!((ae_mode->sam_ae_mask >> ae) & 0x1)) continue; if (qat_ae_is_active(sc, ae)) { device_printf(sc->sc_dev, "AE %d is active\n", ae); return EINVAL; } qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE)); do { DELAY(FW_AUTH_WAIT_PERIOD * 1000); fcu = qat_cap_global_read_4(sc, FCU_STATUS); sts = __SHIFTOUT(fcu, FCU_STATUS_STS); loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE); if (sts == FCU_STATUS_STS_LOAD_DONE && (loaded & (1 << ae))) { break; } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { device_printf(sc->sc_dev, "firmware load timeout: status %08x\n", fcu); return EINVAL; } } return 0; } static int qat_aefw_suof_write(struct qat_softc *sc) { struct qat_suof_image *qsi; int i, error = 0; for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) { qsi = &sc->sc_aefw_suof.qafs_simg[i]; error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf, qsi->qsi_simg_len, &qsi->qsi_dma); if (error) return error; error = qat_aefw_auth(sc, &qsi->qsi_dma); if (error) { qat_free_dmamem(sc, &qsi->qsi_dma); return error; } error = qat_aefw_suof_load(sc, &qsi->qsi_dma); if (error) { qat_free_dmamem(sc, &qsi->qsi_dma); return error; } qat_free_dmamem(sc, &qsi->qsi_dma); } qat_free_mem(sc->sc_aefw_suof.qafs_simg); return 0; } static int qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae, struct qat_uof_image *qui) { struct qat_ae_slice *slice; int i, npages, nregions; if (qae->qae_num_slices >= nitems(qae->qae_slices)) return ENOENT; if (qui->qui_image->ui_ae_mode & (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) { /* XXX */ device_printf(sc->sc_dev, "shared ae mode is not supported yet\n"); return ENOTSUP; } qae->qae_shareable_ustore = 0; /* XXX */ qae->qae_effect_ustore_size = USTORE_SIZE; slice = &qae->qae_slices[qae->qae_num_slices]; slice->qas_image = qui; slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned; nregions = qui->qui_image->ui_num_page_regions; npages = qui->qui_image->ui_num_pages; if (nregions > nitems(slice->qas_regions)) return ENOENT; if (npages > nitems(slice->qas_pages)) return ENOENT; for (i = 0; i < nregions; i++) { STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages); } for (i = 0; i < npages; i++) { struct qat_ae_page *page = &slice->qas_pages[i]; int region; page->qap_page = &qui->qui_pages[i]; region = page->qap_page->qup_page_region; if (region >= nregions) return EINVAL; page->qap_region = &slice->qas_regions[region]; } qae->qae_num_slices++; return 0; } static int qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae) { struct uof_image *image; struct qat_ae *qae = &(QAT_AE(sc, ae)); int s; u_char nn_mode; for (s = 0; s < qae->qae_num_slices; s++) { if (qae->qae_slices[s].qas_image == NULL) continue; image = qae->qae_slices[s].qas_image->qui_image; qat_ae_write_ctx_mode(sc, ae, __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE)); nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE); if (nn_mode != AE_MODE_NN_MODE_DONTCARE) qat_ae_write_nn_mode(sc, ae, nn_mode); qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0, __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0)); qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1, __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1)); qat_ae_write_shared_cs_mode(sc, ae, __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE)); qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size, __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED), qae->qae_reloc_ustore_dram); } return 0; } static int qat_aefw_uof_init(struct qat_softc *sc) { int ae, i, error; uint32_t mask; for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { struct qat_ae *qae; if (!(mask & 1)) continue; qae = &(QAT_AE(sc, ae)); for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned & (1 << ae)) == 0) continue; error = qat_aefw_uof_assign_image(sc, qae, &sc->sc_aefw_uof.qafu_imgs[i]); if (error) return error; } /* XXX UcLo_initNumUwordUsed */ qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */ error = qat_aefw_uof_init_ae(sc, ae); if (error) return error; } return 0; } int qat_aefw_load(struct qat_softc *sc) { int error; error = qat_aefw_load_mof(sc); if (error) return error; error = qat_aefw_load_mmp(sc); if (error) return error; error = qat_aefw_mof_parse(sc); if (error) { device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error); return error; } if (sc->sc_hw.qhw_fw_auth) { error = qat_aefw_suof_parse(sc); if (error) { device_printf(sc->sc_dev, "couldn't parse suof: %d\n", error); return error; } error = qat_aefw_suof_write(sc); if (error) { device_printf(sc->sc_dev, "could not write firmware: %d\n", error); return error; } } else { error = qat_aefw_uof_parse(sc); if (error) { device_printf(sc->sc_dev, "couldn't parse uof: %d\n", error); return error; } error = qat_aefw_uof_init(sc); if (error) { device_printf(sc->sc_dev, "couldn't init for aefw: %d\n", error); return error; } error = qat_aefw_uof_write(sc); if (error) { device_printf(sc->sc_dev, "Could not write firmware: %d\n", error); return error; } } return 0; } void qat_aefw_unload(struct qat_softc *sc) { qat_aefw_unload_mmp(sc); qat_aefw_unload_mof(sc); } int qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask) { uint32_t fcu; int retry = 0; if (sc->sc_hw.qhw_fw_auth) { qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START); do { DELAY(FW_AUTH_WAIT_PERIOD * 1000); fcu = qat_cap_global_read_4(sc, FCU_STATUS); if (fcu & FCU_STATUS_DONE) return 0; } while (retry++ < FW_AUTH_MAX_RETRY); device_printf(sc->sc_dev, "firmware start timeout: status %08x\n", fcu); return EINVAL; } else { qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX, CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_SLEEP); qat_ae_enable_ctx(sc, ae, ctx_mask); } return 0; } static int qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim) { struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; struct qat_ae_batch_init_list *qabi_list; struct uof_mem_val_attr *memattr; size_t *curinit; u_long ael; int i; const char *sym; char *ep; memattr = (struct uof_mem_val_attr *)(uim + 1); switch (uim->uim_region) { case LMEM_REGION: if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) { device_printf(sc->sc_dev, "Invalid lmem addr or bytes\n"); return ENOBUFS; } if (uim->uim_scope != UOF_SCOPE_LOCAL) return EINVAL; sym = qat_aefw_uof_string(sc, uim->uim_sym_name); ael = strtoul(sym, &ep, 10); if (ep == sym || ael > MAX_AE) return EINVAL; if ((sc->sc_ae_mask & (1 << ael)) == 0) return 0; /* ae is fused out */ curinit = &qafu->qafu_num_lm_init[ael]; qabi_list = &qafu->qafu_lm_init[ael]; for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) { struct qat_ae_batch_init *qabi; qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init)); if (*curinit == 0) STAILQ_INIT(qabi_list); STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next); qabi->qabi_ae = (u_int)ael; qabi->qabi_addr = uim->uim_addr + memattr->umva_byte_offset; qabi->qabi_value = &memattr->umva_value; qabi->qabi_size = 4; qafu->qafu_num_lm_init_inst[ael] += qat_ae_get_inst_num(qabi->qabi_size); (*curinit)++; if (*curinit >= MAX_LMEM_REG) { device_printf(sc->sc_dev, "Invalid lmem val attr\n"); return ENOBUFS; } } break; case SRAM_REGION: case DRAM_REGION: case DRAM1_REGION: case SCRATCH_REGION: case UMEM_REGION: /* XXX */ /* fallthrough */ default: device_printf(sc->sc_dev, "unsupported memory region to init: %d\n", uim->uim_region); return ENOTSUP; } return 0; } static void qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae) { struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; struct qat_ae_batch_init *qabi; while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) { STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next); qat_free_mem(qabi); } qafu->qafu_num_lm_init[ae] = 0; qafu->qafu_num_lm_init_inst[ae] = 0; } static int qat_aefw_init_ustore(struct qat_softc *sc) { uint64_t *fill; uint32_t dont_init; int a, i, p; int error = 0; int usz, end, start; u_char ae, nae; fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t)); for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) { struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a]; struct uof_image *ui = qui->qui_image; for (i = 0; i < MAX_USTORE; i++) memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t)); /* * Compute do_not_init value as a value that will not be equal * to fill data when cast to an int */ dont_init = 0; if (dont_init == (uint32_t)fill[0]) dont_init = 0xffffffff; for (p = 0; p < ui->ui_num_pages; p++) { struct qat_uof_page *qup = &qui->qui_pages[p]; if (!qup->qup_def_page) continue; for (i = qup->qup_beg_paddr; i < qup->qup_beg_paddr + qup->qup_num_micro_words; i++ ) { fill[i] = (uint64_t)dont_init; } } for (ae = 0; ae < sc->sc_ae_num; ae++) { MPASS(ae < UOF_MAX_NUM_OF_AE); if ((ui->ui_ae_assigned & (1 << ae)) == 0) continue; if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) { qat_ae_get_shared_ustore_ae(ae, &nae); if (ui->ui_ae_assigned & (1 << ae)) continue; } usz = QAT_AE(sc, ae).qae_effect_ustore_size; /* initialize the areas not going to be overwritten */ end = -1; do { /* find next uword that needs to be initialized */ for (start = end + 1; start < usz; start++) { if ((uint32_t)fill[start] != dont_init) break; } /* see if there are no more such uwords */ if (start >= usz) break; for (end = start + 1; end < usz; end++) { if ((uint32_t)fill[end] == dont_init) break; } if (QAT_AE(sc, ae).qae_shareable_ustore) { error = ENOTSUP; /* XXX */ goto out; } else { error = qat_ae_ucode_write(sc, ae, start, end - start, &fill[start]); if (error) { goto out; } } } while (end < usz); } } out: qat_free_mem(fill); return error; } static int qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask, enum aereg_type regtype, u_short regaddr, u_int value) { int error = 0; u_char ctx; switch (regtype) { case AEREG_GPA_REL: case AEREG_GPB_REL: case AEREG_SR_REL: case AEREG_SR_RD_REL: case AEREG_SR_WR_REL: case AEREG_DR_REL: case AEREG_DR_RD_REL: case AEREG_DR_WR_REL: case AEREG_NEIGH_REL: /* init for all valid ctx */ for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { if ((ctx_mask & (1 << ctx)) == 0) continue; error = qat_aereg_rel_data_write(sc, ae, ctx, regtype, regaddr, value); } break; case AEREG_GPA_ABS: case AEREG_GPB_ABS: case AEREG_SR_ABS: case AEREG_SR_RD_ABS: case AEREG_SR_WR_ABS: case AEREG_DR_ABS: case AEREG_DR_RD_ABS: case AEREG_DR_WR_ABS: error = qat_aereg_abs_data_write(sc, ae, regtype, regaddr, value); break; default: error = EINVAL; break; } return error; } static int qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae, struct qat_uof_image *qui) { u_int i, expres; u_char ctx_mask; for (i = 0; i < qui->qui_num_init_reg_sym; i++) { struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i]; if (uirs->uirs_value_type == EXPR_VAL) { /* XXX */ device_printf(sc->sc_dev, "does not support initializing EXPR_VAL\n"); return ENOTSUP; } else { expres = uirs->uirs_value; } switch (uirs->uirs_init_type) { case INIT_REG: if (__SHIFTOUT(qui->qui_image->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) { ctx_mask = 0xff; /* 8-ctx mode */ } else { ctx_mask = 0x55; /* 4-ctx mode */ } qat_aefw_init_reg(sc, ae, ctx_mask, (enum aereg_type)uirs->uirs_reg_type, (u_short)uirs->uirs_addr_offset, expres); break; case INIT_REG_CTX: if (__SHIFTOUT(qui->qui_image->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) { ctx_mask = 0xff; /* 8-ctx mode */ } else { ctx_mask = 0x55; /* 4-ctx mode */ } if (((1 << uirs->uirs_ctx) & ctx_mask) == 0) return EINVAL; qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx, (enum aereg_type)uirs->uirs_reg_type, (u_short)uirs->uirs_addr_offset, expres); break; case INIT_EXPR: case INIT_EXPR_ENDIAN_SWAP: default: device_printf(sc->sc_dev, "does not support initializing init_type %d\n", uirs->uirs_init_type); return ENOTSUP; } } return 0; } static int qat_aefw_init_memory(struct qat_softc *sc) { struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; size_t uimsz, initmemsz = qafu->qafu_init_mem_size; struct uof_init_mem *uim; int error, i; u_char ae; uim = qafu->qafu_init_mem; for (i = 0; i < qafu->qafu_num_init_mem; i++) { uimsz = sizeof(struct uof_init_mem) + sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr; if (uimsz > initmemsz) { device_printf(sc->sc_dev, "invalid uof_init_mem or uof_mem_val_attr size\n"); return EINVAL; } if (uim->uim_num_bytes > 0) { error = qat_aefw_init_memory_one(sc, uim); if (error) { device_printf(sc->sc_dev, "Could not init ae memory: %d\n", error); return error; } } uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz); initmemsz -= uimsz; } /* run Batch put LM API */ for (ae = 0; ae < MAX_AE; ae++) { error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae], qafu->qafu_num_lm_init_inst[ae]); if (error) device_printf(sc->sc_dev, "Could not put lm\n"); qat_aefw_free_lm_init(sc, ae); } error = qat_aefw_init_ustore(sc); /* XXX run Batch put LM API */ return error; } static int qat_aefw_init_globals(struct qat_softc *sc) { struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; int error, i, p, s; u_char ae; /* initialize the memory segments */ if (qafu->qafu_num_init_mem > 0) { error = qat_aefw_init_memory(sc); if (error) return error; } else { error = qat_aefw_init_ustore(sc); if (error) return error; } /* XXX bind import variables with ivd values */ /* XXX bind the uC global variables * local variables will done on-the-fly */ for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) { struct qat_uof_page *qup = &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p]; if (qup->qup_num_uw_blocks && (qup->qup_num_uc_var || qup->qup_num_imp_var)) { device_printf(sc->sc_dev, "not support uC global variables\n"); return ENOTSUP; } } } for (ae = 0; ae < sc->sc_ae_num; ae++) { struct qat_ae *qae = &(QAT_AE(sc, ae)); for (s = 0; s < qae->qae_num_slices; s++) { struct qat_ae_slice *qas = &qae->qae_slices[s]; if (qas->qas_image == NULL) continue; error = qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image); if (error) return error; } } return 0; } static uint64_t qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup, u_int addr) { uint64_t uinst = 0; u_int i; /* find the block */ for (i = 0; i < qup->qup_num_uw_blocks; i++) { struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i]; if ((addr >= quub->quub_start_addr) && (addr <= (quub->quub_start_addr + (quub->quub_num_words - 1)))) { /* unpack n bytes and assigned to the 64-bit uword value. note: the microwords are stored as packed bytes. */ addr -= quub->quub_start_addr; addr *= AEV2_PACKED_UWORD_BYTES; memcpy(&uinst, (void *)((uintptr_t)quub->quub_micro_words + addr), AEV2_PACKED_UWORD_BYTES); uinst = uinst & UWORD_MASK; return uinst; } } return INVLD_UWORD; } static int qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup) { struct qat_ae *qae = &(QAT_AE(sc, ae)); uint64_t fill, *ucode_cpybuf; u_int error, i, upaddr, ninst, cpylen; if (qup->qup_num_uc_var || qup->qup_num_neigh_reg || qup->qup_num_imp_var || qup->qup_num_imp_expr) { device_printf(sc->sc_dev, "does not support fixup locals\n"); return ENOTSUP; } ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t)); /* XXX get fill-pattern from an image -- they are all the same */ memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern, sizeof(uint64_t)); upaddr = qup->qup_beg_paddr; ninst = qup->qup_num_micro_words; while (ninst > 0) { cpylen = min(ninst, UWORD_CPYBUF_SIZE); /* load the buffer */ for (i = 0; i < cpylen; i++) { /* keep below code structure in case there are * different handling for shared secnarios */ if (!qae->qae_shareable_ustore) { /* qat_aefw_get_uof_inst() takes an address that * is relative to the start of the page. * So we don't need to add in the physical * offset of the page. */ if (qup->qup_page_region != 0) { /* XXX */ device_printf(sc->sc_dev, "region != 0 is not supported\n"); qat_free_mem(ucode_cpybuf); return ENOTSUP; } else { /* for mixing case, it should take * physical address */ ucode_cpybuf[i] = qat_aefw_get_uof_inst( sc, qup, upaddr + i); if (ucode_cpybuf[i] == INVLD_UWORD) { /* fill hole in the uof */ ucode_cpybuf[i] = fill; } } } else { /* XXX */ qat_free_mem(ucode_cpybuf); return ENOTSUP; } } /* copy the buffer to ustore */ if (!qae->qae_shareable_ustore) { error = qat_ae_ucode_write(sc, ae, upaddr, cpylen, ucode_cpybuf); if (error) return error; } else { /* XXX */ qat_free_mem(ucode_cpybuf); return ENOTSUP; } upaddr += cpylen; ninst -= cpylen; } qat_free_mem(ucode_cpybuf); return 0; } static int qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui) { struct uof_image *ui = qui->qui_image; struct qat_ae_page *qap; u_int s, p, c; int error; u_char ae, ctx_mask; if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) ctx_mask = 0xff; /* 8-ctx mode */ else ctx_mask = 0x55; /* 4-ctx mode */ /* load the default page and set assigned CTX PC * to the entrypoint address */ for (ae = 0; ae < sc->sc_ae_num; ae++) { struct qat_ae *qae = &(QAT_AE(sc, ae)); struct qat_ae_slice *qas; u_int metadata; MPASS(ae < UOF_MAX_NUM_OF_AE); if ((ui->ui_ae_assigned & (1 << ae)) == 0) continue; /* find the slice to which this image is assigned */ for (s = 0; s < qae->qae_num_slices; s++) { qas = &qae->qae_slices[s]; if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask) break; } if (s >= qae->qae_num_slices) continue; qas = &qae->qae_slices[s]; for (p = 0; p < ui->ui_num_pages; p++) { qap = &qas->qas_pages[p]; /* Only load pages loaded by default */ if (!qap->qap_page->qup_def_page) continue; error = qat_aefw_do_pagein(sc, ae, qap->qap_page); if (error) return error; } metadata = qas->qas_image->qui_image->ui_app_metadata; if (metadata != 0xffffffff && bootverbose) { device_printf(sc->sc_dev, "loaded firmware: %s\n", qat_aefw_uof_string(sc, metadata)); } /* Assume starting page is page 0 */ qap = &qas->qas_pages[0]; for (c = 0; c < MAX_AE_CTX; c++) { if (ctx_mask & (1 << c)) qas->qas_cur_pages[c] = qap; else qas->qas_cur_pages[c] = NULL; } /* set the live context */ qae->qae_live_ctx_mask = ui->ui_ctx_assigned; /* set context PC to the image entrypoint address */ error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned, ui->ui_entry_address); if (error) return error; } /* XXX store the checksum for convenience */ return 0; } static int qat_aefw_uof_write(struct qat_softc *sc) { int error = 0; int i; error = qat_aefw_init_globals(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize globals\n"); return error; } for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { error = qat_aefw_uof_write_one(sc, &sc->sc_aefw_uof.qafu_imgs[i]); if (error) break; } /* XXX UcLo_computeFreeUstore */ return error; }