1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021 Microsoft Corp. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/types.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 38 #include "mana.h" 39 #include "shm_channel.h" 40 #include "gdma_util.h" 41 42 #define PAGE_FRAME_L48_WIDTH_BYTES 6 43 #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8) 44 #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF 45 #define PAGE_FRAME_H4_WIDTH_BITS 4 46 #define VECTOR_MASK 0xFFFF 47 #define SHMEM_VF_RESET_STATE ((uint32_t)-1) 48 49 #define SMC_MSG_TYPE_ESTABLISH_HWC 1 50 #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0 51 52 #define SMC_MSG_TYPE_DESTROY_HWC 2 53 #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0 54 55 #define SMC_MSG_DIRECTION_REQUEST 0 56 #define SMC_MSG_DIRECTION_RESPONSE 1 57 58 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of 59 * them are naturally aligned and hence don't need __packed. 60 */ 61 62 /* Shared memory channel protocol header 63 * 64 * msg_type: set on request and response; response matches request. 65 * msg_version: newer PF writes back older response (matching request) 66 * older PF acts on latest version known and sets that version in result 67 * (less than request). 68 * direction: 0 for request, VF->PF; 1 for response, PF->VF. 69 * status: 0 on request, 70 * operation result on response (success = 0, failure = 1 or greater). 71 * reset_vf: If set on either establish or destroy request, indicates perform 72 * FLR before/after the operation. 73 * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned. 74 */ 75 union smc_proto_hdr { 76 uint32_t as_uint32; 77 78 struct { 79 uint8_t msg_type : 3; 80 uint8_t msg_version : 3; 81 uint8_t reserved_1 : 1; 82 uint8_t direction : 1; 83 84 uint8_t status; 85 86 uint8_t reserved_2; 87 88 uint8_t reset_vf : 1; 89 uint8_t reserved_3 : 6; 90 uint8_t owner_is_pf : 1; 91 }; 92 }; /* HW DATA */ 93 94 #define SMC_APERTURE_BITS 256 95 #define SMC_BASIC_UNIT (sizeof(uint32_t)) 96 #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8)) 97 #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1) 98 99 static int 100 mana_smc_poll_register(void __iomem *base, bool reset) 101 { 102 void __iomem *ptr = (uint8_t *)base + SMC_LAST_DWORD * SMC_BASIC_UNIT; 103 volatile uint32_t last_dword; 104 int i; 105 106 /* Poll the hardware for the ownership bit. This should be pretty fast, 107 * but let's do it in a loop just in case the hardware or the PF 108 * driver are temporarily busy. 109 */ 110 for (i = 0; i < 20 * 1000; i++) { 111 last_dword = readl(ptr); 112 113 /* shmem reads as 0xFFFFFFFF in the reset case */ 114 if (reset && last_dword == SHMEM_VF_RESET_STATE) 115 return 0; 116 117 /* If bit_31 is set, the PF currently owns the SMC. */ 118 if (!(last_dword & BIT(31))) 119 return 0; 120 121 DELAY(1000); 122 } 123 124 return ETIMEDOUT; 125 } 126 127 static int 128 mana_smc_read_response(struct shm_channel *sc, uint32_t msg_type, 129 uint32_t msg_version, bool reset_vf) 130 { 131 void __iomem *base = sc->base; 132 union smc_proto_hdr hdr; 133 int err; 134 135 /* Wait for PF to respond. */ 136 err = mana_smc_poll_register(base, reset_vf); 137 if (err) 138 return err; 139 140 hdr.as_uint32 = 141 readl((uint8_t *)base + SMC_LAST_DWORD * SMC_BASIC_UNIT); 142 mana_dbg(NULL, "shm response 0x%x\n", hdr.as_uint32); 143 144 if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE) 145 return 0; 146 147 /* Validate protocol fields from the PF driver */ 148 if (hdr.msg_type != msg_type || hdr.msg_version > msg_version || 149 hdr.direction != SMC_MSG_DIRECTION_RESPONSE) { 150 device_printf(sc->dev, 151 "Wrong SMC response 0x%x, type=%d, ver=%d\n", 152 hdr.as_uint32, msg_type, msg_version); 153 return EPROTO; 154 } 155 156 /* Validate the operation result */ 157 if (hdr.status != 0) { 158 device_printf(sc->dev, 159 "SMC operation failed: 0x%x\n", hdr.status); 160 return EPROTO; 161 } 162 163 return 0; 164 } 165 166 void 167 mana_smc_init(struct shm_channel *sc, device_t dev, void __iomem *base) 168 { 169 sc->dev = dev; 170 sc->base = base; 171 } 172 173 int 174 mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, uint64_t eq_addr, 175 uint64_t cq_addr, uint64_t rq_addr, uint64_t sq_addr, 176 uint32_t eq_msix_index) 177 { 178 union smc_proto_hdr *hdr; 179 uint16_t all_addr_h4bits = 0; 180 uint16_t frame_addr_seq = 0; 181 uint64_t frame_addr = 0; 182 uint8_t shm_buf[32]; 183 uint64_t *shmem; 184 uint32_t *dword; 185 uint8_t *ptr; 186 int err; 187 int i; 188 189 /* Ensure VF already has possession of shared memory */ 190 err = mana_smc_poll_register(sc->base, false); 191 if (err) { 192 device_printf(sc->dev, 193 "Timeout when setting up HWC: %d\n", err); 194 return err; 195 } 196 197 if (!IS_ALIGNED(eq_addr, PAGE_SIZE) || 198 !IS_ALIGNED(cq_addr, PAGE_SIZE) || 199 !IS_ALIGNED(rq_addr, PAGE_SIZE) || 200 !IS_ALIGNED(sq_addr, PAGE_SIZE)) 201 return EINVAL; 202 203 if ((eq_msix_index & VECTOR_MASK) != eq_msix_index) 204 return EINVAL; 205 206 /* Scheme for packing four addresses and extra info into 256 bits. 207 * 208 * Addresses must be page frame aligned, so only frame address bits 209 * are transferred. 210 * 211 * 52-bit frame addresses are split into the lower 48 bits and upper 212 * 4 bits. Lower 48 bits of 4 address are written sequentially from 213 * the start of the 256-bit shared memory region followed by 16 bits 214 * containing the upper 4 bits of the 4 addresses in sequence. 215 * 216 * A 16 bit EQ vector number fills out the next-to-last 32-bit dword. 217 * 218 * The final 32-bit dword is used for protocol control information as 219 * defined in smc_proto_hdr. 220 */ 221 222 memset(shm_buf, 0, sizeof(shm_buf)); 223 ptr = shm_buf; 224 225 /* EQ addr: low 48 bits of frame address */ 226 shmem = (uint64_t *)ptr; 227 frame_addr = PHYS_PFN(eq_addr); 228 *shmem = frame_addr & PAGE_FRAME_L48_MASK; 229 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 230 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 231 ptr += PAGE_FRAME_L48_WIDTH_BYTES; 232 233 /* CQ addr: low 48 bits of frame address */ 234 shmem = (uint64_t *)ptr; 235 frame_addr = PHYS_PFN(cq_addr); 236 *shmem = frame_addr & PAGE_FRAME_L48_MASK; 237 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 238 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 239 ptr += PAGE_FRAME_L48_WIDTH_BYTES; 240 241 /* RQ addr: low 48 bits of frame address */ 242 shmem = (uint64_t *)ptr; 243 frame_addr = PHYS_PFN(rq_addr); 244 *shmem = frame_addr & PAGE_FRAME_L48_MASK; 245 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 246 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 247 ptr += PAGE_FRAME_L48_WIDTH_BYTES; 248 249 /* SQ addr: low 48 bits of frame address */ 250 shmem = (uint64_t *)ptr; 251 frame_addr = PHYS_PFN(sq_addr); 252 *shmem = frame_addr & PAGE_FRAME_L48_MASK; 253 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 254 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 255 ptr += PAGE_FRAME_L48_WIDTH_BYTES; 256 257 /* High 4 bits of the four frame addresses */ 258 *((uint16_t *)ptr) = all_addr_h4bits; 259 ptr += sizeof(uint16_t); 260 261 /* EQ MSIX vector number */ 262 *((uint16_t *)ptr) = (uint16_t)eq_msix_index; 263 ptr += sizeof(uint16_t); 264 265 /* 32-bit protocol header in final dword */ 266 *((uint32_t *)ptr) = 0; 267 268 hdr = (union smc_proto_hdr *)ptr; 269 hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC; 270 hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION; 271 hdr->direction = SMC_MSG_DIRECTION_REQUEST; 272 hdr->reset_vf = reset_vf; 273 274 /* Write 256-message buffer to shared memory (final 32-bit write 275 * triggers HW to set possession bit to PF). 276 */ 277 dword = (uint32_t *)shm_buf; 278 for (i = 0; i < SMC_APERTURE_DWORDS; i++) { 279 mana_dbg(NULL, "write shm_buf %d, val: 0x%x\n", 280 i, *dword); 281 writel((char *)sc->base + i * SMC_BASIC_UNIT, *dword++); 282 } 283 284 /* Read shmem response (polling for VF possession) and validate. 285 * For setup, waiting for response on shared memory is not strictly 286 * necessary, since wait occurs later for results to appear in EQE's. 287 */ 288 err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC, 289 SMC_MSG_TYPE_ESTABLISH_HWC_VERSION, reset_vf); 290 if (err) { 291 device_printf(sc->dev, 292 "Error when setting up HWC: %d\n", err); 293 return err; 294 } 295 296 return 0; 297 } 298 299 int 300 mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf) 301 { 302 union smc_proto_hdr hdr = {}; 303 int err; 304 305 /* Ensure already has possession of shared memory */ 306 err = mana_smc_poll_register(sc->base, false); 307 if (err) { 308 device_printf(sc->dev, "Timeout when tearing down HWC\n"); 309 return err; 310 } 311 312 /* Set up protocol header for HWC destroy message */ 313 hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC; 314 hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION; 315 hdr.direction = SMC_MSG_DIRECTION_REQUEST; 316 hdr.reset_vf = reset_vf; 317 318 /* Write message in high 32 bits of 256-bit shared memory, causing HW 319 * to set possession bit to PF. 320 */ 321 writel((char *)sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT, 322 hdr.as_uint32); 323 324 /* Read shmem response (polling for VF possession) and validate. 325 * For teardown, waiting for response is required to ensure hardware 326 * invalidates MST entries before software frees memory. 327 */ 328 err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC, 329 SMC_MSG_TYPE_DESTROY_HWC_VERSION, reset_vf); 330 if (err) { 331 device_printf(sc->dev, 332 "Error when tearing down HWC: %d\n", err); 333 return err; 334 } 335 336 return 0; 337 } 338