1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Microsoft Corp.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/types.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35
36 #include "mana.h"
37 #include "shm_channel.h"
38 #include "gdma_util.h"
39
40 #define PAGE_FRAME_L48_WIDTH_BYTES 6
41 #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
42 #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
43 #define PAGE_FRAME_H4_WIDTH_BITS 4
44 #define VECTOR_MASK 0xFFFF
45 #define SHMEM_VF_RESET_STATE ((uint32_t)-1)
46
47 #define SMC_MSG_TYPE_ESTABLISH_HWC 1
48 #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
49
50 #define SMC_MSG_TYPE_DESTROY_HWC 2
51 #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
52
53 #define SMC_MSG_DIRECTION_REQUEST 0
54 #define SMC_MSG_DIRECTION_RESPONSE 1
55
56 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
57 * them are naturally aligned and hence don't need __packed.
58 */
59
60 /* Shared memory channel protocol header
61 *
62 * msg_type: set on request and response; response matches request.
63 * msg_version: newer PF writes back older response (matching request)
64 * older PF acts on latest version known and sets that version in result
65 * (less than request).
66 * direction: 0 for request, VF->PF; 1 for response, PF->VF.
67 * status: 0 on request,
68 * operation result on response (success = 0, failure = 1 or greater).
69 * reset_vf: If set on either establish or destroy request, indicates perform
70 * FLR before/after the operation.
71 * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
72 */
73 union smc_proto_hdr {
74 uint32_t as_uint32;
75
76 struct {
77 uint8_t msg_type : 3;
78 uint8_t msg_version : 3;
79 uint8_t reserved_1 : 1;
80 uint8_t direction : 1;
81
82 uint8_t status;
83
84 uint8_t reserved_2;
85
86 uint8_t reset_vf : 1;
87 uint8_t reserved_3 : 6;
88 uint8_t owner_is_pf : 1;
89 };
90 }; /* HW DATA */
91
92 #define SMC_APERTURE_BITS 256
93 #define SMC_BASIC_UNIT (sizeof(uint32_t))
94 #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
95 #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
96
97 static int
mana_smc_poll_register(void __iomem * base,bool reset)98 mana_smc_poll_register(void __iomem *base, bool reset)
99 {
100 void __iomem *ptr = (uint8_t *)base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
101 volatile uint32_t last_dword;
102 int i;
103
104 /* Poll the hardware for the ownership bit. This should be pretty fast,
105 * but let's do it in a loop just in case the hardware or the PF
106 * driver are temporarily busy.
107 */
108 for (i = 0; i < 20 * 1000; i++) {
109 last_dword = readl(ptr);
110
111 /* shmem reads as 0xFFFFFFFF in the reset case */
112 if (reset && last_dword == SHMEM_VF_RESET_STATE)
113 return 0;
114
115 /* If bit_31 is set, the PF currently owns the SMC. */
116 if (!(last_dword & BIT(31)))
117 return 0;
118
119 DELAY(1000);
120 }
121
122 return ETIMEDOUT;
123 }
124
125 static int
mana_smc_read_response(struct shm_channel * sc,uint32_t msg_type,uint32_t msg_version,bool reset_vf)126 mana_smc_read_response(struct shm_channel *sc, uint32_t msg_type,
127 uint32_t msg_version, bool reset_vf)
128 {
129 void __iomem *base = sc->base;
130 union smc_proto_hdr hdr;
131 int err;
132
133 /* Wait for PF to respond. */
134 err = mana_smc_poll_register(base, reset_vf);
135 if (err)
136 return err;
137
138 hdr.as_uint32 =
139 readl((uint8_t *)base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
140 mana_dbg(NULL, "shm response 0x%x\n", hdr.as_uint32);
141
142 if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
143 return 0;
144
145 /* Validate protocol fields from the PF driver */
146 if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
147 hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
148 device_printf(sc->dev,
149 "Wrong SMC response 0x%x, type=%d, ver=%d\n",
150 hdr.as_uint32, msg_type, msg_version);
151 return EPROTO;
152 }
153
154 /* Validate the operation result */
155 if (hdr.status != 0) {
156 device_printf(sc->dev,
157 "SMC operation failed: 0x%x\n", hdr.status);
158 return EPROTO;
159 }
160
161 return 0;
162 }
163
164 void
mana_smc_init(struct shm_channel * sc,device_t dev,void __iomem * base)165 mana_smc_init(struct shm_channel *sc, device_t dev, void __iomem *base)
166 {
167 sc->dev = dev;
168 sc->base = base;
169 }
170
171 int
mana_smc_setup_hwc(struct shm_channel * sc,bool reset_vf,uint64_t eq_addr,uint64_t cq_addr,uint64_t rq_addr,uint64_t sq_addr,uint32_t eq_msix_index)172 mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, uint64_t eq_addr,
173 uint64_t cq_addr, uint64_t rq_addr, uint64_t sq_addr,
174 uint32_t eq_msix_index)
175 {
176 union smc_proto_hdr *hdr;
177 uint16_t all_addr_h4bits = 0;
178 uint16_t frame_addr_seq = 0;
179 uint64_t frame_addr = 0;
180 uint8_t shm_buf[32];
181 uint64_t *shmem;
182 uint32_t *dword;
183 uint8_t *ptr;
184 int err;
185 int i;
186
187 /* Ensure VF already has possession of shared memory */
188 err = mana_smc_poll_register(sc->base, false);
189 if (err) {
190 device_printf(sc->dev,
191 "Timeout when setting up HWC: %d\n", err);
192 return err;
193 }
194
195 if (!IS_ALIGNED(eq_addr, PAGE_SIZE) ||
196 !IS_ALIGNED(cq_addr, PAGE_SIZE) ||
197 !IS_ALIGNED(rq_addr, PAGE_SIZE) ||
198 !IS_ALIGNED(sq_addr, PAGE_SIZE))
199 return EINVAL;
200
201 if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
202 return EINVAL;
203
204 /* Scheme for packing four addresses and extra info into 256 bits.
205 *
206 * Addresses must be page frame aligned, so only frame address bits
207 * are transferred.
208 *
209 * 52-bit frame addresses are split into the lower 48 bits and upper
210 * 4 bits. Lower 48 bits of 4 address are written sequentially from
211 * the start of the 256-bit shared memory region followed by 16 bits
212 * containing the upper 4 bits of the 4 addresses in sequence.
213 *
214 * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
215 *
216 * The final 32-bit dword is used for protocol control information as
217 * defined in smc_proto_hdr.
218 */
219
220 memset(shm_buf, 0, sizeof(shm_buf));
221 ptr = shm_buf;
222
223 /* EQ addr: low 48 bits of frame address */
224 shmem = (uint64_t *)ptr;
225 frame_addr = PHYS_PFN(eq_addr);
226 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
227 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
228 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
229 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
230
231 /* CQ addr: low 48 bits of frame address */
232 shmem = (uint64_t *)ptr;
233 frame_addr = PHYS_PFN(cq_addr);
234 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
235 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
236 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
237 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
238
239 /* RQ addr: low 48 bits of frame address */
240 shmem = (uint64_t *)ptr;
241 frame_addr = PHYS_PFN(rq_addr);
242 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
243 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
244 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
245 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
246
247 /* SQ addr: low 48 bits of frame address */
248 shmem = (uint64_t *)ptr;
249 frame_addr = PHYS_PFN(sq_addr);
250 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
251 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
252 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
253 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
254
255 /* High 4 bits of the four frame addresses */
256 *((uint16_t *)ptr) = all_addr_h4bits;
257 ptr += sizeof(uint16_t);
258
259 /* EQ MSIX vector number */
260 *((uint16_t *)ptr) = (uint16_t)eq_msix_index;
261 ptr += sizeof(uint16_t);
262
263 /* 32-bit protocol header in final dword */
264 *((uint32_t *)ptr) = 0;
265
266 hdr = (union smc_proto_hdr *)ptr;
267 hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
268 hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
269 hdr->direction = SMC_MSG_DIRECTION_REQUEST;
270 hdr->reset_vf = reset_vf;
271
272 /* Write 256-message buffer to shared memory (final 32-bit write
273 * triggers HW to set possession bit to PF).
274 */
275 dword = (uint32_t *)shm_buf;
276 for (i = 0; i < SMC_APERTURE_DWORDS; i++) {
277 mana_dbg(NULL, "write shm_buf %d, val: 0x%x\n",
278 i, *dword);
279 writel((char *)sc->base + i * SMC_BASIC_UNIT, *dword++);
280 }
281
282 /* Read shmem response (polling for VF possession) and validate.
283 * For setup, waiting for response on shared memory is not strictly
284 * necessary, since wait occurs later for results to appear in EQE's.
285 */
286 err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
287 SMC_MSG_TYPE_ESTABLISH_HWC_VERSION, reset_vf);
288 if (err) {
289 device_printf(sc->dev,
290 "Error when setting up HWC: %d\n", err);
291 return err;
292 }
293
294 return 0;
295 }
296
297 int
mana_smc_teardown_hwc(struct shm_channel * sc,bool reset_vf)298 mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
299 {
300 union smc_proto_hdr hdr = {};
301 int err;
302
303 /* Ensure already has possession of shared memory */
304 err = mana_smc_poll_register(sc->base, false);
305 if (err) {
306 device_printf(sc->dev, "Timeout when tearing down HWC\n");
307 return err;
308 }
309
310 /* Set up protocol header for HWC destroy message */
311 hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
312 hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
313 hdr.direction = SMC_MSG_DIRECTION_REQUEST;
314 hdr.reset_vf = reset_vf;
315
316 /* Write message in high 32 bits of 256-bit shared memory, causing HW
317 * to set possession bit to PF.
318 */
319 writel((char *)sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT,
320 hdr.as_uint32);
321
322 /* Read shmem response (polling for VF possession) and validate.
323 * For teardown, waiting for response is required to ensure hardware
324 * invalidates MST entries before software frees memory.
325 */
326 err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
327 SMC_MSG_TYPE_DESTROY_HWC_VERSION, reset_vf);
328 if (err) {
329 device_printf(sc->dev,
330 "Error when tearing down HWC: %d\n", err);
331 return err;
332 }
333
334 return 0;
335 }
336