1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2021, Microsoft Corporation. */
3
4 #include <linux/delay.h>
5 #include <linux/device.h>
6 #include <linux/io.h>
7 #include <linux/mm.h>
8
9 #include <net/mana/gdma.h>
10 #include <net/mana/shm_channel.h>
11
12 #define PAGE_FRAME_L48_WIDTH_BYTES 6
13 #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
14 #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
15 #define PAGE_FRAME_H4_WIDTH_BITS 4
16 #define VECTOR_MASK 0xFFFF
17 #define SHMEM_VF_RESET_STATE ((u32)-1)
18
19 #define SMC_MSG_TYPE_ESTABLISH_HWC 1
20 #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
21
22 #define SMC_MSG_TYPE_DESTROY_HWC 2
23 #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
24
25 #define SMC_MSG_DIRECTION_REQUEST 0
26 #define SMC_MSG_DIRECTION_RESPONSE 1
27
28 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
29 * them are naturally aligned and hence don't need __packed.
30 */
31
32 /* Shared memory channel protocol header
33 *
34 * msg_type: set on request and response; response matches request.
35 * msg_version: newer PF writes back older response (matching request)
36 * older PF acts on latest version known and sets that version in result
37 * (less than request).
38 * direction: 0 for request, VF->PF; 1 for response, PF->VF.
39 * status: 0 on request,
40 * operation result on response (success = 0, failure = 1 or greater).
41 * reset_vf: If set on either establish or destroy request, indicates perform
42 * FLR before/after the operation.
43 * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
44 */
45 union smc_proto_hdr {
46 u32 as_uint32;
47
48 struct {
49 u8 msg_type : 3;
50 u8 msg_version : 3;
51 u8 reserved_1 : 1;
52 u8 direction : 1;
53
54 u8 status;
55
56 u8 reserved_2;
57
58 u8 reset_vf : 1;
59 u8 reserved_3 : 6;
60 u8 owner_is_pf : 1;
61 };
62 }; /* HW DATA */
63
mana_smc_poll_register(void __iomem * base,bool reset)64 static int mana_smc_poll_register(void __iomem *base, bool reset)
65 {
66 void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
67 u32 last_dword;
68 int i;
69
70 /* Poll the hardware for the ownership bit. This should be pretty fast,
71 * but let's do it in a loop just in case the hardware or the PF
72 * driver are temporarily busy.
73 */
74 for (i = 0; i < 20 * 1000; i++) {
75 last_dword = readl(ptr);
76
77 /* shmem reads as 0xFFFFFFFF in the reset case */
78 if (reset && last_dword == SHMEM_VF_RESET_STATE)
79 return 0;
80
81 /* If bit_31 is set, the PF currently owns the SMC. */
82 if (!(last_dword & BIT(31)))
83 return 0;
84
85 usleep_range(1000, 2000);
86 }
87
88 return -ETIMEDOUT;
89 }
90
mana_smc_read_response(struct shm_channel * sc,u32 msg_type,u32 msg_version,bool reset_vf)91 static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
92 u32 msg_version, bool reset_vf)
93 {
94 void __iomem *base = sc->base;
95 union smc_proto_hdr hdr;
96 int err;
97
98 /* Wait for PF to respond. */
99 err = mana_smc_poll_register(base, reset_vf);
100 if (err)
101 return err;
102
103 hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
104
105 if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
106 return 0;
107
108 /* Validate protocol fields from the PF driver */
109 if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
110 hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
111 dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
112 hdr.as_uint32, msg_type, msg_version);
113 return -EPROTO;
114 }
115
116 /* Validate the operation result */
117 if (hdr.status != 0) {
118 dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
119 return -EPROTO;
120 }
121
122 return 0;
123 }
124
mana_smc_init(struct shm_channel * sc,struct device * dev,void __iomem * base)125 void mana_smc_init(struct shm_channel *sc, struct device *dev,
126 void __iomem *base)
127 {
128 sc->dev = dev;
129 sc->base = base;
130 }
131
mana_smc_setup_hwc(struct shm_channel * sc,bool reset_vf,u64 eq_addr,u64 cq_addr,u64 rq_addr,u64 sq_addr,u32 eq_msix_index)132 int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
133 u64 cq_addr, u64 rq_addr, u64 sq_addr,
134 u32 eq_msix_index)
135 {
136 union smc_proto_hdr *hdr;
137 u16 all_addr_h4bits = 0;
138 u16 frame_addr_seq = 0;
139 u64 frame_addr = 0;
140 u8 shm_buf[32];
141 u64 *shmem;
142 u32 *dword;
143 u8 *ptr;
144 int err;
145 int i;
146
147 /* Ensure VF already has possession of shared memory */
148 err = mana_smc_poll_register(sc->base, false);
149 if (err) {
150 dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
151 return err;
152 }
153
154 if (!MANA_PAGE_ALIGNED(eq_addr) || !MANA_PAGE_ALIGNED(cq_addr) ||
155 !MANA_PAGE_ALIGNED(rq_addr) || !MANA_PAGE_ALIGNED(sq_addr))
156 return -EINVAL;
157
158 if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
159 return -EINVAL;
160
161 /* Scheme for packing four addresses and extra info into 256 bits.
162 *
163 * Addresses must be page frame aligned, so only frame address bits
164 * are transferred.
165 *
166 * 52-bit frame addresses are split into the lower 48 bits and upper
167 * 4 bits. Lower 48 bits of 4 address are written sequentially from
168 * the start of the 256-bit shared memory region followed by 16 bits
169 * containing the upper 4 bits of the 4 addresses in sequence.
170 *
171 * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
172 *
173 * The final 32-bit dword is used for protocol control information as
174 * defined in smc_proto_hdr.
175 */
176
177 memset(shm_buf, 0, sizeof(shm_buf));
178 ptr = shm_buf;
179
180 /* EQ addr: low 48 bits of frame address */
181 shmem = (u64 *)ptr;
182 frame_addr = MANA_PFN(eq_addr);
183 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
184 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
185 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
186 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
187
188 /* CQ addr: low 48 bits of frame address */
189 shmem = (u64 *)ptr;
190 frame_addr = MANA_PFN(cq_addr);
191 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
192 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
193 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
194 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
195
196 /* RQ addr: low 48 bits of frame address */
197 shmem = (u64 *)ptr;
198 frame_addr = MANA_PFN(rq_addr);
199 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
200 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
201 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
202 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
203
204 /* SQ addr: low 48 bits of frame address */
205 shmem = (u64 *)ptr;
206 frame_addr = MANA_PFN(sq_addr);
207 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
208 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
209 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
210 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
211
212 /* High 4 bits of the four frame addresses */
213 *((u16 *)ptr) = all_addr_h4bits;
214 ptr += sizeof(u16);
215
216 /* EQ MSIX vector number */
217 *((u16 *)ptr) = (u16)eq_msix_index;
218 ptr += sizeof(u16);
219
220 /* 32-bit protocol header in final dword */
221 *((u32 *)ptr) = 0;
222
223 hdr = (union smc_proto_hdr *)ptr;
224 hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
225 hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
226 hdr->direction = SMC_MSG_DIRECTION_REQUEST;
227 hdr->reset_vf = reset_vf;
228
229 /* Write 256-message buffer to shared memory (final 32-bit write
230 * triggers HW to set possession bit to PF).
231 */
232 dword = (u32 *)shm_buf;
233 for (i = 0; i < SMC_APERTURE_DWORDS; i++)
234 writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
235
236 /* Read shmem response (polling for VF possession) and validate.
237 * For setup, waiting for response on shared memory is not strictly
238 * necessary, since wait occurs later for results to appear in EQE's.
239 */
240 err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
241 SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
242 reset_vf);
243 if (err) {
244 dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
245 return err;
246 }
247
248 return 0;
249 }
250
mana_smc_teardown_hwc(struct shm_channel * sc,bool reset_vf)251 int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
252 {
253 union smc_proto_hdr hdr = {};
254 int err;
255
256 /* Ensure already has possession of shared memory */
257 err = mana_smc_poll_register(sc->base, false);
258 if (err) {
259 dev_err(sc->dev, "Timeout when tearing down HWC\n");
260 return err;
261 }
262
263 /* Set up protocol header for HWC destroy message */
264 hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
265 hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
266 hdr.direction = SMC_MSG_DIRECTION_REQUEST;
267 hdr.reset_vf = reset_vf;
268
269 /* Write message in high 32 bits of 256-bit shared memory, causing HW
270 * to set possession bit to PF.
271 */
272 writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
273
274 /* Read shmem response (polling for VF possession) and validate.
275 * For teardown, waiting for response is required to ensure hardware
276 * invalidates MST entries before software frees memory.
277 */
278 err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
279 SMC_MSG_TYPE_DESTROY_HWC_VERSION,
280 reset_vf);
281 if (err) {
282 dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
283 return err;
284 }
285
286 return 0;
287 }
288