1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2023-2024 Google LLC
5 *
6 * Redistribution and use in source and binary forms, with or without modification,
7 * are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * 3. Neither the name of the copyright holder nor the names of its contributors
17 * may be used to endorse or promote products derived from this software without
18 * specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31 #include <sys/endian.h>
32 #include <sys/socket.h>
33 #include <sys/time.h>
34
35 #include <net/ethernet.h>
36 #include <net/if.h>
37 #include <net/if_var.h>
38
39 #include "gve.h"
40 #include "gve_adminq.h"
41
42 #define GVE_ADMINQ_SLEEP_LEN_MS 20
43 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 10
44 #define GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION 1
45 #define GVE_REG_ADMINQ_ADDR 16
46 #define ADMINQ_SLOTS (ADMINQ_SIZE / sizeof(struct gve_adminq_command))
47
48 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
49 "Expected: length=%d, feature_mask=%x.\n" \
50 "Actual: length=%d, feature_mask=%x.\n"
51
52 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected." \
53 " Possible older version of guest driver.\n"
54
55 static
gve_parse_device_option(struct gve_priv * priv,struct gve_device_descriptor * device_descriptor,struct gve_device_option * option,struct gve_device_option_gqi_qpl ** dev_op_gqi_qpl,struct gve_device_option_dqo_rda ** dev_op_dqo_rda,struct gve_device_option_dqo_qpl ** dev_op_dqo_qpl,struct gve_device_option_modify_ring ** dev_op_modify_ring,struct gve_device_option_jumbo_frames ** dev_op_jumbo_frames)56 void gve_parse_device_option(struct gve_priv *priv,
57 struct gve_device_descriptor *device_descriptor,
58 struct gve_device_option *option,
59 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
60 struct gve_device_option_dqo_rda **dev_op_dqo_rda,
61 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl,
62 struct gve_device_option_modify_ring **dev_op_modify_ring,
63 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
64 {
65 uint32_t req_feat_mask = be32toh(option->required_features_mask);
66 uint16_t option_length = be16toh(option->option_length);
67 uint16_t option_id = be16toh(option->option_id);
68
69 /*
70 * If the length or feature mask doesn't match, continue without
71 * enabling the feature.
72 */
73 switch (option_id) {
74 case GVE_DEV_OPT_ID_GQI_QPL:
75 if (option_length < sizeof(**dev_op_gqi_qpl) ||
76 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
77 device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
78 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
79 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
80 option_length, req_feat_mask);
81 break;
82 }
83
84 if (option_length > sizeof(**dev_op_gqi_qpl)) {
85 device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
86 "GQI QPL");
87 }
88 *dev_op_gqi_qpl = (void *)(option + 1);
89 break;
90
91 case GVE_DEV_OPT_ID_DQO_RDA:
92 if (option_length < sizeof(**dev_op_dqo_rda) ||
93 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
94 device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
95 "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
96 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
97 option_length, req_feat_mask);
98 break;
99 }
100
101 if (option_length > sizeof(**dev_op_dqo_rda)) {
102 device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
103 "DQO RDA");
104 }
105 *dev_op_dqo_rda = (void *)(option + 1);
106 break;
107
108 case GVE_DEV_OPT_ID_DQO_QPL:
109 if (option_length < sizeof(**dev_op_dqo_qpl) ||
110 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) {
111 device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
112 "DQO QPL", (int)sizeof(**dev_op_dqo_qpl),
113 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL,
114 option_length, req_feat_mask);
115 break;
116 }
117
118 if (option_length > sizeof(**dev_op_dqo_qpl)) {
119 device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
120 "DQO QPL");
121 }
122 *dev_op_dqo_qpl = (void *)(option + 1);
123 break;
124
125 case GVE_DEV_OPT_ID_MODIFY_RING:
126 if (option_length < (sizeof(**dev_op_modify_ring) -
127 sizeof(struct gve_ring_size_bound)) ||
128 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING) {
129 device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
130 "Modify Ring", (int)sizeof(**dev_op_modify_ring),
131 GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING,
132 option_length, req_feat_mask);
133 break;
134 }
135
136 if (option_length > sizeof(**dev_op_modify_ring)) {
137 device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
138 "Modify Ring");
139 }
140 *dev_op_modify_ring = (void *)(option + 1);
141
142 /* Min ring size included; set the minimum ring size. */
143 if (option_length == sizeof(**dev_op_modify_ring)) {
144 priv->min_rx_desc_cnt = max(
145 be16toh((*dev_op_modify_ring)->min_ring_size.rx),
146 GVE_DEFAULT_MIN_RX_RING_SIZE);
147 priv->min_tx_desc_cnt = max(
148 be16toh((*dev_op_modify_ring)->min_ring_size.tx),
149 GVE_DEFAULT_MIN_TX_RING_SIZE);
150 }
151 break;
152
153 case GVE_DEV_OPT_ID_JUMBO_FRAMES:
154 if (option_length < sizeof(**dev_op_jumbo_frames) ||
155 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
156 device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
157 "Jumbo Frames", (int)sizeof(**dev_op_jumbo_frames),
158 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
159 option_length, req_feat_mask);
160 break;
161 }
162
163 if (option_length > sizeof(**dev_op_jumbo_frames)) {
164 device_printf(priv->dev,
165 GVE_DEVICE_OPTION_TOO_BIG_FMT, "Jumbo Frames");
166 }
167 *dev_op_jumbo_frames = (void *)(option + 1);
168 break;
169
170 default:
171 /*
172 * If we don't recognize the option just continue
173 * without doing anything.
174 */
175 device_printf(priv->dev, "Unrecognized device option 0x%hx not enabled.\n",
176 option_id);
177 }
178 }
179
180 /* Process all device options for a given describe device call. */
181 static int
gve_process_device_options(struct gve_priv * priv,struct gve_device_descriptor * descriptor,struct gve_device_option_gqi_qpl ** dev_op_gqi_qpl,struct gve_device_option_dqo_rda ** dev_op_dqo_rda,struct gve_device_option_dqo_qpl ** dev_op_dqo_qpl,struct gve_device_option_modify_ring ** dev_op_modify_ring,struct gve_device_option_jumbo_frames ** dev_op_jumbo_frames)182 gve_process_device_options(struct gve_priv *priv,
183 struct gve_device_descriptor *descriptor,
184 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
185 struct gve_device_option_dqo_rda **dev_op_dqo_rda,
186 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl,
187 struct gve_device_option_modify_ring **dev_op_modify_ring,
188 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
189 {
190 char *desc_end = (char *)descriptor + be16toh(descriptor->total_length);
191 const int num_options = be16toh(descriptor->num_device_options);
192 struct gve_device_option *dev_opt;
193 int i;
194
195 /* The options struct directly follows the device descriptor. */
196 dev_opt = (void *)(descriptor + 1);
197 for (i = 0; i < num_options; i++) {
198 if ((char *)(dev_opt + 1) > desc_end ||
199 (char *)(dev_opt + 1) + be16toh(dev_opt->option_length) > desc_end) {
200 device_printf(priv->dev,
201 "options exceed device descriptor's total length.\n");
202 return (EINVAL);
203 }
204
205 gve_parse_device_option(priv, descriptor, dev_opt,
206 dev_op_gqi_qpl,
207 dev_op_dqo_rda,
208 dev_op_dqo_qpl,
209 dev_op_modify_ring,
210 dev_op_jumbo_frames);
211 dev_opt = (void *)((char *)(dev_opt + 1) + be16toh(dev_opt->option_length));
212 }
213
214 return (0);
215 }
216
217 static int gve_adminq_issue_cmd(struct gve_priv *priv,
218 struct gve_adminq_command *cmd);
219 static int gve_adminq_kick_and_wait(struct gve_priv *priv);
220 static int gve_adminq_execute_cmd(struct gve_priv *priv,
221 struct gve_adminq_command *cmd);
222
223 static int
gve_adminq_destroy_tx_queue(struct gve_priv * priv,uint32_t id)224 gve_adminq_destroy_tx_queue(struct gve_priv *priv, uint32_t id)
225 {
226 struct gve_adminq_command cmd = (struct gve_adminq_command){};
227
228 cmd.opcode = htobe32(GVE_ADMINQ_DESTROY_TX_QUEUE);
229 cmd.destroy_tx_queue.queue_id = htobe32(id);
230
231 return (gve_adminq_issue_cmd(priv, &cmd));
232 }
233
234 static int
gve_adminq_destroy_rx_queue(struct gve_priv * priv,uint32_t id)235 gve_adminq_destroy_rx_queue(struct gve_priv *priv, uint32_t id)
236 {
237 struct gve_adminq_command cmd = (struct gve_adminq_command){};
238
239 cmd.opcode = htobe32(GVE_ADMINQ_DESTROY_RX_QUEUE);
240 cmd.destroy_rx_queue.queue_id = htobe32(id);
241
242 return (gve_adminq_issue_cmd(priv, &cmd));
243 }
244
245 int
gve_adminq_destroy_rx_queues(struct gve_priv * priv,uint32_t num_queues)246 gve_adminq_destroy_rx_queues(struct gve_priv *priv, uint32_t num_queues)
247 {
248 int err;
249 int i;
250
251 for (i = 0; i < num_queues; i++) {
252 err = gve_adminq_destroy_rx_queue(priv, i);
253 if (err != 0) {
254 device_printf(priv->dev, "Failed to issue destroy rxq %d, err: %d\n",
255 i, err);
256 return (err);
257 }
258 }
259
260 err = gve_adminq_kick_and_wait(priv);
261 if (err != 0) {
262 device_printf(priv->dev, "Failed to batch destroy rx queues, err: %d\n",
263 err);
264 return (err);
265 }
266
267 device_printf(priv->dev, "Destroyed %d rx queues\n", num_queues);
268 return (0);
269 }
270
271 int
gve_adminq_destroy_tx_queues(struct gve_priv * priv,uint32_t num_queues)272 gve_adminq_destroy_tx_queues(struct gve_priv *priv, uint32_t num_queues)
273 {
274 int err;
275 int i;
276
277 for (i = 0; i < num_queues; i++) {
278 err = gve_adminq_destroy_tx_queue(priv, i);
279 if (err != 0) {
280 device_printf(priv->dev, "Failed to issue destroy txq %d, err: %d\n",
281 i, err);
282 return (err);
283 }
284 }
285
286 err = gve_adminq_kick_and_wait(priv);
287 if (err != 0) {
288 device_printf(priv->dev, "Failed to batch destroy tx queues, err: %d\n",
289 err);
290 return (err);
291 }
292
293 device_printf(priv->dev, "Destroyed %d tx queues\n", num_queues);
294 return (0);
295 }
296
297 static int
gve_adminq_create_rx_queue(struct gve_priv * priv,uint32_t queue_index)298 gve_adminq_create_rx_queue(struct gve_priv *priv, uint32_t queue_index)
299 {
300 struct gve_adminq_command cmd = (struct gve_adminq_command){};
301 struct gve_rx_ring *rx = &priv->rx[queue_index];
302 struct gve_dma_handle *qres_dma = &rx->com.q_resources_mem;
303
304 bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);
305
306 cmd.opcode = htobe32(GVE_ADMINQ_CREATE_RX_QUEUE);
307 cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
308 .queue_id = htobe32(queue_index),
309 .ntfy_id = htobe32(rx->com.ntfy_id),
310 .queue_resources_addr = htobe64(qres_dma->bus_addr),
311 .rx_ring_size = htobe16(priv->rx_desc_cnt),
312 };
313
314 if (gve_is_gqi(priv)) {
315 cmd.create_rx_queue.rx_desc_ring_addr =
316 htobe64(rx->desc_ring_mem.bus_addr);
317 cmd.create_rx_queue.rx_data_ring_addr =
318 htobe64(rx->data_ring_mem.bus_addr);
319 cmd.create_rx_queue.index =
320 htobe32(queue_index);
321 cmd.create_rx_queue.queue_page_list_id =
322 htobe32((rx->com.qpl)->id);
323 cmd.create_rx_queue.packet_buffer_size =
324 htobe16(GVE_DEFAULT_RX_BUFFER_SIZE);
325 } else {
326 cmd.create_rx_queue.queue_page_list_id =
327 htobe32(GVE_RAW_ADDRESSING_QPL_ID);
328 cmd.create_rx_queue.rx_desc_ring_addr =
329 htobe64(rx->dqo.compl_ring_mem.bus_addr);
330 cmd.create_rx_queue.rx_data_ring_addr =
331 htobe64(rx->desc_ring_mem.bus_addr);
332 cmd.create_rx_queue.rx_buff_ring_size =
333 htobe16(priv->rx_desc_cnt);
334 cmd.create_rx_queue.enable_rsc =
335 !!((if_getcapenable(priv->ifp) & IFCAP_LRO) &&
336 !gve_disable_hw_lro);
337 cmd.create_rx_queue.packet_buffer_size =
338 htobe16(priv->rx_buf_size_dqo);
339 }
340
341 return (gve_adminq_issue_cmd(priv, &cmd));
342 }
343
344 int
gve_adminq_create_rx_queues(struct gve_priv * priv,uint32_t num_queues)345 gve_adminq_create_rx_queues(struct gve_priv *priv, uint32_t num_queues)
346 {
347 int err;
348 int i;
349
350 for (i = 0; i < num_queues; i++) {
351 err = gve_adminq_create_rx_queue(priv, i);
352 if (err != 0) {
353 device_printf(priv->dev, "Failed to issue create rxq %d, err: %d\n",
354 i, err);
355 goto abort;
356 }
357 }
358
359 err = gve_adminq_kick_and_wait(priv);
360 if (err != 0) {
361 device_printf(priv->dev, "Failed to batch create rx queues, err: %d\n",
362 err);
363 goto abort;
364 }
365
366 if (bootverbose)
367 device_printf(priv->dev, "Created %d rx queues\n", num_queues);
368 return (0);
369
370 abort:
371 gve_adminq_destroy_rx_queues(priv, i);
372 return (err);
373 }
374
375 static int
gve_adminq_create_tx_queue(struct gve_priv * priv,uint32_t queue_index)376 gve_adminq_create_tx_queue(struct gve_priv *priv, uint32_t queue_index)
377 {
378 struct gve_adminq_command cmd = (struct gve_adminq_command){};
379 struct gve_tx_ring *tx = &priv->tx[queue_index];
380 struct gve_dma_handle *qres_dma = &tx->com.q_resources_mem;
381
382 bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);
383
384 cmd.opcode = htobe32(GVE_ADMINQ_CREATE_TX_QUEUE);
385 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
386 .queue_id = htobe32(queue_index),
387 .queue_resources_addr = htobe64(qres_dma->bus_addr),
388 .tx_ring_addr = htobe64(tx->desc_ring_mem.bus_addr),
389 .ntfy_id = htobe32(tx->com.ntfy_id),
390 .tx_ring_size = htobe16(priv->tx_desc_cnt),
391 };
392
393 if (gve_is_gqi(priv)) {
394 cmd.create_tx_queue.queue_page_list_id =
395 htobe32((tx->com.qpl)->id);
396 } else {
397 cmd.create_tx_queue.queue_page_list_id =
398 htobe32(GVE_RAW_ADDRESSING_QPL_ID);
399 cmd.create_tx_queue.tx_comp_ring_addr =
400 htobe64(tx->dqo.compl_ring_mem.bus_addr);
401 cmd.create_tx_queue.tx_comp_ring_size =
402 htobe16(priv->tx_desc_cnt);
403 }
404 return (gve_adminq_issue_cmd(priv, &cmd));
405 }
406
407 int
gve_adminq_create_tx_queues(struct gve_priv * priv,uint32_t num_queues)408 gve_adminq_create_tx_queues(struct gve_priv *priv, uint32_t num_queues)
409 {
410 int err;
411 int i;
412
413 for (i = 0; i < num_queues; i++) {
414 err = gve_adminq_create_tx_queue(priv, i);
415 if (err != 0) {
416 device_printf(priv->dev, "Failed to issue create txq %d, err: %d\n",
417 i, err);
418 goto abort;
419 }
420 }
421
422 err = gve_adminq_kick_and_wait(priv);
423 if (err != 0) {
424 device_printf(priv->dev, "Failed to batch create tx queues, err: %d\n",
425 err);
426 goto abort;
427 }
428
429 if (bootverbose)
430 device_printf(priv->dev, "Created %d tx queues\n", num_queues);
431 return (0);
432
433 abort:
434 gve_adminq_destroy_tx_queues(priv, i);
435 return (err);
436 }
437
438 int
gve_adminq_set_mtu(struct gve_priv * priv,uint32_t mtu)439 gve_adminq_set_mtu(struct gve_priv *priv, uint32_t mtu) {
440 struct gve_adminq_command cmd = (struct gve_adminq_command){};
441
442 cmd.opcode = htobe32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
443 cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
444 .parameter_type = htobe32(GVE_SET_PARAM_MTU),
445 .parameter_value = htobe64(mtu),
446 };
447
448 return (gve_adminq_execute_cmd(priv, &cmd));
449 }
450
451 static void
gve_enable_supported_features(struct gve_priv * priv,uint32_t supported_features_mask,const struct gve_device_option_modify_ring * dev_op_modify_ring,const struct gve_device_option_jumbo_frames * dev_op_jumbo_frames)452 gve_enable_supported_features(struct gve_priv *priv,
453 uint32_t supported_features_mask,
454 const struct gve_device_option_modify_ring *dev_op_modify_ring,
455 const struct gve_device_option_jumbo_frames *dev_op_jumbo_frames)
456 {
457 if (dev_op_modify_ring &&
458 (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
459 if (bootverbose)
460 device_printf(priv->dev, "MODIFY RING device option enabled.\n");
461 priv->modify_ringsize_enabled = true;
462 priv->max_rx_desc_cnt = be16toh(dev_op_modify_ring->max_ring_size.rx);
463 priv->max_tx_desc_cnt = be16toh(dev_op_modify_ring->max_ring_size.tx);
464 }
465
466 if (dev_op_jumbo_frames &&
467 (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
468 if (bootverbose)
469 device_printf(priv->dev, "JUMBO FRAMES device option enabled: %u.\n",
470 be16toh(dev_op_jumbo_frames->max_mtu));
471 priv->max_mtu = be16toh(dev_op_jumbo_frames->max_mtu);
472 }
473 }
474
475 int
gve_adminq_describe_device(struct gve_priv * priv)476 gve_adminq_describe_device(struct gve_priv *priv)
477 {
478 struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
479 struct gve_device_descriptor *desc;
480 struct gve_dma_handle desc_mem;
481 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
482 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
483 struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL;
484 struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
485 struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
486 uint32_t supported_features_mask = 0;
487 int rc;
488 int i;
489
490 rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE, &desc_mem);
491 if (rc != 0) {
492 device_printf(priv->dev, "Failed to alloc DMA mem for DescribeDevice.\n");
493 return (rc);
494 }
495
496 desc = desc_mem.cpu_addr;
497
498 aq_cmd.opcode = htobe32(GVE_ADMINQ_DESCRIBE_DEVICE);
499 aq_cmd.describe_device.device_descriptor_addr = htobe64(
500 desc_mem.bus_addr);
501 aq_cmd.describe_device.device_descriptor_version = htobe32(
502 GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
503 aq_cmd.describe_device.available_length = htobe32(ADMINQ_SIZE);
504
505 bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_PREWRITE);
506
507 rc = gve_adminq_execute_cmd(priv, &aq_cmd);
508 if (rc != 0)
509 goto free_device_descriptor;
510
511 bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_POSTREAD);
512
513 /* Default min in case device options don't have min values */
514 priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
515 priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
516
517 rc = gve_process_device_options(priv, desc,
518 &dev_op_gqi_qpl,
519 &dev_op_dqo_rda,
520 &dev_op_dqo_qpl,
521 &dev_op_modify_ring,
522 &dev_op_jumbo_frames);
523 if (rc != 0)
524 goto free_device_descriptor;
525
526 if (dev_op_dqo_rda != NULL) {
527 snprintf(gve_queue_format, sizeof(gve_queue_format),
528 "%s", "DQO RDA");
529 priv->queue_format = GVE_DQO_RDA_FORMAT;
530 supported_features_mask = be32toh(
531 dev_op_dqo_rda->supported_features_mask);
532 if (bootverbose)
533 device_printf(priv->dev,
534 "Driver is running with DQO RDA queue format.\n");
535 } else if (dev_op_dqo_qpl != NULL) {
536 snprintf(gve_queue_format, sizeof(gve_queue_format),
537 "%s", "DQO QPL");
538 priv->queue_format = GVE_DQO_QPL_FORMAT;
539 supported_features_mask = be32toh(
540 dev_op_dqo_qpl->supported_features_mask);
541 if (bootverbose)
542 device_printf(priv->dev,
543 "Driver is running with DQO QPL queue format.\n");
544 } else if (dev_op_gqi_qpl != NULL) {
545 snprintf(gve_queue_format, sizeof(gve_queue_format),
546 "%s", "GQI QPL");
547 priv->queue_format = GVE_GQI_QPL_FORMAT;
548 supported_features_mask = be32toh(
549 dev_op_gqi_qpl->supported_features_mask);
550 if (bootverbose)
551 device_printf(priv->dev,
552 "Driver is running with GQI QPL queue format.\n");
553 } else {
554 device_printf(priv->dev, "No compatible queue formats\n");
555 rc = EINVAL;
556 goto free_device_descriptor;
557 }
558
559 priv->num_event_counters = be16toh(desc->counters);
560 priv->default_num_queues = be16toh(desc->default_num_queues);
561 priv->tx_desc_cnt = be16toh(desc->tx_queue_entries);
562 priv->rx_desc_cnt = be16toh(desc->rx_queue_entries);
563 priv->rx_pages_per_qpl = be16toh(desc->rx_pages_per_qpl);
564 priv->max_registered_pages = be64toh(desc->max_registered_pages);
565 priv->max_mtu = be16toh(desc->mtu);
566 priv->default_num_queues = be16toh(desc->default_num_queues);
567 priv->supported_features = supported_features_mask;
568
569 /* Default max to current in case modify ring size option is disabled */
570 priv->max_rx_desc_cnt = priv->rx_desc_cnt;
571 priv->max_tx_desc_cnt = priv->tx_desc_cnt;
572
573 gve_enable_supported_features(priv, supported_features_mask,
574 dev_op_modify_ring, dev_op_jumbo_frames);
575
576 for (i = 0; i < ETHER_ADDR_LEN; i++)
577 priv->mac[i] = desc->mac[i];
578
579 free_device_descriptor:
580 gve_dma_free_coherent(&desc_mem);
581
582 return (rc);
583 }
584
585 int
gve_adminq_register_page_list(struct gve_priv * priv,struct gve_queue_page_list * qpl)586 gve_adminq_register_page_list(struct gve_priv *priv,
587 struct gve_queue_page_list *qpl)
588 {
589 struct gve_adminq_command cmd = (struct gve_adminq_command){};
590 uint32_t num_entries = qpl->num_pages;
591 uint32_t size = num_entries * sizeof(qpl->dmas[0].bus_addr);
592 __be64 *page_list;
593 struct gve_dma_handle dma;
594 int err;
595 int i;
596
597 err = gve_dma_alloc_coherent(priv, size, PAGE_SIZE, &dma);
598 if (err != 0)
599 return (ENOMEM);
600
601 page_list = dma.cpu_addr;
602
603 for (i = 0; i < num_entries; i++)
604 page_list[i] = htobe64(qpl->dmas[i].bus_addr);
605
606 bus_dmamap_sync(dma.tag, dma.map, BUS_DMASYNC_PREWRITE);
607
608 cmd.opcode = htobe32(GVE_ADMINQ_REGISTER_PAGE_LIST);
609 cmd.reg_page_list = (struct gve_adminq_register_page_list) {
610 .page_list_id = htobe32(qpl->id),
611 .num_pages = htobe32(num_entries),
612 .page_address_list_addr = htobe64(dma.bus_addr),
613 .page_size = htobe64(PAGE_SIZE),
614 };
615
616 err = gve_adminq_execute_cmd(priv, &cmd);
617 gve_dma_free_coherent(&dma);
618 return (err);
619 }
620
621 int
gve_adminq_unregister_page_list(struct gve_priv * priv,uint32_t page_list_id)622 gve_adminq_unregister_page_list(struct gve_priv *priv, uint32_t page_list_id)
623 {
624 struct gve_adminq_command cmd = (struct gve_adminq_command){};
625
626 cmd.opcode = htobe32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
627 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
628 .page_list_id = htobe32(page_list_id),
629 };
630
631 return (gve_adminq_execute_cmd(priv, &cmd));
632 }
633
634 #define GVE_NTFY_BLK_BASE_MSIX_IDX 0
635 int
gve_adminq_configure_device_resources(struct gve_priv * priv)636 gve_adminq_configure_device_resources(struct gve_priv *priv)
637 {
638 struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
639
640 bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,
641 BUS_DMASYNC_PREREAD);
642 bus_dmamap_sync(priv->counter_array_mem.tag,
643 priv->counter_array_mem.map, BUS_DMASYNC_PREREAD);
644
645 aq_cmd.opcode = htobe32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
646 aq_cmd.configure_device_resources =
647 (struct gve_adminq_configure_device_resources) {
648 .counter_array = htobe64(priv->counter_array_mem.bus_addr),
649 .irq_db_addr = htobe64(priv->irqs_db_mem.bus_addr),
650 .num_counters = htobe32(priv->num_event_counters),
651 .num_irq_dbs = htobe32(priv->num_queues),
652 .irq_db_stride = htobe32(sizeof(struct gve_irq_db)),
653 .ntfy_blk_msix_base_idx = htobe32(GVE_NTFY_BLK_BASE_MSIX_IDX),
654 .queue_format = priv->queue_format,
655 };
656
657 return (gve_adminq_execute_cmd(priv, &aq_cmd));
658 }
659
660 int
gve_adminq_deconfigure_device_resources(struct gve_priv * priv)661 gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
662 {
663 struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
664
665 aq_cmd.opcode = htobe32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
666 return (gve_adminq_execute_cmd(priv, &aq_cmd));
667 }
668
669 int
gve_adminq_verify_driver_compatibility(struct gve_priv * priv,uint64_t driver_info_len,vm_paddr_t driver_info_addr)670 gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
671 uint64_t driver_info_len,
672 vm_paddr_t driver_info_addr)
673 {
674 struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
675
676 aq_cmd.opcode = htobe32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
677 aq_cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
678 .driver_info_len = htobe64(driver_info_len),
679 .driver_info_addr = htobe64(driver_info_addr),
680 };
681
682 return (gve_adminq_execute_cmd(priv, &aq_cmd));
683 }
684
685 int
gve_adminq_get_ptype_map_dqo(struct gve_priv * priv,struct gve_ptype_lut * ptype_lut_dqo)686 gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
687 struct gve_ptype_lut *ptype_lut_dqo)
688 {
689 struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
690 struct gve_ptype_map *ptype_map;
691 struct gve_dma_handle dma;
692 int err = 0;
693 int i;
694
695 err = gve_dma_alloc_coherent(priv, sizeof(*ptype_map), PAGE_SIZE, &dma);
696 if (err)
697 return (err);
698 ptype_map = dma.cpu_addr;
699
700 aq_cmd.opcode = htobe32(GVE_ADMINQ_GET_PTYPE_MAP);
701 aq_cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
702 .ptype_map_len = htobe64(sizeof(*ptype_map)),
703 .ptype_map_addr = htobe64(dma.bus_addr),
704 };
705
706 err = gve_adminq_execute_cmd(priv, &aq_cmd);
707 if (err)
708 goto err;
709
710 /* Populate ptype_lut_dqo. */
711 for (i = 0; i < GVE_NUM_PTYPES; i++) {
712 ptype_lut_dqo->ptypes[i].l3_type = ptype_map->ptypes[i].l3_type;
713 ptype_lut_dqo->ptypes[i].l4_type = ptype_map->ptypes[i].l4_type;
714 }
715 err:
716 gve_dma_free_coherent(&dma);
717 return (err);
718 }
719
720 int
gve_adminq_alloc(struct gve_priv * priv)721 gve_adminq_alloc(struct gve_priv *priv)
722 {
723 int rc;
724
725 if (gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))
726 return (0);
727
728 if (priv->aq_mem.cpu_addr == NULL) {
729 rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE,
730 &priv->aq_mem);
731 if (rc != 0) {
732 device_printf(priv->dev, "Failed to allocate admin queue mem\n");
733 return (rc);
734 }
735 }
736
737 priv->adminq = priv->aq_mem.cpu_addr;
738 priv->adminq_bus_addr = priv->aq_mem.bus_addr;
739
740 if (priv->adminq == NULL)
741 return (ENOMEM);
742
743 priv->adminq_mask = ADMINQ_SLOTS - 1;
744 priv->adminq_prod_cnt = 0;
745 priv->adminq_cmd_fail = 0;
746 priv->adminq_timeouts = 0;
747 priv->adminq_describe_device_cnt = 0;
748 priv->adminq_cfg_device_resources_cnt = 0;
749 priv->adminq_register_page_list_cnt = 0;
750 priv->adminq_unregister_page_list_cnt = 0;
751 priv->adminq_create_tx_queue_cnt = 0;
752 priv->adminq_create_rx_queue_cnt = 0;
753 priv->adminq_destroy_tx_queue_cnt = 0;
754 priv->adminq_destroy_rx_queue_cnt = 0;
755 priv->adminq_dcfg_device_resources_cnt = 0;
756 priv->adminq_set_driver_parameter_cnt = 0;
757 priv->adminq_get_ptype_map_cnt = 0;
758
759 gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR,
760 priv->adminq_bus_addr / ADMINQ_SIZE);
761
762 gve_set_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);
763 return (0);
764 }
765
766 void
gve_release_adminq(struct gve_priv * priv)767 gve_release_adminq(struct gve_priv *priv)
768 {
769 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))
770 return;
771
772 gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR, 0);
773 while (gve_reg_bar_read_4(priv, GVE_REG_ADMINQ_ADDR)) {
774 device_printf(priv->dev, "Waiting until admin queue is released.\n");
775 pause("gve release adminq", GVE_ADMINQ_SLEEP_LEN_MS);
776 }
777
778 gve_dma_free_coherent(&priv->aq_mem);
779 priv->aq_mem = (struct gve_dma_handle){};
780 priv->adminq = 0;
781 priv->adminq_bus_addr = 0;
782
783 gve_clear_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);
784
785 if (bootverbose)
786 device_printf(priv->dev, "Admin queue released\n");
787 }
788
789 static int
gve_adminq_parse_err(struct gve_priv * priv,uint32_t opcode,uint32_t status)790 gve_adminq_parse_err(struct gve_priv *priv, uint32_t opcode, uint32_t status)
791 {
792 if (status != GVE_ADMINQ_COMMAND_PASSED &&
793 status != GVE_ADMINQ_COMMAND_UNSET) {
794 device_printf(priv->dev, "AQ command(%u): failed with status %d\n", opcode, status);
795 priv->adminq_cmd_fail++;
796 }
797 switch (status) {
798 case GVE_ADMINQ_COMMAND_PASSED:
799 return (0);
800
801 case GVE_ADMINQ_COMMAND_UNSET:
802 device_printf(priv->dev,
803 "AQ command(%u): err and status both unset, this should not be possible.\n",
804 opcode);
805 return (EINVAL);
806
807 case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
808 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
809 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
810 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
811 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
812 return (EAGAIN);
813
814 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
815 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
816 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
817 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
818 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
819 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
820 return (EINVAL);
821
822 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
823 return (ETIMEDOUT);
824
825 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
826 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
827 return (EACCES);
828
829 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
830 return (ENOMEM);
831
832 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
833 return (EOPNOTSUPP);
834
835 default:
836 device_printf(priv->dev, "AQ command(%u): unknown status code %d\n",
837 opcode, status);
838 return (EINVAL);
839 }
840 }
841
842 static void
gve_adminq_kick_cmd(struct gve_priv * priv,uint32_t prod_cnt)843 gve_adminq_kick_cmd(struct gve_priv *priv, uint32_t prod_cnt)
844 {
845 gve_reg_bar_write_4(priv, ADMINQ_DOORBELL, prod_cnt);
846
847 }
848
849 static bool
gve_adminq_wait_for_cmd(struct gve_priv * priv,uint32_t prod_cnt)850 gve_adminq_wait_for_cmd(struct gve_priv *priv, uint32_t prod_cnt)
851 {
852 int i;
853
854 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
855 if (gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER) == prod_cnt)
856 return (true);
857 pause("gve adminq cmd", GVE_ADMINQ_SLEEP_LEN_MS);
858 }
859
860 return (false);
861 }
862
863 /*
864 * Flushes all AQ commands currently queued and waits for them to complete.
865 * If there are failures, it will return the first error.
866 */
867 static int
gve_adminq_kick_and_wait(struct gve_priv * priv)868 gve_adminq_kick_and_wait(struct gve_priv *priv)
869 {
870 struct gve_adminq_command *cmd;
871 uint32_t status, err;
872 uint32_t tail, head;
873 uint32_t opcode;
874 int i;
875
876 tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
877 head = priv->adminq_prod_cnt;
878
879 gve_adminq_kick_cmd(priv, head);
880 if (!gve_adminq_wait_for_cmd(priv, head)) {
881 device_printf(priv->dev, "AQ commands timed out, need to reset AQ\n");
882 priv->adminq_timeouts++;
883 return (ENOTRECOVERABLE);
884 }
885 bus_dmamap_sync(
886 priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_POSTREAD);
887
888 for (i = tail; i < head; i++) {
889 cmd = &priv->adminq[i & priv->adminq_mask];
890 status = be32toh(cmd->status);
891 opcode = be32toh(cmd->opcode);
892 err = gve_adminq_parse_err(priv, opcode, status);
893 if (err != 0)
894 return (err);
895 }
896
897 return (0);
898 }
899
900 /*
901 * This function is not threadsafe - the caller is responsible for any
902 * necessary locks.
903 */
904 static int
gve_adminq_issue_cmd(struct gve_priv * priv,struct gve_adminq_command * cmd_orig)905 gve_adminq_issue_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)
906 {
907 struct gve_adminq_command *cmd;
908 uint32_t opcode;
909 uint32_t tail;
910 int err;
911
912 tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
913
914 /* Check if next command will overflow the buffer. */
915 if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {
916 /* Flush existing commands to make room. */
917 err = gve_adminq_kick_and_wait(priv);
918 if (err != 0)
919 return (err);
920
921 /* Retry. */
922 tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
923 if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {
924 /*
925 * This should never happen. We just flushed the
926 * command queue so there should be enough space.
927 */
928 return (ENOMEM);
929 }
930 }
931
932 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
933 priv->adminq_prod_cnt++;
934
935 memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
936
937 bus_dmamap_sync(
938 priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_PREWRITE);
939
940 opcode = be32toh(cmd->opcode);
941
942 switch (opcode) {
943 case GVE_ADMINQ_DESCRIBE_DEVICE:
944 priv->adminq_describe_device_cnt++;
945 break;
946
947 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
948 priv->adminq_cfg_device_resources_cnt++;
949 break;
950
951 case GVE_ADMINQ_REGISTER_PAGE_LIST:
952 priv->adminq_register_page_list_cnt++;
953 break;
954
955 case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
956 priv->adminq_unregister_page_list_cnt++;
957 break;
958
959 case GVE_ADMINQ_CREATE_TX_QUEUE:
960 priv->adminq_create_tx_queue_cnt++;
961 break;
962
963 case GVE_ADMINQ_CREATE_RX_QUEUE:
964 priv->adminq_create_rx_queue_cnt++;
965 break;
966
967 case GVE_ADMINQ_DESTROY_TX_QUEUE:
968 priv->adminq_destroy_tx_queue_cnt++;
969 break;
970
971 case GVE_ADMINQ_DESTROY_RX_QUEUE:
972 priv->adminq_destroy_rx_queue_cnt++;
973 break;
974
975 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
976 priv->adminq_dcfg_device_resources_cnt++;
977 break;
978
979 case GVE_ADMINQ_SET_DRIVER_PARAMETER:
980 priv->adminq_set_driver_parameter_cnt++;
981 break;
982
983 case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
984 priv->adminq_verify_driver_compatibility_cnt++;
985 break;
986
987 case GVE_ADMINQ_GET_PTYPE_MAP:
988 priv->adminq_get_ptype_map_cnt++;
989 break;
990
991 default:
992 device_printf(priv->dev, "Unknown AQ command opcode %d\n", opcode);
993 }
994
995 return (0);
996 }
997
998 /*
999 * This function is not threadsafe - the caller is responsible for any
1000 * necessary locks.
1001 * The caller is also responsible for making sure there are no commands
1002 * waiting to be executed.
1003 */
1004 static int
gve_adminq_execute_cmd(struct gve_priv * priv,struct gve_adminq_command * cmd_orig)1005 gve_adminq_execute_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)
1006 {
1007 uint32_t tail, head;
1008 int err;
1009
1010 tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
1011 head = priv->adminq_prod_cnt;
1012
1013 if (tail != head)
1014 return (EINVAL);
1015 err = gve_adminq_issue_cmd(priv, cmd_orig);
1016 if (err != 0)
1017 return (err);
1018 return (gve_adminq_kick_and_wait(priv));
1019 }
1020