1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include <linux/etherdevice.h> 8 #include <linux/pci.h> 9 #include "gve.h" 10 #include "gve_adminq.h" 11 #include "gve_register.h" 12 13 #define GVE_MAX_ADMINQ_RELEASE_CHECK 500 14 #define GVE_ADMINQ_SLEEP_LEN 20 15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100 16 17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \ 18 "Expected: length=%d, feature_mask=%x.\n" \ 19 "Actual: length=%d, feature_mask=%x.\n" 20 21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n" 22 23 static 24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor, 25 struct gve_device_option *option) 26 { 27 void *option_end, *descriptor_end; 28 29 option_end = (void *)(option + 1) + be16_to_cpu(option->option_length); 30 descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length); 31 32 return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end; 33 } 34 35 static 36 void gve_parse_device_option(struct gve_priv *priv, 37 struct gve_device_descriptor *device_descriptor, 38 struct gve_device_option *option, 39 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 40 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 41 struct gve_device_option_dqo_rda **dev_op_dqo_rda) 42 { 43 u32 req_feat_mask = be32_to_cpu(option->required_features_mask); 44 u16 option_length = be16_to_cpu(option->option_length); 45 u16 option_id = be16_to_cpu(option->option_id); 46 47 /* If the length or feature mask doesn't match, continue without 48 * enabling the feature. 49 */ 50 switch (option_id) { 51 case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING: 52 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING || 53 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) { 54 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 55 "Raw Addressing", 56 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING, 57 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING, 58 option_length, req_feat_mask); 59 break; 60 } 61 62 dev_info(&priv->pdev->dev, 63 "Gqi raw addressing device option enabled.\n"); 64 priv->queue_format = GVE_GQI_RDA_FORMAT; 65 break; 66 case GVE_DEV_OPT_ID_GQI_RDA: 67 if (option_length < sizeof(**dev_op_gqi_rda) || 68 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) { 69 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 70 "GQI RDA", (int)sizeof(**dev_op_gqi_rda), 71 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA, 72 option_length, req_feat_mask); 73 break; 74 } 75 76 if (option_length > sizeof(**dev_op_gqi_rda)) { 77 dev_warn(&priv->pdev->dev, 78 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA"); 79 } 80 *dev_op_gqi_rda = (void *)(option + 1); 81 break; 82 case GVE_DEV_OPT_ID_GQI_QPL: 83 if (option_length < sizeof(**dev_op_gqi_qpl) || 84 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) { 85 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 86 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl), 87 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL, 88 option_length, req_feat_mask); 89 break; 90 } 91 92 if (option_length > sizeof(**dev_op_gqi_qpl)) { 93 dev_warn(&priv->pdev->dev, 94 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL"); 95 } 96 *dev_op_gqi_qpl = (void *)(option + 1); 97 break; 98 case GVE_DEV_OPT_ID_DQO_RDA: 99 if (option_length < sizeof(**dev_op_dqo_rda) || 100 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) { 101 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 102 "DQO RDA", (int)sizeof(**dev_op_dqo_rda), 103 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA, 104 option_length, req_feat_mask); 105 break; 106 } 107 108 if (option_length > sizeof(**dev_op_dqo_rda)) { 109 dev_warn(&priv->pdev->dev, 110 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA"); 111 } 112 *dev_op_dqo_rda = (void *)(option + 1); 113 break; 114 default: 115 /* If we don't recognize the option just continue 116 * without doing anything. 117 */ 118 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n", 119 option_id); 120 } 121 } 122 123 /* Process all device options for a given describe device call. */ 124 static int 125 gve_process_device_options(struct gve_priv *priv, 126 struct gve_device_descriptor *descriptor, 127 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 128 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 129 struct gve_device_option_dqo_rda **dev_op_dqo_rda) 130 { 131 const int num_options = be16_to_cpu(descriptor->num_device_options); 132 struct gve_device_option *dev_opt; 133 int i; 134 135 /* The options struct directly follows the device descriptor. */ 136 dev_opt = (void *)(descriptor + 1); 137 for (i = 0; i < num_options; i++) { 138 struct gve_device_option *next_opt; 139 140 next_opt = gve_get_next_option(descriptor, dev_opt); 141 if (!next_opt) { 142 dev_err(&priv->dev->dev, 143 "options exceed device_descriptor's total length.\n"); 144 return -EINVAL; 145 } 146 147 gve_parse_device_option(priv, descriptor, dev_opt, 148 dev_op_gqi_rda, dev_op_gqi_qpl, 149 dev_op_dqo_rda); 150 dev_opt = next_opt; 151 } 152 153 return 0; 154 } 155 156 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) 157 { 158 priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE, 159 &priv->adminq_bus_addr, GFP_KERNEL); 160 if (unlikely(!priv->adminq)) 161 return -ENOMEM; 162 163 priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1; 164 priv->adminq_prod_cnt = 0; 165 priv->adminq_cmd_fail = 0; 166 priv->adminq_timeouts = 0; 167 priv->adminq_describe_device_cnt = 0; 168 priv->adminq_cfg_device_resources_cnt = 0; 169 priv->adminq_register_page_list_cnt = 0; 170 priv->adminq_unregister_page_list_cnt = 0; 171 priv->adminq_create_tx_queue_cnt = 0; 172 priv->adminq_create_rx_queue_cnt = 0; 173 priv->adminq_destroy_tx_queue_cnt = 0; 174 priv->adminq_destroy_rx_queue_cnt = 0; 175 priv->adminq_dcfg_device_resources_cnt = 0; 176 priv->adminq_set_driver_parameter_cnt = 0; 177 priv->adminq_report_stats_cnt = 0; 178 priv->adminq_report_link_speed_cnt = 0; 179 priv->adminq_get_ptype_map_cnt = 0; 180 181 /* Setup Admin queue with the device */ 182 iowrite32be(priv->adminq_bus_addr / PAGE_SIZE, 183 &priv->reg_bar0->adminq_pfn); 184 185 gve_set_admin_queue_ok(priv); 186 return 0; 187 } 188 189 void gve_adminq_release(struct gve_priv *priv) 190 { 191 int i = 0; 192 193 /* Tell the device the adminq is leaving */ 194 iowrite32be(0x0, &priv->reg_bar0->adminq_pfn); 195 while (ioread32be(&priv->reg_bar0->adminq_pfn)) { 196 /* If this is reached the device is unrecoverable and still 197 * holding memory. Continue looping to avoid memory corruption, 198 * but WARN so it is visible what is going on. 199 */ 200 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK) 201 WARN(1, "Unrecoverable platform error!"); 202 i++; 203 msleep(GVE_ADMINQ_SLEEP_LEN); 204 } 205 gve_clear_device_rings_ok(priv); 206 gve_clear_device_resources_ok(priv); 207 gve_clear_admin_queue_ok(priv); 208 } 209 210 void gve_adminq_free(struct device *dev, struct gve_priv *priv) 211 { 212 if (!gve_get_admin_queue_ok(priv)) 213 return; 214 gve_adminq_release(priv); 215 dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr); 216 gve_clear_admin_queue_ok(priv); 217 } 218 219 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt) 220 { 221 iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell); 222 } 223 224 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt) 225 { 226 int i; 227 228 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) { 229 if (ioread32be(&priv->reg_bar0->adminq_event_counter) 230 == prod_cnt) 231 return true; 232 msleep(GVE_ADMINQ_SLEEP_LEN); 233 } 234 235 return false; 236 } 237 238 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status) 239 { 240 if (status != GVE_ADMINQ_COMMAND_PASSED && 241 status != GVE_ADMINQ_COMMAND_UNSET) { 242 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status); 243 priv->adminq_cmd_fail++; 244 } 245 switch (status) { 246 case GVE_ADMINQ_COMMAND_PASSED: 247 return 0; 248 case GVE_ADMINQ_COMMAND_UNSET: 249 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n"); 250 return -EINVAL; 251 case GVE_ADMINQ_COMMAND_ERROR_ABORTED: 252 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED: 253 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS: 254 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION: 255 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE: 256 return -EAGAIN; 257 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS: 258 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR: 259 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT: 260 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND: 261 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE: 262 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR: 263 return -EINVAL; 264 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED: 265 return -ETIME; 266 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED: 267 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED: 268 return -EACCES; 269 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED: 270 return -ENOMEM; 271 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED: 272 return -ENOTSUPP; 273 default: 274 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status); 275 return -EINVAL; 276 } 277 } 278 279 /* Flushes all AQ commands currently queued and waits for them to complete. 280 * If there are failures, it will return the first error. 281 */ 282 static int gve_adminq_kick_and_wait(struct gve_priv *priv) 283 { 284 u32 tail, head; 285 int i; 286 287 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 288 head = priv->adminq_prod_cnt; 289 290 gve_adminq_kick_cmd(priv, head); 291 if (!gve_adminq_wait_for_cmd(priv, head)) { 292 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n"); 293 priv->adminq_timeouts++; 294 return -ENOTRECOVERABLE; 295 } 296 297 for (i = tail; i < head; i++) { 298 union gve_adminq_command *cmd; 299 u32 status, err; 300 301 cmd = &priv->adminq[i & priv->adminq_mask]; 302 status = be32_to_cpu(READ_ONCE(cmd->status)); 303 err = gve_adminq_parse_err(priv, status); 304 if (err) 305 // Return the first error if we failed. 306 return err; 307 } 308 309 return 0; 310 } 311 312 /* This function is not threadsafe - the caller is responsible for any 313 * necessary locks. 314 */ 315 static int gve_adminq_issue_cmd(struct gve_priv *priv, 316 union gve_adminq_command *cmd_orig) 317 { 318 union gve_adminq_command *cmd; 319 u32 opcode; 320 u32 tail; 321 322 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 323 324 // Check if next command will overflow the buffer. 325 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 326 (tail & priv->adminq_mask)) { 327 int err; 328 329 // Flush existing commands to make room. 330 err = gve_adminq_kick_and_wait(priv); 331 if (err) 332 return err; 333 334 // Retry. 335 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 336 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 337 (tail & priv->adminq_mask)) { 338 // This should never happen. We just flushed the 339 // command queue so there should be enough space. 340 return -ENOMEM; 341 } 342 } 343 344 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask]; 345 priv->adminq_prod_cnt++; 346 347 memcpy(cmd, cmd_orig, sizeof(*cmd_orig)); 348 opcode = be32_to_cpu(READ_ONCE(cmd->opcode)); 349 350 switch (opcode) { 351 case GVE_ADMINQ_DESCRIBE_DEVICE: 352 priv->adminq_describe_device_cnt++; 353 break; 354 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES: 355 priv->adminq_cfg_device_resources_cnt++; 356 break; 357 case GVE_ADMINQ_REGISTER_PAGE_LIST: 358 priv->adminq_register_page_list_cnt++; 359 break; 360 case GVE_ADMINQ_UNREGISTER_PAGE_LIST: 361 priv->adminq_unregister_page_list_cnt++; 362 break; 363 case GVE_ADMINQ_CREATE_TX_QUEUE: 364 priv->adminq_create_tx_queue_cnt++; 365 break; 366 case GVE_ADMINQ_CREATE_RX_QUEUE: 367 priv->adminq_create_rx_queue_cnt++; 368 break; 369 case GVE_ADMINQ_DESTROY_TX_QUEUE: 370 priv->adminq_destroy_tx_queue_cnt++; 371 break; 372 case GVE_ADMINQ_DESTROY_RX_QUEUE: 373 priv->adminq_destroy_rx_queue_cnt++; 374 break; 375 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES: 376 priv->adminq_dcfg_device_resources_cnt++; 377 break; 378 case GVE_ADMINQ_SET_DRIVER_PARAMETER: 379 priv->adminq_set_driver_parameter_cnt++; 380 break; 381 case GVE_ADMINQ_REPORT_STATS: 382 priv->adminq_report_stats_cnt++; 383 break; 384 case GVE_ADMINQ_REPORT_LINK_SPEED: 385 priv->adminq_report_link_speed_cnt++; 386 break; 387 case GVE_ADMINQ_GET_PTYPE_MAP: 388 priv->adminq_get_ptype_map_cnt++; 389 break; 390 default: 391 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); 392 } 393 394 return 0; 395 } 396 397 /* This function is not threadsafe - the caller is responsible for any 398 * necessary locks. 399 * The caller is also responsible for making sure there are no commands 400 * waiting to be executed. 401 */ 402 static int gve_adminq_execute_cmd(struct gve_priv *priv, 403 union gve_adminq_command *cmd_orig) 404 { 405 u32 tail, head; 406 int err; 407 408 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 409 head = priv->adminq_prod_cnt; 410 if (tail != head) 411 // This is not a valid path 412 return -EINVAL; 413 414 err = gve_adminq_issue_cmd(priv, cmd_orig); 415 if (err) 416 return err; 417 418 return gve_adminq_kick_and_wait(priv); 419 } 420 421 /* The device specifies that the management vector can either be the first irq 422 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to 423 * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then 424 * the management vector is first. 425 * 426 * gve arranges the msix vectors so that the management vector is last. 427 */ 428 #define GVE_NTFY_BLK_BASE_MSIX_IDX 0 429 int gve_adminq_configure_device_resources(struct gve_priv *priv, 430 dma_addr_t counter_array_bus_addr, 431 u32 num_counters, 432 dma_addr_t db_array_bus_addr, 433 u32 num_ntfy_blks) 434 { 435 union gve_adminq_command cmd; 436 437 memset(&cmd, 0, sizeof(cmd)); 438 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES); 439 cmd.configure_device_resources = 440 (struct gve_adminq_configure_device_resources) { 441 .counter_array = cpu_to_be64(counter_array_bus_addr), 442 .num_counters = cpu_to_be32(num_counters), 443 .irq_db_addr = cpu_to_be64(db_array_bus_addr), 444 .num_irq_dbs = cpu_to_be32(num_ntfy_blks), 445 .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])), 446 .ntfy_blk_msix_base_idx = 447 cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX), 448 .queue_format = priv->queue_format, 449 }; 450 451 return gve_adminq_execute_cmd(priv, &cmd); 452 } 453 454 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv) 455 { 456 union gve_adminq_command cmd; 457 458 memset(&cmd, 0, sizeof(cmd)); 459 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES); 460 461 return gve_adminq_execute_cmd(priv, &cmd); 462 } 463 464 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) 465 { 466 struct gve_tx_ring *tx = &priv->tx[queue_index]; 467 union gve_adminq_command cmd; 468 469 memset(&cmd, 0, sizeof(cmd)); 470 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE); 471 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) { 472 .queue_id = cpu_to_be32(queue_index), 473 .queue_resources_addr = 474 cpu_to_be64(tx->q_resources_bus), 475 .tx_ring_addr = cpu_to_be64(tx->bus), 476 .ntfy_id = cpu_to_be32(tx->ntfy_id), 477 }; 478 479 if (gve_is_gqi(priv)) { 480 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 481 GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; 482 483 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 484 } else { 485 cmd.create_tx_queue.tx_ring_size = 486 cpu_to_be16(priv->tx_desc_cnt); 487 cmd.create_tx_queue.tx_comp_ring_addr = 488 cpu_to_be64(tx->complq_bus_dqo); 489 cmd.create_tx_queue.tx_comp_ring_size = 490 cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries); 491 } 492 493 return gve_adminq_issue_cmd(priv, &cmd); 494 } 495 496 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues) 497 { 498 int err; 499 int i; 500 501 for (i = 0; i < num_queues; i++) { 502 err = gve_adminq_create_tx_queue(priv, i); 503 if (err) 504 return err; 505 } 506 507 return gve_adminq_kick_and_wait(priv); 508 } 509 510 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 511 { 512 struct gve_rx_ring *rx = &priv->rx[queue_index]; 513 union gve_adminq_command cmd; 514 515 memset(&cmd, 0, sizeof(cmd)); 516 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 517 cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) { 518 .queue_id = cpu_to_be32(queue_index), 519 .ntfy_id = cpu_to_be32(rx->ntfy_id), 520 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), 521 }; 522 523 if (gve_is_gqi(priv)) { 524 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 525 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; 526 527 cmd.create_rx_queue.rx_desc_ring_addr = 528 cpu_to_be64(rx->desc.bus), 529 cmd.create_rx_queue.rx_data_ring_addr = 530 cpu_to_be64(rx->data.data_bus), 531 cmd.create_rx_queue.index = cpu_to_be32(queue_index); 532 cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 533 } else { 534 cmd.create_rx_queue.rx_ring_size = 535 cpu_to_be16(priv->rx_desc_cnt); 536 cmd.create_rx_queue.rx_desc_ring_addr = 537 cpu_to_be64(rx->dqo.complq.bus); 538 cmd.create_rx_queue.rx_data_ring_addr = 539 cpu_to_be64(rx->dqo.bufq.bus); 540 cmd.create_rx_queue.packet_buffer_size = 541 cpu_to_be16(priv->data_buffer_size_dqo); 542 cmd.create_rx_queue.rx_buff_ring_size = 543 cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries); 544 cmd.create_rx_queue.enable_rsc = 545 !!(priv->dev->features & NETIF_F_LRO); 546 } 547 548 return gve_adminq_issue_cmd(priv, &cmd); 549 } 550 551 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) 552 { 553 int err; 554 int i; 555 556 for (i = 0; i < num_queues; i++) { 557 err = gve_adminq_create_rx_queue(priv, i); 558 if (err) 559 return err; 560 } 561 562 return gve_adminq_kick_and_wait(priv); 563 } 564 565 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index) 566 { 567 union gve_adminq_command cmd; 568 int err; 569 570 memset(&cmd, 0, sizeof(cmd)); 571 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE); 572 cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) { 573 .queue_id = cpu_to_be32(queue_index), 574 }; 575 576 err = gve_adminq_issue_cmd(priv, &cmd); 577 if (err) 578 return err; 579 580 return 0; 581 } 582 583 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues) 584 { 585 int err; 586 int i; 587 588 for (i = 0; i < num_queues; i++) { 589 err = gve_adminq_destroy_tx_queue(priv, i); 590 if (err) 591 return err; 592 } 593 594 return gve_adminq_kick_and_wait(priv); 595 } 596 597 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index) 598 { 599 union gve_adminq_command cmd; 600 int err; 601 602 memset(&cmd, 0, sizeof(cmd)); 603 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 604 cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 605 .queue_id = cpu_to_be32(queue_index), 606 }; 607 608 err = gve_adminq_issue_cmd(priv, &cmd); 609 if (err) 610 return err; 611 612 return 0; 613 } 614 615 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues) 616 { 617 int err; 618 int i; 619 620 for (i = 0; i < num_queues; i++) { 621 err = gve_adminq_destroy_rx_queue(priv, i); 622 if (err) 623 return err; 624 } 625 626 return gve_adminq_kick_and_wait(priv); 627 } 628 629 static int gve_set_desc_cnt(struct gve_priv *priv, 630 struct gve_device_descriptor *descriptor) 631 { 632 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 633 if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) { 634 dev_err(&priv->pdev->dev, "Tx desc count %d too low\n", 635 priv->tx_desc_cnt); 636 return -EINVAL; 637 } 638 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 639 if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0]) 640 < PAGE_SIZE) { 641 dev_err(&priv->pdev->dev, "Rx desc count %d too low\n", 642 priv->rx_desc_cnt); 643 return -EINVAL; 644 } 645 return 0; 646 } 647 648 static int 649 gve_set_desc_cnt_dqo(struct gve_priv *priv, 650 const struct gve_device_descriptor *descriptor, 651 const struct gve_device_option_dqo_rda *dev_op_dqo_rda) 652 { 653 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 654 priv->options_dqo_rda.tx_comp_ring_entries = 655 be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries); 656 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 657 priv->options_dqo_rda.rx_buff_ring_entries = 658 be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries); 659 660 return 0; 661 } 662 663 int gve_adminq_describe_device(struct gve_priv *priv) 664 { 665 struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; 666 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; 667 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; 668 struct gve_device_descriptor *descriptor; 669 union gve_adminq_command cmd; 670 dma_addr_t descriptor_bus; 671 int err = 0; 672 u8 *mac; 673 u16 mtu; 674 675 memset(&cmd, 0, sizeof(cmd)); 676 descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE, 677 &descriptor_bus, GFP_KERNEL); 678 if (!descriptor) 679 return -ENOMEM; 680 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE); 681 cmd.describe_device.device_descriptor_addr = 682 cpu_to_be64(descriptor_bus); 683 cmd.describe_device.device_descriptor_version = 684 cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION); 685 cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE); 686 687 err = gve_adminq_execute_cmd(priv, &cmd); 688 if (err) 689 goto free_device_descriptor; 690 691 err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, 692 &dev_op_gqi_qpl, &dev_op_dqo_rda); 693 if (err) 694 goto free_device_descriptor; 695 696 /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format 697 * is not set to GqiRda, choose the queue format in a priority order: 698 * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default. 699 */ 700 if (priv->queue_format == GVE_GQI_RDA_FORMAT) { 701 dev_info(&priv->pdev->dev, 702 "Driver is running with GQI RDA queue format.\n"); 703 } else if (dev_op_dqo_rda) { 704 priv->queue_format = GVE_DQO_RDA_FORMAT; 705 dev_info(&priv->pdev->dev, 706 "Driver is running with DQO RDA queue format.\n"); 707 } else if (dev_op_gqi_rda) { 708 priv->queue_format = GVE_GQI_RDA_FORMAT; 709 dev_info(&priv->pdev->dev, 710 "Driver is running with GQI RDA queue format.\n"); 711 } else { 712 priv->queue_format = GVE_GQI_QPL_FORMAT; 713 dev_info(&priv->pdev->dev, 714 "Driver is running with GQI QPL queue format.\n"); 715 } 716 if (gve_is_gqi(priv)) { 717 err = gve_set_desc_cnt(priv, descriptor); 718 } else { 719 /* DQO supports LRO. */ 720 priv->dev->hw_features |= NETIF_F_LRO; 721 err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda); 722 } 723 if (err) 724 goto free_device_descriptor; 725 726 priv->max_registered_pages = 727 be64_to_cpu(descriptor->max_registered_pages); 728 mtu = be16_to_cpu(descriptor->mtu); 729 if (mtu < ETH_MIN_MTU) { 730 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 731 err = -EINVAL; 732 goto free_device_descriptor; 733 } 734 priv->dev->max_mtu = mtu; 735 priv->num_event_counters = be16_to_cpu(descriptor->counters); 736 ether_addr_copy(priv->dev->dev_addr, descriptor->mac); 737 mac = descriptor->mac; 738 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 739 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); 740 priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl); 741 742 if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) { 743 dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n", 744 priv->rx_data_slot_cnt); 745 priv->rx_desc_cnt = priv->rx_data_slot_cnt; 746 } 747 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); 748 749 free_device_descriptor: 750 dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor, 751 descriptor_bus); 752 return err; 753 } 754 755 int gve_adminq_register_page_list(struct gve_priv *priv, 756 struct gve_queue_page_list *qpl) 757 { 758 struct device *hdev = &priv->pdev->dev; 759 u32 num_entries = qpl->num_entries; 760 u32 size = num_entries * sizeof(qpl->page_buses[0]); 761 union gve_adminq_command cmd; 762 dma_addr_t page_list_bus; 763 __be64 *page_list; 764 int err; 765 int i; 766 767 memset(&cmd, 0, sizeof(cmd)); 768 page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL); 769 if (!page_list) 770 return -ENOMEM; 771 772 for (i = 0; i < num_entries; i++) 773 page_list[i] = cpu_to_be64(qpl->page_buses[i]); 774 775 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST); 776 cmd.reg_page_list = (struct gve_adminq_register_page_list) { 777 .page_list_id = cpu_to_be32(qpl->id), 778 .num_pages = cpu_to_be32(num_entries), 779 .page_address_list_addr = cpu_to_be64(page_list_bus), 780 }; 781 782 err = gve_adminq_execute_cmd(priv, &cmd); 783 dma_free_coherent(hdev, size, page_list, page_list_bus); 784 return err; 785 } 786 787 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id) 788 { 789 union gve_adminq_command cmd; 790 791 memset(&cmd, 0, sizeof(cmd)); 792 cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST); 793 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) { 794 .page_list_id = cpu_to_be32(page_list_id), 795 }; 796 797 return gve_adminq_execute_cmd(priv, &cmd); 798 } 799 800 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu) 801 { 802 union gve_adminq_command cmd; 803 804 memset(&cmd, 0, sizeof(cmd)); 805 cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER); 806 cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) { 807 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU), 808 .parameter_value = cpu_to_be64(mtu), 809 }; 810 811 return gve_adminq_execute_cmd(priv, &cmd); 812 } 813 814 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, 815 dma_addr_t stats_report_addr, u64 interval) 816 { 817 union gve_adminq_command cmd; 818 819 memset(&cmd, 0, sizeof(cmd)); 820 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS); 821 cmd.report_stats = (struct gve_adminq_report_stats) { 822 .stats_report_len = cpu_to_be64(stats_report_len), 823 .stats_report_addr = cpu_to_be64(stats_report_addr), 824 .interval = cpu_to_be64(interval), 825 }; 826 827 return gve_adminq_execute_cmd(priv, &cmd); 828 } 829 830 int gve_adminq_report_link_speed(struct gve_priv *priv) 831 { 832 union gve_adminq_command gvnic_cmd; 833 dma_addr_t link_speed_region_bus; 834 __be64 *link_speed_region; 835 int err; 836 837 link_speed_region = 838 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region), 839 &link_speed_region_bus, GFP_KERNEL); 840 841 if (!link_speed_region) 842 return -ENOMEM; 843 844 memset(&gvnic_cmd, 0, sizeof(gvnic_cmd)); 845 gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED); 846 gvnic_cmd.report_link_speed.link_speed_address = 847 cpu_to_be64(link_speed_region_bus); 848 849 err = gve_adminq_execute_cmd(priv, &gvnic_cmd); 850 851 priv->link_speed = be64_to_cpu(*link_speed_region); 852 dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region, 853 link_speed_region_bus); 854 return err; 855 } 856 857 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, 858 struct gve_ptype_lut *ptype_lut) 859 { 860 struct gve_ptype_map *ptype_map; 861 union gve_adminq_command cmd; 862 dma_addr_t ptype_map_bus; 863 int err = 0; 864 int i; 865 866 memset(&cmd, 0, sizeof(cmd)); 867 ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map), 868 &ptype_map_bus, GFP_KERNEL); 869 if (!ptype_map) 870 return -ENOMEM; 871 872 cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP); 873 cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) { 874 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)), 875 .ptype_map_addr = cpu_to_be64(ptype_map_bus), 876 }; 877 878 err = gve_adminq_execute_cmd(priv, &cmd); 879 if (err) 880 goto err; 881 882 /* Populate ptype_lut. */ 883 for (i = 0; i < GVE_NUM_PTYPES; i++) { 884 ptype_lut->ptypes[i].l3_type = 885 ptype_map->ptypes[i].l3_type; 886 ptype_lut->ptypes[i].l4_type = 887 ptype_map->ptypes[i].l4_type; 888 } 889 err: 890 dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map, 891 ptype_map_bus); 892 return err; 893 } 894