1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include <linux/etherdevice.h> 8 #include <linux/pci.h> 9 #include "gve.h" 10 #include "gve_adminq.h" 11 #include "gve_register.h" 12 13 #define GVE_MAX_ADMINQ_RELEASE_CHECK 500 14 #define GVE_ADMINQ_SLEEP_LEN 20 15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100 16 17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \ 18 "Expected: length=%d, feature_mask=%x.\n" \ 19 "Actual: length=%d, feature_mask=%x.\n" 20 21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n" 22 23 static 24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor, 25 struct gve_device_option *option) 26 { 27 void *option_end, *descriptor_end; 28 29 option_end = (void *)(option + 1) + be16_to_cpu(option->option_length); 30 descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length); 31 32 return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end; 33 } 34 35 #define GVE_DEVICE_OPTION_NO_MIN_RING_SIZE 8 36 37 static 38 void gve_parse_device_option(struct gve_priv *priv, 39 struct gve_device_descriptor *device_descriptor, 40 struct gve_device_option *option, 41 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 42 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 43 struct gve_device_option_dqo_rda **dev_op_dqo_rda, 44 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames, 45 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl, 46 struct gve_device_option_buffer_sizes **dev_op_buffer_sizes, 47 struct gve_device_option_flow_steering **dev_op_flow_steering, 48 struct gve_device_option_rss_config **dev_op_rss_config, 49 struct gve_device_option_nic_timestamp **dev_op_nic_timestamp, 50 struct gve_device_option_modify_ring **dev_op_modify_ring) 51 { 52 u32 req_feat_mask = be32_to_cpu(option->required_features_mask); 53 u16 option_length = be16_to_cpu(option->option_length); 54 u16 option_id = be16_to_cpu(option->option_id); 55 56 /* If the length or feature mask doesn't match, continue without 57 * enabling the feature. 58 */ 59 switch (option_id) { 60 case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING: 61 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING || 62 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) { 63 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 64 "Raw Addressing", 65 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING, 66 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING, 67 option_length, req_feat_mask); 68 break; 69 } 70 71 dev_info(&priv->pdev->dev, 72 "Gqi raw addressing device option enabled.\n"); 73 priv->queue_format = GVE_GQI_RDA_FORMAT; 74 break; 75 case GVE_DEV_OPT_ID_GQI_RDA: 76 if (option_length < sizeof(**dev_op_gqi_rda) || 77 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) { 78 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 79 "GQI RDA", (int)sizeof(**dev_op_gqi_rda), 80 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA, 81 option_length, req_feat_mask); 82 break; 83 } 84 85 if (option_length > sizeof(**dev_op_gqi_rda)) { 86 dev_warn(&priv->pdev->dev, 87 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA"); 88 } 89 *dev_op_gqi_rda = (void *)(option + 1); 90 break; 91 case GVE_DEV_OPT_ID_GQI_QPL: 92 if (option_length < sizeof(**dev_op_gqi_qpl) || 93 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) { 94 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 95 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl), 96 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL, 97 option_length, req_feat_mask); 98 break; 99 } 100 101 if (option_length > sizeof(**dev_op_gqi_qpl)) { 102 dev_warn(&priv->pdev->dev, 103 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL"); 104 } 105 *dev_op_gqi_qpl = (void *)(option + 1); 106 break; 107 case GVE_DEV_OPT_ID_DQO_RDA: 108 if (option_length < sizeof(**dev_op_dqo_rda) || 109 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) { 110 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 111 "DQO RDA", (int)sizeof(**dev_op_dqo_rda), 112 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA, 113 option_length, req_feat_mask); 114 break; 115 } 116 117 if (option_length > sizeof(**dev_op_dqo_rda)) { 118 dev_warn(&priv->pdev->dev, 119 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA"); 120 } 121 *dev_op_dqo_rda = (void *)(option + 1); 122 break; 123 case GVE_DEV_OPT_ID_DQO_QPL: 124 if (option_length < sizeof(**dev_op_dqo_qpl) || 125 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) { 126 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 127 "DQO QPL", (int)sizeof(**dev_op_dqo_qpl), 128 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL, 129 option_length, req_feat_mask); 130 break; 131 } 132 133 if (option_length > sizeof(**dev_op_dqo_qpl)) { 134 dev_warn(&priv->pdev->dev, 135 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL"); 136 } 137 *dev_op_dqo_qpl = (void *)(option + 1); 138 break; 139 case GVE_DEV_OPT_ID_JUMBO_FRAMES: 140 if (option_length < sizeof(**dev_op_jumbo_frames) || 141 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) { 142 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 143 "Jumbo Frames", 144 (int)sizeof(**dev_op_jumbo_frames), 145 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES, 146 option_length, req_feat_mask); 147 break; 148 } 149 150 if (option_length > sizeof(**dev_op_jumbo_frames)) { 151 dev_warn(&priv->pdev->dev, 152 GVE_DEVICE_OPTION_TOO_BIG_FMT, 153 "Jumbo Frames"); 154 } 155 *dev_op_jumbo_frames = (void *)(option + 1); 156 break; 157 case GVE_DEV_OPT_ID_BUFFER_SIZES: 158 if (option_length < sizeof(**dev_op_buffer_sizes) || 159 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_BUFFER_SIZES) { 160 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 161 "Buffer Sizes", 162 (int)sizeof(**dev_op_buffer_sizes), 163 GVE_DEV_OPT_REQ_FEAT_MASK_BUFFER_SIZES, 164 option_length, req_feat_mask); 165 break; 166 } 167 168 if (option_length > sizeof(**dev_op_buffer_sizes)) 169 dev_warn(&priv->pdev->dev, 170 GVE_DEVICE_OPTION_TOO_BIG_FMT, 171 "Buffer Sizes"); 172 *dev_op_buffer_sizes = (void *)(option + 1); 173 break; 174 case GVE_DEV_OPT_ID_MODIFY_RING: 175 if (option_length < GVE_DEVICE_OPTION_NO_MIN_RING_SIZE || 176 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING) { 177 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 178 "Modify Ring", (int)sizeof(**dev_op_modify_ring), 179 GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING, 180 option_length, req_feat_mask); 181 break; 182 } 183 184 if (option_length > sizeof(**dev_op_modify_ring)) { 185 dev_warn(&priv->pdev->dev, 186 GVE_DEVICE_OPTION_TOO_BIG_FMT, "Modify Ring"); 187 } 188 189 *dev_op_modify_ring = (void *)(option + 1); 190 191 /* device has not provided min ring size */ 192 if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE) 193 priv->default_min_ring_size = true; 194 break; 195 case GVE_DEV_OPT_ID_FLOW_STEERING: 196 if (option_length < sizeof(**dev_op_flow_steering) || 197 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING) { 198 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 199 "Flow Steering", 200 (int)sizeof(**dev_op_flow_steering), 201 GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING, 202 option_length, req_feat_mask); 203 break; 204 } 205 206 if (option_length > sizeof(**dev_op_flow_steering)) 207 dev_warn(&priv->pdev->dev, 208 GVE_DEVICE_OPTION_TOO_BIG_FMT, 209 "Flow Steering"); 210 *dev_op_flow_steering = (void *)(option + 1); 211 break; 212 case GVE_DEV_OPT_ID_RSS_CONFIG: 213 if (option_length < sizeof(**dev_op_rss_config) || 214 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_RSS_CONFIG) { 215 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 216 "RSS config", 217 (int)sizeof(**dev_op_rss_config), 218 GVE_DEV_OPT_REQ_FEAT_MASK_RSS_CONFIG, 219 option_length, req_feat_mask); 220 break; 221 } 222 223 if (option_length > sizeof(**dev_op_rss_config)) 224 dev_warn(&priv->pdev->dev, 225 GVE_DEVICE_OPTION_TOO_BIG_FMT, 226 "RSS config"); 227 *dev_op_rss_config = (void *)(option + 1); 228 break; 229 case GVE_DEV_OPT_ID_NIC_TIMESTAMP: 230 if (option_length < sizeof(**dev_op_nic_timestamp) || 231 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP) { 232 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 233 "Nic Timestamp", 234 (int)sizeof(**dev_op_nic_timestamp), 235 GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP, 236 option_length, req_feat_mask); 237 break; 238 } 239 240 if (option_length > sizeof(**dev_op_nic_timestamp)) 241 dev_warn(&priv->pdev->dev, 242 GVE_DEVICE_OPTION_TOO_BIG_FMT, 243 "Nic Timestamp"); 244 *dev_op_nic_timestamp = (void *)(option + 1); 245 break; 246 default: 247 /* If we don't recognize the option just continue 248 * without doing anything. 249 */ 250 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n", 251 option_id); 252 } 253 } 254 255 /* Process all device options for a given describe device call. */ 256 static int 257 gve_process_device_options(struct gve_priv *priv, 258 struct gve_device_descriptor *descriptor, 259 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 260 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 261 struct gve_device_option_dqo_rda **dev_op_dqo_rda, 262 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames, 263 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl, 264 struct gve_device_option_buffer_sizes **dev_op_buffer_sizes, 265 struct gve_device_option_flow_steering **dev_op_flow_steering, 266 struct gve_device_option_rss_config **dev_op_rss_config, 267 struct gve_device_option_nic_timestamp **dev_op_nic_timestamp, 268 struct gve_device_option_modify_ring **dev_op_modify_ring) 269 { 270 const int num_options = be16_to_cpu(descriptor->num_device_options); 271 struct gve_device_option *dev_opt; 272 int i; 273 274 /* The options struct directly follows the device descriptor. */ 275 dev_opt = (void *)(descriptor + 1); 276 for (i = 0; i < num_options; i++) { 277 struct gve_device_option *next_opt; 278 279 next_opt = gve_get_next_option(descriptor, dev_opt); 280 if (!next_opt) { 281 dev_err(&priv->dev->dev, 282 "options exceed device_descriptor's total length.\n"); 283 return -EINVAL; 284 } 285 286 gve_parse_device_option(priv, descriptor, dev_opt, 287 dev_op_gqi_rda, dev_op_gqi_qpl, 288 dev_op_dqo_rda, dev_op_jumbo_frames, 289 dev_op_dqo_qpl, dev_op_buffer_sizes, 290 dev_op_flow_steering, dev_op_rss_config, 291 dev_op_nic_timestamp, 292 dev_op_modify_ring); 293 dev_opt = next_opt; 294 } 295 296 return 0; 297 } 298 299 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) 300 { 301 priv->adminq_pool = dma_pool_create("adminq_pool", dev, 302 GVE_ADMINQ_BUFFER_SIZE, 0, 0); 303 if (unlikely(!priv->adminq_pool)) 304 return -ENOMEM; 305 priv->adminq = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, 306 &priv->adminq_bus_addr); 307 if (unlikely(!priv->adminq)) { 308 dma_pool_destroy(priv->adminq_pool); 309 return -ENOMEM; 310 } 311 312 priv->adminq_mask = 313 (GVE_ADMINQ_BUFFER_SIZE / sizeof(union gve_adminq_command)) - 1; 314 priv->adminq_prod_cnt = 0; 315 priv->adminq_cmd_fail = 0; 316 priv->adminq_timeouts = 0; 317 priv->adminq_describe_device_cnt = 0; 318 priv->adminq_cfg_device_resources_cnt = 0; 319 priv->adminq_register_page_list_cnt = 0; 320 priv->adminq_unregister_page_list_cnt = 0; 321 priv->adminq_create_tx_queue_cnt = 0; 322 priv->adminq_create_rx_queue_cnt = 0; 323 priv->adminq_destroy_tx_queue_cnt = 0; 324 priv->adminq_destroy_rx_queue_cnt = 0; 325 priv->adminq_dcfg_device_resources_cnt = 0; 326 priv->adminq_set_driver_parameter_cnt = 0; 327 priv->adminq_report_stats_cnt = 0; 328 priv->adminq_report_link_speed_cnt = 0; 329 priv->adminq_report_nic_timestamp_cnt = 0; 330 priv->adminq_get_ptype_map_cnt = 0; 331 priv->adminq_query_flow_rules_cnt = 0; 332 priv->adminq_cfg_flow_rule_cnt = 0; 333 priv->adminq_cfg_rss_cnt = 0; 334 priv->adminq_query_rss_cnt = 0; 335 336 /* Setup Admin queue with the device */ 337 if (priv->pdev->revision < 0x1) { 338 iowrite32be(priv->adminq_bus_addr / PAGE_SIZE, 339 &priv->reg_bar0->adminq_pfn); 340 } else { 341 iowrite16be(GVE_ADMINQ_BUFFER_SIZE, 342 &priv->reg_bar0->adminq_length); 343 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 344 iowrite32be(priv->adminq_bus_addr >> 32, 345 &priv->reg_bar0->adminq_base_address_hi); 346 #endif 347 iowrite32be(priv->adminq_bus_addr, 348 &priv->reg_bar0->adminq_base_address_lo); 349 iowrite32be(GVE_DRIVER_STATUS_RUN_MASK, &priv->reg_bar0->driver_status); 350 } 351 mutex_init(&priv->adminq_lock); 352 gve_set_admin_queue_ok(priv); 353 return 0; 354 } 355 356 void gve_adminq_release(struct gve_priv *priv) 357 { 358 int i = 0; 359 360 /* Tell the device the adminq is leaving */ 361 if (priv->pdev->revision < 0x1) { 362 iowrite32be(0x0, &priv->reg_bar0->adminq_pfn); 363 while (ioread32be(&priv->reg_bar0->adminq_pfn)) { 364 /* If this is reached the device is unrecoverable and still 365 * holding memory. Continue looping to avoid memory corruption, 366 * but WARN so it is visible what is going on. 367 */ 368 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK) 369 WARN(1, "Unrecoverable platform error!"); 370 i++; 371 msleep(GVE_ADMINQ_SLEEP_LEN); 372 } 373 } else { 374 iowrite32be(GVE_DRIVER_STATUS_RESET_MASK, &priv->reg_bar0->driver_status); 375 while (!(ioread32be(&priv->reg_bar0->device_status) 376 & GVE_DEVICE_STATUS_DEVICE_IS_RESET)) { 377 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK) 378 WARN(1, "Unrecoverable platform error!"); 379 i++; 380 msleep(GVE_ADMINQ_SLEEP_LEN); 381 } 382 } 383 gve_clear_device_rings_ok(priv); 384 gve_clear_device_resources_ok(priv); 385 gve_clear_admin_queue_ok(priv); 386 } 387 388 void gve_adminq_free(struct device *dev, struct gve_priv *priv) 389 { 390 if (!gve_get_admin_queue_ok(priv)) 391 return; 392 gve_adminq_release(priv); 393 dma_pool_free(priv->adminq_pool, priv->adminq, priv->adminq_bus_addr); 394 dma_pool_destroy(priv->adminq_pool); 395 gve_clear_admin_queue_ok(priv); 396 } 397 398 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt) 399 { 400 iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell); 401 } 402 403 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt) 404 { 405 int i; 406 407 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) { 408 if (ioread32be(&priv->reg_bar0->adminq_event_counter) 409 == prod_cnt) 410 return true; 411 msleep(GVE_ADMINQ_SLEEP_LEN); 412 } 413 414 return false; 415 } 416 417 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status) 418 { 419 if (status != GVE_ADMINQ_COMMAND_PASSED && 420 status != GVE_ADMINQ_COMMAND_UNSET) { 421 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status); 422 priv->adminq_cmd_fail++; 423 } 424 switch (status) { 425 case GVE_ADMINQ_COMMAND_PASSED: 426 return 0; 427 case GVE_ADMINQ_COMMAND_UNSET: 428 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n"); 429 return -EINVAL; 430 case GVE_ADMINQ_COMMAND_ERROR_ABORTED: 431 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED: 432 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS: 433 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION: 434 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE: 435 return -EAGAIN; 436 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS: 437 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR: 438 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT: 439 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND: 440 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE: 441 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR: 442 return -EINVAL; 443 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED: 444 return -ETIME; 445 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED: 446 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED: 447 return -EACCES; 448 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED: 449 return -ENOMEM; 450 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED: 451 return -EOPNOTSUPP; 452 default: 453 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status); 454 return -EINVAL; 455 } 456 } 457 458 /* Flushes all AQ commands currently queued and waits for them to complete. 459 * If there are failures, it will return the first error. 460 */ 461 static int gve_adminq_kick_and_wait(struct gve_priv *priv) 462 { 463 int tail, head; 464 int i; 465 466 lockdep_assert_held(&priv->adminq_lock); 467 468 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 469 head = priv->adminq_prod_cnt; 470 471 gve_adminq_kick_cmd(priv, head); 472 if (!gve_adminq_wait_for_cmd(priv, head)) { 473 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n"); 474 priv->adminq_timeouts++; 475 return -ENOTRECOVERABLE; 476 } 477 478 for (i = tail; i < head; i++) { 479 union gve_adminq_command *cmd; 480 u32 status, err; 481 482 cmd = &priv->adminq[i & priv->adminq_mask]; 483 status = be32_to_cpu(READ_ONCE(cmd->status)); 484 err = gve_adminq_parse_err(priv, status); 485 if (err) 486 // Return the first error if we failed. 487 return err; 488 } 489 490 return 0; 491 } 492 493 static int gve_adminq_issue_cmd(struct gve_priv *priv, 494 union gve_adminq_command *cmd_orig) 495 { 496 union gve_adminq_command *cmd; 497 u32 opcode; 498 u32 tail; 499 500 lockdep_assert_held(&priv->adminq_lock); 501 502 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 503 504 // Check if next command will overflow the buffer. 505 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 506 (tail & priv->adminq_mask)) { 507 int err; 508 509 // Flush existing commands to make room. 510 err = gve_adminq_kick_and_wait(priv); 511 if (err) 512 return err; 513 514 // Retry. 515 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 516 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 517 (tail & priv->adminq_mask)) { 518 // This should never happen. We just flushed the 519 // command queue so there should be enough space. 520 return -ENOMEM; 521 } 522 } 523 524 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask]; 525 priv->adminq_prod_cnt++; 526 527 memcpy(cmd, cmd_orig, sizeof(*cmd_orig)); 528 opcode = be32_to_cpu(READ_ONCE(cmd->opcode)); 529 if (opcode == GVE_ADMINQ_EXTENDED_COMMAND) 530 opcode = be32_to_cpu(cmd->extended_command.inner_opcode); 531 532 switch (opcode) { 533 case GVE_ADMINQ_DESCRIBE_DEVICE: 534 priv->adminq_describe_device_cnt++; 535 break; 536 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES: 537 priv->adminq_cfg_device_resources_cnt++; 538 break; 539 case GVE_ADMINQ_REGISTER_PAGE_LIST: 540 priv->adminq_register_page_list_cnt++; 541 break; 542 case GVE_ADMINQ_UNREGISTER_PAGE_LIST: 543 priv->adminq_unregister_page_list_cnt++; 544 break; 545 case GVE_ADMINQ_CREATE_TX_QUEUE: 546 priv->adminq_create_tx_queue_cnt++; 547 break; 548 case GVE_ADMINQ_CREATE_RX_QUEUE: 549 priv->adminq_create_rx_queue_cnt++; 550 break; 551 case GVE_ADMINQ_DESTROY_TX_QUEUE: 552 priv->adminq_destroy_tx_queue_cnt++; 553 break; 554 case GVE_ADMINQ_DESTROY_RX_QUEUE: 555 priv->adminq_destroy_rx_queue_cnt++; 556 break; 557 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES: 558 priv->adminq_dcfg_device_resources_cnt++; 559 break; 560 case GVE_ADMINQ_SET_DRIVER_PARAMETER: 561 priv->adminq_set_driver_parameter_cnt++; 562 break; 563 case GVE_ADMINQ_REPORT_STATS: 564 priv->adminq_report_stats_cnt++; 565 break; 566 case GVE_ADMINQ_REPORT_LINK_SPEED: 567 priv->adminq_report_link_speed_cnt++; 568 break; 569 case GVE_ADMINQ_REPORT_NIC_TIMESTAMP: 570 priv->adminq_report_nic_timestamp_cnt++; 571 break; 572 case GVE_ADMINQ_GET_PTYPE_MAP: 573 priv->adminq_get_ptype_map_cnt++; 574 break; 575 case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY: 576 priv->adminq_verify_driver_compatibility_cnt++; 577 break; 578 case GVE_ADMINQ_QUERY_FLOW_RULES: 579 priv->adminq_query_flow_rules_cnt++; 580 break; 581 case GVE_ADMINQ_CONFIGURE_FLOW_RULE: 582 priv->adminq_cfg_flow_rule_cnt++; 583 break; 584 case GVE_ADMINQ_CONFIGURE_RSS: 585 priv->adminq_cfg_rss_cnt++; 586 break; 587 case GVE_ADMINQ_QUERY_RSS: 588 priv->adminq_query_rss_cnt++; 589 break; 590 default: 591 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); 592 return -EINVAL; 593 } 594 595 return 0; 596 } 597 598 static int gve_adminq_execute_cmd(struct gve_priv *priv, 599 union gve_adminq_command *cmd_orig) 600 { 601 u32 tail, head; 602 int err; 603 604 mutex_lock(&priv->adminq_lock); 605 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 606 head = priv->adminq_prod_cnt; 607 if (tail != head) { 608 err = -EINVAL; 609 goto out; 610 } 611 612 err = gve_adminq_issue_cmd(priv, cmd_orig); 613 if (err) 614 goto out; 615 616 err = gve_adminq_kick_and_wait(priv); 617 618 out: 619 mutex_unlock(&priv->adminq_lock); 620 return err; 621 } 622 623 static int gve_adminq_execute_extended_cmd(struct gve_priv *priv, u32 opcode, 624 size_t cmd_size, void *cmd_orig) 625 { 626 union gve_adminq_command cmd; 627 dma_addr_t inner_cmd_bus; 628 void *inner_cmd; 629 int err; 630 631 inner_cmd = dma_alloc_coherent(&priv->pdev->dev, cmd_size, 632 &inner_cmd_bus, GFP_KERNEL); 633 if (!inner_cmd) 634 return -ENOMEM; 635 636 memcpy(inner_cmd, cmd_orig, cmd_size); 637 638 memset(&cmd, 0, sizeof(cmd)); 639 cmd.opcode = cpu_to_be32(GVE_ADMINQ_EXTENDED_COMMAND); 640 cmd.extended_command = (struct gve_adminq_extended_command) { 641 .inner_opcode = cpu_to_be32(opcode), 642 .inner_length = cpu_to_be32(cmd_size), 643 .inner_command_addr = cpu_to_be64(inner_cmd_bus), 644 }; 645 646 err = gve_adminq_execute_cmd(priv, &cmd); 647 648 dma_free_coherent(&priv->pdev->dev, cmd_size, inner_cmd, inner_cmd_bus); 649 return err; 650 } 651 652 /* The device specifies that the management vector can either be the first irq 653 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to 654 * the ntfy blks. If it is 0 then the management vector is last, if it is 1 then 655 * the management vector is first. 656 * 657 * gve arranges the msix vectors so that the management vector is last. 658 */ 659 #define GVE_NTFY_BLK_BASE_MSIX_IDX 0 660 int gve_adminq_configure_device_resources(struct gve_priv *priv, 661 dma_addr_t counter_array_bus_addr, 662 u32 num_counters, 663 dma_addr_t db_array_bus_addr, 664 u32 num_ntfy_blks) 665 { 666 union gve_adminq_command cmd; 667 668 memset(&cmd, 0, sizeof(cmd)); 669 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES); 670 cmd.configure_device_resources = 671 (struct gve_adminq_configure_device_resources) { 672 .counter_array = cpu_to_be64(counter_array_bus_addr), 673 .num_counters = cpu_to_be32(num_counters), 674 .irq_db_addr = cpu_to_be64(db_array_bus_addr), 675 .num_irq_dbs = cpu_to_be32(num_ntfy_blks), 676 .irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)), 677 .ntfy_blk_msix_base_idx = 678 cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX), 679 .queue_format = priv->queue_format, 680 }; 681 682 return gve_adminq_execute_cmd(priv, &cmd); 683 } 684 685 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv) 686 { 687 union gve_adminq_command cmd; 688 689 memset(&cmd, 0, sizeof(cmd)); 690 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES); 691 692 return gve_adminq_execute_cmd(priv, &cmd); 693 } 694 695 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) 696 { 697 struct gve_tx_ring *tx = &priv->tx[queue_index]; 698 union gve_adminq_command cmd; 699 700 memset(&cmd, 0, sizeof(cmd)); 701 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE); 702 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) { 703 .queue_id = cpu_to_be32(queue_index), 704 .queue_resources_addr = 705 cpu_to_be64(tx->q_resources_bus), 706 .tx_ring_addr = cpu_to_be64(tx->bus), 707 .ntfy_id = cpu_to_be32(tx->ntfy_id), 708 .tx_ring_size = cpu_to_be16(priv->tx_desc_cnt), 709 }; 710 711 if (gve_is_gqi(priv)) { 712 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 713 GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; 714 715 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 716 } else { 717 u32 qpl_id = 0; 718 719 if (priv->queue_format == GVE_DQO_RDA_FORMAT) 720 qpl_id = GVE_RAW_ADDRESSING_QPL_ID; 721 else 722 qpl_id = tx->dqo.qpl->id; 723 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 724 cmd.create_tx_queue.tx_comp_ring_addr = 725 cpu_to_be64(tx->complq_bus_dqo); 726 cmd.create_tx_queue.tx_comp_ring_size = 727 cpu_to_be16(priv->tx_desc_cnt); 728 } 729 730 return gve_adminq_issue_cmd(priv, &cmd); 731 } 732 733 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues) 734 { 735 int err; 736 int i; 737 738 mutex_lock(&priv->adminq_lock); 739 740 for (i = start_id; i < start_id + num_queues; i++) { 741 err = gve_adminq_create_tx_queue(priv, i); 742 if (err) 743 goto out; 744 } 745 746 err = gve_adminq_kick_and_wait(priv); 747 748 out: 749 mutex_unlock(&priv->adminq_lock); 750 return err; 751 } 752 753 static void gve_adminq_get_create_rx_queue_cmd(struct gve_priv *priv, 754 union gve_adminq_command *cmd, 755 u32 queue_index) 756 { 757 struct gve_rx_ring *rx = &priv->rx[queue_index]; 758 759 memset(cmd, 0, sizeof(*cmd)); 760 cmd->opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 761 cmd->create_rx_queue = (struct gve_adminq_create_rx_queue) { 762 .queue_id = cpu_to_be32(queue_index), 763 .ntfy_id = cpu_to_be32(rx->ntfy_id), 764 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), 765 .rx_ring_size = cpu_to_be16(priv->rx_desc_cnt), 766 .packet_buffer_size = cpu_to_be16(rx->packet_buffer_size), 767 }; 768 769 if (gve_is_gqi(priv)) { 770 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 771 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; 772 773 cmd->create_rx_queue.rx_desc_ring_addr = 774 cpu_to_be64(rx->desc.bus); 775 cmd->create_rx_queue.rx_data_ring_addr = 776 cpu_to_be64(rx->data.data_bus); 777 cmd->create_rx_queue.index = cpu_to_be32(queue_index); 778 cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 779 } else { 780 u32 qpl_id = 0; 781 782 if (priv->queue_format == GVE_DQO_RDA_FORMAT) 783 qpl_id = GVE_RAW_ADDRESSING_QPL_ID; 784 else 785 qpl_id = rx->dqo.qpl->id; 786 cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 787 cmd->create_rx_queue.rx_desc_ring_addr = 788 cpu_to_be64(rx->dqo.complq.bus); 789 cmd->create_rx_queue.rx_data_ring_addr = 790 cpu_to_be64(rx->dqo.bufq.bus); 791 cmd->create_rx_queue.rx_buff_ring_size = 792 cpu_to_be16(priv->rx_desc_cnt); 793 cmd->create_rx_queue.enable_rsc = 794 !!(priv->dev->features & NETIF_F_LRO); 795 if (priv->header_split_enabled) 796 cmd->create_rx_queue.header_buffer_size = 797 cpu_to_be16(priv->header_buf_size); 798 } 799 } 800 801 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 802 { 803 union gve_adminq_command cmd; 804 805 gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index); 806 return gve_adminq_issue_cmd(priv, &cmd); 807 } 808 809 /* Unlike gve_adminq_create_rx_queue, this actually rings the doorbell */ 810 int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index) 811 { 812 union gve_adminq_command cmd; 813 814 gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index); 815 return gve_adminq_execute_cmd(priv, &cmd); 816 } 817 818 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) 819 { 820 int err; 821 int i; 822 823 mutex_lock(&priv->adminq_lock); 824 825 for (i = 0; i < num_queues; i++) { 826 err = gve_adminq_create_rx_queue(priv, i); 827 if (err) 828 goto out; 829 } 830 831 err = gve_adminq_kick_and_wait(priv); 832 833 out: 834 mutex_unlock(&priv->adminq_lock); 835 return err; 836 } 837 838 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index) 839 { 840 union gve_adminq_command cmd; 841 int err; 842 843 memset(&cmd, 0, sizeof(cmd)); 844 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE); 845 cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) { 846 .queue_id = cpu_to_be32(queue_index), 847 }; 848 849 err = gve_adminq_issue_cmd(priv, &cmd); 850 if (err) 851 return err; 852 853 return 0; 854 } 855 856 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues) 857 { 858 int err; 859 int i; 860 861 mutex_lock(&priv->adminq_lock); 862 863 for (i = start_id; i < start_id + num_queues; i++) { 864 err = gve_adminq_destroy_tx_queue(priv, i); 865 if (err) 866 goto out; 867 } 868 869 err = gve_adminq_kick_and_wait(priv); 870 871 out: 872 mutex_unlock(&priv->adminq_lock); 873 return err; 874 } 875 876 static void gve_adminq_make_destroy_rx_queue_cmd(union gve_adminq_command *cmd, 877 u32 queue_index) 878 { 879 memset(cmd, 0, sizeof(*cmd)); 880 cmd->opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 881 cmd->destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 882 .queue_id = cpu_to_be32(queue_index), 883 }; 884 } 885 886 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index) 887 { 888 union gve_adminq_command cmd; 889 890 gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index); 891 return gve_adminq_issue_cmd(priv, &cmd); 892 } 893 894 /* Unlike gve_adminq_destroy_rx_queue, this actually rings the doorbell */ 895 int gve_adminq_destroy_single_rx_queue(struct gve_priv *priv, u32 queue_index) 896 { 897 union gve_adminq_command cmd; 898 899 gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index); 900 return gve_adminq_execute_cmd(priv, &cmd); 901 } 902 903 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues) 904 { 905 int err; 906 int i; 907 908 mutex_lock(&priv->adminq_lock); 909 910 for (i = 0; i < num_queues; i++) { 911 err = gve_adminq_destroy_rx_queue(priv, i); 912 if (err) 913 goto out; 914 } 915 916 err = gve_adminq_kick_and_wait(priv); 917 918 out: 919 mutex_unlock(&priv->adminq_lock); 920 return err; 921 } 922 923 static void gve_set_default_desc_cnt(struct gve_priv *priv, 924 const struct gve_device_descriptor *descriptor) 925 { 926 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 927 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 928 929 /* set default ranges */ 930 priv->max_tx_desc_cnt = priv->tx_desc_cnt; 931 priv->max_rx_desc_cnt = priv->rx_desc_cnt; 932 priv->min_tx_desc_cnt = priv->tx_desc_cnt; 933 priv->min_rx_desc_cnt = priv->rx_desc_cnt; 934 } 935 936 static void gve_set_default_rss_sizes(struct gve_priv *priv) 937 { 938 if (!gve_is_gqi(priv)) { 939 priv->rss_key_size = GVE_RSS_KEY_SIZE; 940 priv->rss_lut_size = GVE_RSS_INDIR_SIZE; 941 priv->cache_rss_config = true; 942 } 943 } 944 945 static void gve_enable_supported_features(struct gve_priv *priv, 946 u32 supported_features_mask, 947 const struct gve_device_option_jumbo_frames 948 *dev_op_jumbo_frames, 949 const struct gve_device_option_dqo_qpl 950 *dev_op_dqo_qpl, 951 const struct gve_device_option_buffer_sizes 952 *dev_op_buffer_sizes, 953 const struct gve_device_option_flow_steering 954 *dev_op_flow_steering, 955 const struct gve_device_option_rss_config 956 *dev_op_rss_config, 957 const struct gve_device_option_nic_timestamp 958 *dev_op_nic_timestamp, 959 const struct gve_device_option_modify_ring 960 *dev_op_modify_ring) 961 { 962 /* Before control reaches this point, the page-size-capped max MTU from 963 * the gve_device_descriptor field has already been stored in 964 * priv->dev->max_mtu. We overwrite it with the true max MTU below. 965 */ 966 if (dev_op_jumbo_frames && 967 (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) { 968 dev_info(&priv->pdev->dev, 969 "JUMBO FRAMES device option enabled.\n"); 970 priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); 971 } 972 973 /* Override pages for qpl for DQO-QPL */ 974 if (dev_op_dqo_qpl) { 975 priv->tx_pages_per_qpl = 976 be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl); 977 if (priv->tx_pages_per_qpl == 0) 978 priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES; 979 } 980 981 if (dev_op_buffer_sizes && 982 (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) { 983 priv->max_rx_buffer_size = 984 be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size); 985 priv->header_buf_size = 986 be16_to_cpu(dev_op_buffer_sizes->header_buffer_size); 987 dev_info(&priv->pdev->dev, 988 "BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n", 989 priv->max_rx_buffer_size, priv->header_buf_size); 990 if (gve_is_dqo(priv) && 991 priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE) 992 priv->rx_cfg.packet_buffer_size = 993 priv->max_rx_buffer_size; 994 } 995 996 /* Read and store ring size ranges given by device */ 997 if (dev_op_modify_ring && 998 (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) { 999 priv->modify_ring_size_enabled = true; 1000 1001 /* max ring size for DQO QPL should not be overwritten because of device limit */ 1002 if (priv->queue_format != GVE_DQO_QPL_FORMAT) { 1003 priv->max_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_rx_ring_size); 1004 priv->max_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_tx_ring_size); 1005 } 1006 if (priv->default_min_ring_size) { 1007 /* If device hasn't provided minimums, use default minimums */ 1008 priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE; 1009 priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE; 1010 } else { 1011 priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size); 1012 priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size); 1013 } 1014 } 1015 1016 if (dev_op_flow_steering && 1017 (supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) { 1018 if (dev_op_flow_steering->max_flow_rules) { 1019 priv->max_flow_rules = 1020 be32_to_cpu(dev_op_flow_steering->max_flow_rules); 1021 priv->dev->hw_features |= NETIF_F_NTUPLE; 1022 dev_info(&priv->pdev->dev, 1023 "FLOW STEERING device option enabled with max rule limit of %u.\n", 1024 priv->max_flow_rules); 1025 } 1026 } 1027 1028 if (dev_op_rss_config && 1029 (supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) { 1030 priv->rss_key_size = 1031 be16_to_cpu(dev_op_rss_config->hash_key_size); 1032 priv->rss_lut_size = 1033 be16_to_cpu(dev_op_rss_config->hash_lut_size); 1034 priv->cache_rss_config = false; 1035 dev_dbg(&priv->pdev->dev, 1036 "RSS device option enabled with key size of %u, lut size of %u.\n", 1037 priv->rss_key_size, priv->rss_lut_size); 1038 } 1039 1040 if (dev_op_nic_timestamp && 1041 (supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK)) 1042 priv->nic_timestamp_supported = true; 1043 } 1044 1045 int gve_adminq_describe_device(struct gve_priv *priv) 1046 { 1047 struct gve_device_option_nic_timestamp *dev_op_nic_timestamp = NULL; 1048 struct gve_device_option_flow_steering *dev_op_flow_steering = NULL; 1049 struct gve_device_option_buffer_sizes *dev_op_buffer_sizes = NULL; 1050 struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL; 1051 struct gve_device_option_modify_ring *dev_op_modify_ring = NULL; 1052 struct gve_device_option_rss_config *dev_op_rss_config = NULL; 1053 struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; 1054 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; 1055 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; 1056 struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL; 1057 struct gve_device_descriptor *descriptor; 1058 u32 supported_features_mask = 0; 1059 union gve_adminq_command cmd; 1060 dma_addr_t descriptor_bus; 1061 int err = 0; 1062 u8 *mac; 1063 u16 mtu; 1064 1065 memset(&cmd, 0, sizeof(cmd)); 1066 descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, 1067 &descriptor_bus); 1068 if (!descriptor) 1069 return -ENOMEM; 1070 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE); 1071 cmd.describe_device.device_descriptor_addr = 1072 cpu_to_be64(descriptor_bus); 1073 cmd.describe_device.device_descriptor_version = 1074 cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION); 1075 cmd.describe_device.available_length = 1076 cpu_to_be32(GVE_ADMINQ_BUFFER_SIZE); 1077 1078 err = gve_adminq_execute_cmd(priv, &cmd); 1079 if (err) 1080 goto free_device_descriptor; 1081 1082 err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, 1083 &dev_op_gqi_qpl, &dev_op_dqo_rda, 1084 &dev_op_jumbo_frames, &dev_op_dqo_qpl, 1085 &dev_op_buffer_sizes, 1086 &dev_op_flow_steering, 1087 &dev_op_rss_config, 1088 &dev_op_nic_timestamp, 1089 &dev_op_modify_ring); 1090 if (err) 1091 goto free_device_descriptor; 1092 1093 /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format 1094 * is not set to GqiRda, choose the queue format in a priority order: 1095 * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default. 1096 */ 1097 if (dev_op_dqo_rda) { 1098 priv->queue_format = GVE_DQO_RDA_FORMAT; 1099 dev_info(&priv->pdev->dev, 1100 "Driver is running with DQO RDA queue format.\n"); 1101 supported_features_mask = 1102 be32_to_cpu(dev_op_dqo_rda->supported_features_mask); 1103 } else if (dev_op_dqo_qpl) { 1104 priv->queue_format = GVE_DQO_QPL_FORMAT; 1105 supported_features_mask = 1106 be32_to_cpu(dev_op_dqo_qpl->supported_features_mask); 1107 } else if (dev_op_gqi_rda) { 1108 priv->queue_format = GVE_GQI_RDA_FORMAT; 1109 dev_info(&priv->pdev->dev, 1110 "Driver is running with GQI RDA queue format.\n"); 1111 supported_features_mask = 1112 be32_to_cpu(dev_op_gqi_rda->supported_features_mask); 1113 } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) { 1114 dev_info(&priv->pdev->dev, 1115 "Driver is running with GQI RDA queue format.\n"); 1116 } else { 1117 priv->queue_format = GVE_GQI_QPL_FORMAT; 1118 if (dev_op_gqi_qpl) 1119 supported_features_mask = 1120 be32_to_cpu(dev_op_gqi_qpl->supported_features_mask); 1121 dev_info(&priv->pdev->dev, 1122 "Driver is running with GQI QPL queue format.\n"); 1123 } 1124 1125 /* set default descriptor counts */ 1126 gve_set_default_desc_cnt(priv, descriptor); 1127 1128 gve_set_default_rss_sizes(priv); 1129 1130 /* DQO supports LRO. */ 1131 if (!gve_is_gqi(priv)) 1132 priv->dev->hw_features |= NETIF_F_LRO; 1133 1134 priv->max_registered_pages = 1135 be64_to_cpu(descriptor->max_registered_pages); 1136 mtu = be16_to_cpu(descriptor->mtu); 1137 if (mtu < ETH_MIN_MTU) { 1138 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 1139 err = -EINVAL; 1140 goto free_device_descriptor; 1141 } 1142 priv->dev->max_mtu = mtu; 1143 priv->num_event_counters = be16_to_cpu(descriptor->counters); 1144 eth_hw_addr_set(priv->dev, descriptor->mac); 1145 mac = descriptor->mac; 1146 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 1147 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); 1148 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); 1149 1150 gve_enable_supported_features(priv, supported_features_mask, 1151 dev_op_jumbo_frames, dev_op_dqo_qpl, 1152 dev_op_buffer_sizes, dev_op_flow_steering, 1153 dev_op_rss_config, dev_op_nic_timestamp, 1154 dev_op_modify_ring); 1155 1156 free_device_descriptor: 1157 dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus); 1158 return err; 1159 } 1160 1161 int gve_adminq_register_page_list(struct gve_priv *priv, 1162 struct gve_queue_page_list *qpl) 1163 { 1164 struct device *hdev = &priv->pdev->dev; 1165 u32 num_entries = qpl->num_entries; 1166 u32 size = num_entries * sizeof(qpl->page_buses[0]); 1167 union gve_adminq_command cmd; 1168 dma_addr_t page_list_bus; 1169 __be64 *page_list; 1170 int err; 1171 int i; 1172 1173 memset(&cmd, 0, sizeof(cmd)); 1174 page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL); 1175 if (!page_list) 1176 return -ENOMEM; 1177 1178 for (i = 0; i < num_entries; i++) 1179 page_list[i] = cpu_to_be64(qpl->page_buses[i]); 1180 1181 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST); 1182 cmd.reg_page_list = (struct gve_adminq_register_page_list) { 1183 .page_list_id = cpu_to_be32(qpl->id), 1184 .num_pages = cpu_to_be32(num_entries), 1185 .page_address_list_addr = cpu_to_be64(page_list_bus), 1186 .page_size = cpu_to_be64(PAGE_SIZE), 1187 }; 1188 1189 err = gve_adminq_execute_cmd(priv, &cmd); 1190 dma_free_coherent(hdev, size, page_list, page_list_bus); 1191 return err; 1192 } 1193 1194 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id) 1195 { 1196 union gve_adminq_command cmd; 1197 1198 memset(&cmd, 0, sizeof(cmd)); 1199 cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST); 1200 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) { 1201 .page_list_id = cpu_to_be32(page_list_id), 1202 }; 1203 1204 return gve_adminq_execute_cmd(priv, &cmd); 1205 } 1206 1207 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, 1208 dma_addr_t stats_report_addr, u64 interval) 1209 { 1210 union gve_adminq_command cmd; 1211 1212 memset(&cmd, 0, sizeof(cmd)); 1213 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS); 1214 cmd.report_stats = (struct gve_adminq_report_stats) { 1215 .stats_report_len = cpu_to_be64(stats_report_len), 1216 .stats_report_addr = cpu_to_be64(stats_report_addr), 1217 .interval = cpu_to_be64(interval), 1218 }; 1219 1220 return gve_adminq_execute_cmd(priv, &cmd); 1221 } 1222 1223 int gve_adminq_verify_driver_compatibility(struct gve_priv *priv, 1224 u64 driver_info_len, 1225 dma_addr_t driver_info_addr) 1226 { 1227 union gve_adminq_command cmd; 1228 1229 memset(&cmd, 0, sizeof(cmd)); 1230 cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY); 1231 cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) { 1232 .driver_info_len = cpu_to_be64(driver_info_len), 1233 .driver_info_addr = cpu_to_be64(driver_info_addr), 1234 }; 1235 1236 return gve_adminq_execute_cmd(priv, &cmd); 1237 } 1238 1239 int gve_adminq_report_link_speed(struct gve_priv *priv) 1240 { 1241 union gve_adminq_command gvnic_cmd; 1242 dma_addr_t link_speed_region_bus; 1243 __be64 *link_speed_region; 1244 int err; 1245 1246 link_speed_region = 1247 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region), 1248 &link_speed_region_bus, GFP_KERNEL); 1249 1250 if (!link_speed_region) 1251 return -ENOMEM; 1252 1253 memset(&gvnic_cmd, 0, sizeof(gvnic_cmd)); 1254 gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED); 1255 gvnic_cmd.report_link_speed.link_speed_address = 1256 cpu_to_be64(link_speed_region_bus); 1257 1258 err = gve_adminq_execute_cmd(priv, &gvnic_cmd); 1259 1260 priv->link_speed = be64_to_cpu(*link_speed_region); 1261 dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region, 1262 link_speed_region_bus); 1263 return err; 1264 } 1265 1266 int gve_adminq_report_nic_ts(struct gve_priv *priv, 1267 dma_addr_t nic_ts_report_addr) 1268 { 1269 union gve_adminq_command cmd; 1270 1271 memset(&cmd, 0, sizeof(cmd)); 1272 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_NIC_TIMESTAMP); 1273 cmd.report_nic_ts = (struct gve_adminq_report_nic_ts) { 1274 .nic_ts_report_len = 1275 cpu_to_be64(sizeof(struct gve_nic_ts_report)), 1276 .nic_ts_report_addr = cpu_to_be64(nic_ts_report_addr), 1277 }; 1278 1279 return gve_adminq_execute_cmd(priv, &cmd); 1280 } 1281 1282 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, 1283 struct gve_ptype_lut *ptype_lut) 1284 { 1285 struct gve_ptype_map *ptype_map; 1286 union gve_adminq_command cmd; 1287 dma_addr_t ptype_map_bus; 1288 int err = 0; 1289 int i; 1290 1291 memset(&cmd, 0, sizeof(cmd)); 1292 ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map), 1293 &ptype_map_bus, GFP_KERNEL); 1294 if (!ptype_map) 1295 return -ENOMEM; 1296 1297 cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP); 1298 cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) { 1299 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)), 1300 .ptype_map_addr = cpu_to_be64(ptype_map_bus), 1301 }; 1302 1303 err = gve_adminq_execute_cmd(priv, &cmd); 1304 if (err) 1305 goto err; 1306 1307 /* Populate ptype_lut. */ 1308 for (i = 0; i < GVE_NUM_PTYPES; i++) { 1309 ptype_lut->ptypes[i].l3_type = 1310 ptype_map->ptypes[i].l3_type; 1311 ptype_lut->ptypes[i].l4_type = 1312 ptype_map->ptypes[i].l4_type; 1313 } 1314 err: 1315 dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map, 1316 ptype_map_bus); 1317 return err; 1318 } 1319 1320 static int 1321 gve_adminq_configure_flow_rule(struct gve_priv *priv, 1322 struct gve_adminq_configure_flow_rule *flow_rule_cmd) 1323 { 1324 int err = gve_adminq_execute_extended_cmd(priv, 1325 GVE_ADMINQ_CONFIGURE_FLOW_RULE, 1326 sizeof(struct gve_adminq_configure_flow_rule), 1327 flow_rule_cmd); 1328 1329 if (err == -ETIME) { 1330 dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset"); 1331 gve_reset(priv, true); 1332 } else if (!err) { 1333 priv->flow_rules_cache.rules_cache_synced = false; 1334 } 1335 1336 return err; 1337 } 1338 1339 int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc) 1340 { 1341 struct gve_adminq_configure_flow_rule flow_rule_cmd = { 1342 .opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_ADD), 1343 .location = cpu_to_be32(loc), 1344 .rule = *rule, 1345 }; 1346 1347 return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd); 1348 } 1349 1350 int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc) 1351 { 1352 struct gve_adminq_configure_flow_rule flow_rule_cmd = { 1353 .opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_DEL), 1354 .location = cpu_to_be32(loc), 1355 }; 1356 1357 return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd); 1358 } 1359 1360 int gve_adminq_reset_flow_rules(struct gve_priv *priv) 1361 { 1362 struct gve_adminq_configure_flow_rule flow_rule_cmd = { 1363 .opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_RESET), 1364 }; 1365 1366 return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd); 1367 } 1368 1369 int gve_adminq_configure_rss(struct gve_priv *priv, struct ethtool_rxfh_param *rxfh) 1370 { 1371 const u32 *hash_lut_to_config = NULL; 1372 const u8 *hash_key_to_config = NULL; 1373 dma_addr_t lut_bus = 0, key_bus = 0; 1374 union gve_adminq_command cmd; 1375 __be32 *lut = NULL; 1376 u8 hash_alg = 0; 1377 u8 *key = NULL; 1378 int err = 0; 1379 u16 i; 1380 1381 switch (rxfh->hfunc) { 1382 case ETH_RSS_HASH_NO_CHANGE: 1383 fallthrough; 1384 case ETH_RSS_HASH_TOP: 1385 hash_alg = ETH_RSS_HASH_TOP; 1386 break; 1387 default: 1388 return -EOPNOTSUPP; 1389 } 1390 1391 if (rxfh->indir) { 1392 if (rxfh->indir_size != priv->rss_lut_size) 1393 return -EINVAL; 1394 1395 hash_lut_to_config = rxfh->indir; 1396 } else if (priv->cache_rss_config) { 1397 hash_lut_to_config = priv->rss_config.hash_lut; 1398 } 1399 1400 if (hash_lut_to_config) { 1401 lut = dma_alloc_coherent(&priv->pdev->dev, 1402 priv->rss_lut_size * sizeof(*lut), 1403 &lut_bus, GFP_KERNEL); 1404 if (!lut) 1405 return -ENOMEM; 1406 1407 for (i = 0; i < priv->rss_lut_size; i++) 1408 lut[i] = cpu_to_be32(hash_lut_to_config[i]); 1409 } 1410 1411 if (rxfh->key) { 1412 if (rxfh->key_size != priv->rss_key_size) { 1413 err = -EINVAL; 1414 goto out; 1415 } 1416 1417 hash_key_to_config = rxfh->key; 1418 } else if (priv->cache_rss_config) { 1419 hash_key_to_config = priv->rss_config.hash_key; 1420 } 1421 1422 if (hash_key_to_config) { 1423 key = dma_alloc_coherent(&priv->pdev->dev, 1424 priv->rss_key_size, 1425 &key_bus, GFP_KERNEL); 1426 if (!key) { 1427 err = -ENOMEM; 1428 goto out; 1429 } 1430 1431 memcpy(key, hash_key_to_config, priv->rss_key_size); 1432 } 1433 1434 /* Zero-valued fields in the cmd.configure_rss instruct the device to 1435 * not update those fields. 1436 */ 1437 memset(&cmd, 0, sizeof(cmd)); 1438 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_RSS); 1439 cmd.configure_rss = (struct gve_adminq_configure_rss) { 1440 .hash_types = cpu_to_be16(BIT(GVE_RSS_HASH_TCPV4) | 1441 BIT(GVE_RSS_HASH_UDPV4) | 1442 BIT(GVE_RSS_HASH_TCPV6) | 1443 BIT(GVE_RSS_HASH_UDPV6)), 1444 .hash_alg = hash_alg, 1445 .hash_key_size = 1446 cpu_to_be16((key_bus) ? priv->rss_key_size : 0), 1447 .hash_lut_size = 1448 cpu_to_be16((lut_bus) ? priv->rss_lut_size : 0), 1449 .hash_key_addr = cpu_to_be64(key_bus), 1450 .hash_lut_addr = cpu_to_be64(lut_bus), 1451 }; 1452 1453 err = gve_adminq_execute_cmd(priv, &cmd); 1454 1455 out: 1456 if (lut) 1457 dma_free_coherent(&priv->pdev->dev, 1458 priv->rss_lut_size * sizeof(*lut), 1459 lut, lut_bus); 1460 if (key) 1461 dma_free_coherent(&priv->pdev->dev, 1462 priv->rss_key_size, key, key_bus); 1463 return err; 1464 } 1465 1466 /* In the dma memory that the driver allocated for the device to query the flow rules, the device 1467 * will first write it with a struct of gve_query_flow_rules_descriptor. Next to it, the device 1468 * will write an array of rules or rule ids with the count that specified in the descriptor. 1469 * For GVE_FLOW_RULE_QUERY_STATS, the device will only write the descriptor. 1470 */ 1471 static int gve_adminq_process_flow_rules_query(struct gve_priv *priv, u16 query_opcode, 1472 struct gve_query_flow_rules_descriptor *descriptor) 1473 { 1474 struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache; 1475 u32 num_queried_rules, total_memory_len, rule_info_len; 1476 void *rule_info; 1477 1478 total_memory_len = be32_to_cpu(descriptor->total_length); 1479 num_queried_rules = be32_to_cpu(descriptor->num_queried_rules); 1480 rule_info = (void *)(descriptor + 1); 1481 1482 switch (query_opcode) { 1483 case GVE_FLOW_RULE_QUERY_RULES: 1484 rule_info_len = num_queried_rules * sizeof(*flow_rules_cache->rules_cache); 1485 if (sizeof(*descriptor) + rule_info_len != total_memory_len) { 1486 dev_err(&priv->dev->dev, "flow rules query is out of memory.\n"); 1487 return -ENOMEM; 1488 } 1489 1490 memcpy(flow_rules_cache->rules_cache, rule_info, rule_info_len); 1491 flow_rules_cache->rules_cache_num = num_queried_rules; 1492 break; 1493 case GVE_FLOW_RULE_QUERY_IDS: 1494 rule_info_len = num_queried_rules * sizeof(*flow_rules_cache->rule_ids_cache); 1495 if (sizeof(*descriptor) + rule_info_len != total_memory_len) { 1496 dev_err(&priv->dev->dev, "flow rule ids query is out of memory.\n"); 1497 return -ENOMEM; 1498 } 1499 1500 memcpy(flow_rules_cache->rule_ids_cache, rule_info, rule_info_len); 1501 flow_rules_cache->rule_ids_cache_num = num_queried_rules; 1502 break; 1503 case GVE_FLOW_RULE_QUERY_STATS: 1504 priv->num_flow_rules = be32_to_cpu(descriptor->num_flow_rules); 1505 priv->max_flow_rules = be32_to_cpu(descriptor->max_flow_rules); 1506 return 0; 1507 default: 1508 return -EINVAL; 1509 } 1510 1511 return 0; 1512 } 1513 1514 int gve_adminq_query_flow_rules(struct gve_priv *priv, u16 query_opcode, u32 starting_loc) 1515 { 1516 struct gve_query_flow_rules_descriptor *descriptor; 1517 union gve_adminq_command cmd; 1518 dma_addr_t descriptor_bus; 1519 int err = 0; 1520 1521 memset(&cmd, 0, sizeof(cmd)); 1522 descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, &descriptor_bus); 1523 if (!descriptor) 1524 return -ENOMEM; 1525 1526 cmd.opcode = cpu_to_be32(GVE_ADMINQ_QUERY_FLOW_RULES); 1527 cmd.query_flow_rules = (struct gve_adminq_query_flow_rules) { 1528 .opcode = cpu_to_be16(query_opcode), 1529 .starting_rule_id = cpu_to_be32(starting_loc), 1530 .available_length = cpu_to_be64(GVE_ADMINQ_BUFFER_SIZE), 1531 .rule_descriptor_addr = cpu_to_be64(descriptor_bus), 1532 }; 1533 err = gve_adminq_execute_cmd(priv, &cmd); 1534 if (err) 1535 goto out; 1536 1537 err = gve_adminq_process_flow_rules_query(priv, query_opcode, descriptor); 1538 1539 out: 1540 dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus); 1541 return err; 1542 } 1543 1544 static int gve_adminq_process_rss_query(struct gve_priv *priv, 1545 struct gve_query_rss_descriptor *descriptor, 1546 struct ethtool_rxfh_param *rxfh) 1547 { 1548 u32 total_memory_length; 1549 u16 hash_lut_length; 1550 void *rss_info_addr; 1551 __be32 *lut; 1552 u16 i; 1553 1554 total_memory_length = be32_to_cpu(descriptor->total_length); 1555 hash_lut_length = priv->rss_lut_size * sizeof(*rxfh->indir); 1556 1557 if (sizeof(*descriptor) + priv->rss_key_size + hash_lut_length != total_memory_length) { 1558 dev_err(&priv->dev->dev, 1559 "rss query desc from device has invalid length parameter.\n"); 1560 return -EINVAL; 1561 } 1562 1563 rxfh->hfunc = descriptor->hash_alg; 1564 1565 rss_info_addr = (void *)(descriptor + 1); 1566 if (rxfh->key) { 1567 rxfh->key_size = priv->rss_key_size; 1568 memcpy(rxfh->key, rss_info_addr, priv->rss_key_size); 1569 } 1570 1571 rss_info_addr += priv->rss_key_size; 1572 lut = (__be32 *)rss_info_addr; 1573 if (rxfh->indir) { 1574 rxfh->indir_size = priv->rss_lut_size; 1575 for (i = 0; i < priv->rss_lut_size; i++) 1576 rxfh->indir[i] = be32_to_cpu(lut[i]); 1577 } 1578 1579 return 0; 1580 } 1581 1582 int gve_adminq_query_rss_config(struct gve_priv *priv, struct ethtool_rxfh_param *rxfh) 1583 { 1584 struct gve_query_rss_descriptor *descriptor; 1585 union gve_adminq_command cmd; 1586 dma_addr_t descriptor_bus; 1587 int err = 0; 1588 1589 descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, &descriptor_bus); 1590 if (!descriptor) 1591 return -ENOMEM; 1592 1593 memset(&cmd, 0, sizeof(cmd)); 1594 cmd.opcode = cpu_to_be32(GVE_ADMINQ_QUERY_RSS); 1595 cmd.query_rss = (struct gve_adminq_query_rss) { 1596 .available_length = cpu_to_be64(GVE_ADMINQ_BUFFER_SIZE), 1597 .rss_descriptor_addr = cpu_to_be64(descriptor_bus), 1598 }; 1599 err = gve_adminq_execute_cmd(priv, &cmd); 1600 if (err) 1601 goto out; 1602 1603 err = gve_adminq_process_rss_query(priv, descriptor, rxfh); 1604 1605 out: 1606 dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus); 1607 return err; 1608 } 1609