1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2023 Intel Corporation */ 3 4 #include <linux/export.h> 5 #include <linux/net/intel/libie/pci.h> 6 #include <net/libeth/rx.h> 7 8 #include "idpf.h" 9 #include "idpf_virtchnl.h" 10 #include "idpf_ptp.h" 11 12 /** 13 * idpf_vid_to_vport - Translate vport id to vport pointer 14 * @adapter: private data struct 15 * @v_id: vport id to translate 16 * 17 * Returns vport matching v_id, NULL if not found. 18 */ 19 static 20 struct idpf_vport *idpf_vid_to_vport(struct idpf_adapter *adapter, u32 v_id) 21 { 22 u16 num_max_vports = idpf_get_max_vports(adapter); 23 int i; 24 25 for (i = 0; i < num_max_vports; i++) 26 if (adapter->vport_ids[i] == v_id) 27 return adapter->vports[i]; 28 29 return NULL; 30 } 31 32 /** 33 * idpf_handle_event_link - Handle link event message 34 * @adapter: private data struct 35 * @v2e: virtchnl event message 36 */ 37 static void idpf_handle_event_link(struct idpf_adapter *adapter, 38 const struct virtchnl2_event *v2e) 39 { 40 struct idpf_netdev_priv *np; 41 struct idpf_vport *vport; 42 43 vport = idpf_vid_to_vport(adapter, le32_to_cpu(v2e->vport_id)); 44 if (!vport) { 45 dev_err_ratelimited(&adapter->pdev->dev, "Failed to find vport_id %d for link event\n", 46 v2e->vport_id); 47 return; 48 } 49 np = netdev_priv(vport->netdev); 50 51 np->link_speed_mbps = le32_to_cpu(v2e->link_speed); 52 53 if (vport->link_up == v2e->link_status) 54 return; 55 56 vport->link_up = v2e->link_status; 57 58 if (!test_bit(IDPF_VPORT_UP, np->state)) 59 return; 60 61 if (vport->link_up) { 62 netif_tx_start_all_queues(vport->netdev); 63 netif_carrier_on(vport->netdev); 64 } else { 65 netif_tx_stop_all_queues(vport->netdev); 66 netif_carrier_off(vport->netdev); 67 } 68 } 69 70 /** 71 * idpf_recv_event_msg - Receive virtchnl event message 72 * @ctx: control queue context 73 * @ctlq_msg: message to copy from 74 * 75 * Receive virtchnl event message 76 */ 77 void idpf_recv_event_msg(struct libie_ctlq_ctx *ctx, 78 struct libie_ctlq_msg *ctlq_msg) 79 { 80 struct kvec *buff = &ctlq_msg->recv_mem; 81 int payload_size = buff->iov_len; 82 struct idpf_adapter *adapter; 83 struct virtchnl2_event *v2e; 84 u32 event; 85 86 adapter = container_of(ctx, struct idpf_adapter, ctlq_ctx); 87 if (ctlq_msg->chnl_opcode != VIRTCHNL2_OP_EVENT) { 88 dev_dbg(&adapter->pdev->dev, 89 "Unhandled message with opcode %u from CP\n", 90 ctlq_msg->chnl_opcode); 91 goto free_rx_buf; 92 } 93 94 if (payload_size < sizeof(*v2e)) { 95 dev_err_ratelimited(&adapter->pdev->dev, "Failed to receive valid payload for event msg (op %d len %d)\n", 96 ctlq_msg->chnl_opcode, 97 payload_size); 98 goto free_rx_buf; 99 } 100 101 v2e = (struct virtchnl2_event *)buff->iov_base; 102 event = le32_to_cpu(v2e->event); 103 104 switch (event) { 105 case VIRTCHNL2_EVENT_LINK_CHANGE: 106 idpf_handle_event_link(adapter, v2e); 107 break; 108 default: 109 dev_err(&adapter->pdev->dev, 110 "Unknown event %d from PF\n", event); 111 break; 112 } 113 114 free_rx_buf: 115 libie_ctlq_release_rx_buf(buff); 116 } 117 118 /** 119 * idpf_mb_clean - Reclaim the send mailbox queue entries 120 * @asq: send control queue info 121 * @deinit: release all buffers before destroying the queue 122 * 123 * This is a helper function to clean the send mailbox queue entries. 124 */ 125 static void idpf_mb_clean(struct libie_ctlq_info *asq, bool deinit) 126 { 127 libie_ctlq_xn_send_clean(asq, kfree, deinit); 128 } 129 130 #if IS_ENABLED(CONFIG_PTP_1588_CLOCK) 131 /** 132 * idpf_ptp_is_mb_msg - Check if the message is PTP-related 133 * @op: virtchnl opcode 134 * 135 * Return: true if msg is PTP-related, false otherwise. 136 */ 137 static bool idpf_ptp_is_mb_msg(u32 op) 138 { 139 switch (op) { 140 case VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME: 141 case VIRTCHNL2_OP_PTP_GET_CROSS_TIME: 142 case VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME: 143 case VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE: 144 case VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME: 145 case VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS: 146 case VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP: 147 return true; 148 default: 149 return false; 150 } 151 } 152 153 /** 154 * idpf_prepare_ptp_mb_msg - Prepare PTP related message 155 * 156 * @adapter: Driver specific private structure 157 * @op: virtchnl opcode 158 * @ctlq_msg: Corresponding control queue message 159 */ 160 static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, 161 struct libie_ctlq_msg *ctlq_msg) 162 { 163 /* If the message is PTP-related and the secondary mailbox is available, 164 * send the message through the secondary mailbox. 165 */ 166 if (!idpf_ptp_is_mb_msg(op) || !adapter->ptp->secondary_mbx.valid) 167 return; 168 169 ctlq_msg->opcode = LIBIE_CTLQ_SEND_MSG_TO_PEER; 170 ctlq_msg->func_id = adapter->ptp->secondary_mbx.peer_mbx_q_id; 171 ctlq_msg->flags = FIELD_PREP(LIBIE_CTLQ_DESC_FLAG_HOST_ID, 172 adapter->ptp->secondary_mbx.peer_id); 173 } 174 #else /* !CONFIG_PTP_1588_CLOCK */ 175 static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, 176 struct libie_ctlq_msg *ctlq_msg) 177 { } 178 #endif /* CONFIG_PTP_1588_CLOCK */ 179 180 /** 181 * idpf_send_mb_msg - send mailbox message to the device control plane 182 * @adapter: driver specific private structure 183 * @xn_params: Xn send parameters to fill 184 * @send_buf: buffer to send 185 * @send_buf_size: size of the send buffer 186 * 187 * Fill the Xn parameters with the required info to send a virtchnl message. 188 * The send buffer is DMA mapped in the libie to avoid memcpy. 189 * 190 * Cleanup the mailbox queue entries of the previously sent message to 191 * unmap and release the buffer. 192 * 193 * Return: 0 if the request was successful, -%EBUSY if reset is detected 194 * or Tx control queue is full, other negative error code on failure. 195 */ 196 int idpf_send_mb_msg(struct idpf_adapter *adapter, 197 struct libie_ctlq_xn_send_params *xn_params, 198 void *send_buf, size_t send_buf_size) 199 { 200 struct libie_ctlq_msg ctlq_msg = {}; 201 202 if (idpf_is_reset_detected(adapter)) { 203 if (!libie_cp_can_send_onstack(send_buf_size)) 204 kfree(send_buf); 205 206 return -EBUSY; 207 } 208 209 idpf_prepare_ptp_mb_msg(adapter, xn_params->chnl_opcode, &ctlq_msg); 210 xn_params->ctlq_msg = ctlq_msg.opcode ? &ctlq_msg : NULL; 211 212 xn_params->send_buf.iov_base = send_buf; 213 xn_params->send_buf.iov_len = send_buf_size; 214 xn_params->xnm = adapter->xnm; 215 xn_params->ctlq = xn_params->ctlq ? xn_params->ctlq : adapter->asq; 216 xn_params->rel_tx_buf = kfree; 217 218 idpf_mb_clean(xn_params->ctlq, false); 219 220 return libie_ctlq_xn_send(xn_params); 221 } 222 223 /** 224 * idpf_send_mb_msg_kfree - send mailbox message and free the send buffer 225 * @adapter: driver specific private structure 226 * @xn_params: Xn send parameters to fill 227 * @send_buf: buffer to send, can be released with kfree() 228 * @send_buf_size: size of the send buffer 229 * 230 * libie_cp functions consume only buffers above certain size, 231 * smaller buffers are assumed to be on the stack. However, for some 232 * commands with variable message size it makes sense to always use kzalloc(), 233 * which means we have to free smaller buffers ourselves. 234 * 235 * Return: 0 if no unexpected errors were encountered, 236 * negative error code otherwise. 237 */ 238 int idpf_send_mb_msg_kfree(struct idpf_adapter *adapter, 239 struct libie_ctlq_xn_send_params *xn_params, 240 void *send_buf, size_t send_buf_size) 241 { 242 int err = idpf_send_mb_msg(adapter, xn_params, send_buf, send_buf_size); 243 244 if (libie_cp_can_send_onstack(send_buf_size)) 245 kfree(send_buf); 246 247 return err; 248 } 249 250 /** 251 * idpf_send_vf_reset_msg - send one way VF reset message 252 * @adapter: driver specific private structure 253 */ 254 void idpf_send_vf_reset_msg(struct idpf_adapter *adapter) 255 { 256 struct libie_ctlq_info *ctlq = adapter->asq; 257 258 /* Forcefully claim send queue slot */ 259 idpf_mb_clean(ctlq, true); 260 261 scoped_guard(spinlock, &ctlq->lock) { 262 *ctlq->tx_msg[ctlq->next_to_use] = (struct libie_ctlq_msg) { 263 .opcode = LIBIE_CTLQ_SEND_MSG_TO_CP, 264 .chnl_opcode = VIRTCHNL2_OP_RESET_VF, 265 }; 266 267 libie_ctlq_send(adapter->asq, 1); 268 } 269 } 270 271 struct idpf_chunked_msg_params { 272 u32 (*prepare_msg)(u32 vport_id, void *buf, 273 const void *pos, u32 num); 274 275 const void *chunks; 276 u32 num_chunks; 277 278 u32 chunk_sz; 279 u32 config_sz; 280 281 u32 vc_op; 282 u32 vport_id; 283 }; 284 285 struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_adapter *adapter, 286 struct idpf_q_vec_rsrc *qv_rsrc, 287 u32 vport_id, u32 num) 288 { 289 struct idpf_queue_set *qp; 290 291 qp = kzalloc_flex(*qp, qs, num); 292 if (!qp) 293 return NULL; 294 295 qp->adapter = adapter; 296 qp->qv_rsrc = qv_rsrc; 297 qp->vport_id = vport_id; 298 qp->num = num; 299 300 return qp; 301 } 302 303 /** 304 * idpf_send_chunked_msg - send VC message consisting of chunks 305 * @adapter: Driver specific private structure 306 * @params: message params 307 * 308 * Helper function for preparing a message describing queues to be enabled 309 * or disabled. 310 * 311 * Return: the total size of the prepared message. 312 */ 313 static int idpf_send_chunked_msg(struct idpf_adapter *adapter, 314 const struct idpf_chunked_msg_params *params) 315 { 316 struct libie_ctlq_xn_send_params xn_params = { 317 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 318 .chnl_opcode = params->vc_op, 319 }; 320 const void *pos = params->chunks; 321 u32 totqs = params->num_chunks; 322 u32 vid = params->vport_id; 323 u32 num_chunks, num_msgs; 324 325 num_chunks = IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, 326 params->chunk_sz); 327 num_msgs = DIV_ROUND_UP(totqs, num_chunks); 328 329 for (u32 i = 0; i < num_msgs; i++) { 330 u32 buf_sz; 331 void *buf; 332 int err; 333 334 num_chunks = min(num_chunks, totqs); 335 buf_sz = params->config_sz + num_chunks * params->chunk_sz; 336 buf = kzalloc(buf_sz, GFP_KERNEL); 337 if (!buf) 338 return -ENOMEM; 339 340 if (params->prepare_msg(vid, buf, pos, num_chunks) != buf_sz) { 341 kfree(buf); 342 return -EINVAL; 343 } 344 345 err = idpf_send_mb_msg_kfree(adapter, &xn_params, buf, buf_sz); 346 if (err) 347 return err; 348 349 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 350 xn_params.recv_mem = (struct kvec) {}; 351 pos += num_chunks * params->chunk_sz; 352 totqs -= num_chunks; 353 } 354 355 return 0; 356 } 357 358 /** 359 * idpf_wait_for_marker_event_set - wait for software marker response for 360 * selected Tx queues 361 * @qs: set of the Tx queues 362 * 363 * Return: 0 success, -errno on failure. 364 */ 365 static int idpf_wait_for_marker_event_set(const struct idpf_queue_set *qs) 366 { 367 struct net_device *netdev; 368 struct idpf_tx_queue *txq; 369 bool markers_rcvd = true; 370 371 for (u32 i = 0; i < qs->num; i++) { 372 switch (qs->qs[i].type) { 373 case VIRTCHNL2_QUEUE_TYPE_TX: 374 txq = qs->qs[i].txq; 375 376 netdev = txq->netdev; 377 378 idpf_queue_set(SW_MARKER, txq); 379 idpf_wait_for_sw_marker_completion(txq); 380 markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); 381 break; 382 default: 383 break; 384 } 385 } 386 387 if (!markers_rcvd) { 388 netdev_warn(netdev, 389 "Failed to receive marker packets\n"); 390 return -ETIMEDOUT; 391 } 392 393 return 0; 394 } 395 396 /** 397 * idpf_wait_for_marker_event - wait for software marker response 398 * @vport: virtual port data structure 399 * 400 * Return: 0 success, negative on failure. 401 **/ 402 static int idpf_wait_for_marker_event(struct idpf_vport *vport) 403 { 404 struct idpf_queue_set *qs __free(kfree) = NULL; 405 406 qs = idpf_alloc_queue_set(vport->adapter, &vport->dflt_qv_rsrc, 407 vport->vport_id, vport->num_txq); 408 if (!qs) 409 return -ENOMEM; 410 411 for (u32 i = 0; i < qs->num; i++) { 412 qs->qs[i].type = VIRTCHNL2_QUEUE_TYPE_TX; 413 qs->qs[i].txq = vport->txqs[i]; 414 } 415 416 return idpf_wait_for_marker_event_set(qs); 417 } 418 419 /** 420 * idpf_send_ver_msg - send virtchnl version message 421 * @adapter: Driver specific private structure 422 * 423 * Send virtchnl version message. Returns 0 on success, negative on failure. 424 */ 425 static int idpf_send_ver_msg(struct idpf_adapter *adapter) 426 { 427 struct libie_ctlq_xn_send_params xn_params = { 428 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 429 .chnl_opcode = VIRTCHNL2_OP_VERSION, 430 }; 431 struct virtchnl2_version_info *vvi_recv; 432 struct virtchnl2_version_info vvi; 433 u32 major, minor; 434 int err; 435 436 if (adapter->virt_ver_maj) { 437 vvi.major = cpu_to_le32(adapter->virt_ver_maj); 438 vvi.minor = cpu_to_le32(adapter->virt_ver_min); 439 } else { 440 vvi.major = cpu_to_le32(IDPF_VIRTCHNL_VERSION_MAJOR); 441 vvi.minor = cpu_to_le32(IDPF_VIRTCHNL_VERSION_MINOR); 442 } 443 444 err = idpf_send_mb_msg_stack(adapter, &xn_params, &vvi); 445 if (err) 446 return err; 447 448 if (xn_params.recv_mem.iov_len < sizeof(*vvi_recv)) { 449 err = -EIO; 450 goto free_rx_buf; 451 } 452 453 vvi_recv = xn_params.recv_mem.iov_base; 454 major = le32_to_cpu(vvi_recv->major); 455 minor = le32_to_cpu(vvi_recv->minor); 456 457 if (major > IDPF_VIRTCHNL_VERSION_MAJOR) { 458 dev_warn(&adapter->pdev->dev, "Virtchnl major version greater than supported\n"); 459 err = -EINVAL; 460 goto free_rx_buf; 461 } 462 463 if (major == IDPF_VIRTCHNL_VERSION_MAJOR && 464 minor > IDPF_VIRTCHNL_VERSION_MINOR) 465 dev_warn(&adapter->pdev->dev, "Virtchnl minor version didn't match\n"); 466 467 /* If we have a mismatch, resend version to update receiver on what 468 * version we will use. 469 */ 470 if (!adapter->virt_ver_maj && 471 major != IDPF_VIRTCHNL_VERSION_MAJOR && 472 minor != IDPF_VIRTCHNL_VERSION_MINOR) 473 err = -EAGAIN; 474 475 adapter->virt_ver_maj = major; 476 adapter->virt_ver_min = minor; 477 478 free_rx_buf: 479 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 480 481 return err; 482 } 483 484 /** 485 * idpf_send_get_caps_msg - Send virtchnl get capabilities message 486 * @adapter: Driver specific private structure 487 * 488 * Send virtchl get capabilities message. Returns 0 on success, negative on 489 * failure. 490 */ 491 static int idpf_send_get_caps_msg(struct idpf_adapter *adapter) 492 { 493 struct libie_ctlq_xn_send_params xn_params = { 494 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 495 .chnl_opcode = VIRTCHNL2_OP_GET_CAPS, 496 }; 497 struct virtchnl2_get_capabilities caps = {}; 498 int err; 499 500 caps.csum_caps = 501 cpu_to_le32(VIRTCHNL2_CAP_TX_CSUM_L3_IPV4 | 502 VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_TCP | 503 VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_UDP | 504 VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_SCTP | 505 VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_TCP | 506 VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_UDP | 507 VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_SCTP | 508 VIRTCHNL2_CAP_RX_CSUM_L3_IPV4 | 509 VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_TCP | 510 VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_UDP | 511 VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_SCTP | 512 VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_TCP | 513 VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_UDP | 514 VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_SCTP | 515 VIRTCHNL2_CAP_TX_CSUM_L3_SINGLE_TUNNEL | 516 VIRTCHNL2_CAP_RX_CSUM_L3_SINGLE_TUNNEL | 517 VIRTCHNL2_CAP_TX_CSUM_L4_SINGLE_TUNNEL | 518 VIRTCHNL2_CAP_RX_CSUM_L4_SINGLE_TUNNEL | 519 VIRTCHNL2_CAP_RX_CSUM_GENERIC); 520 521 caps.seg_caps = 522 cpu_to_le32(VIRTCHNL2_CAP_SEG_IPV4_TCP | 523 VIRTCHNL2_CAP_SEG_IPV4_UDP | 524 VIRTCHNL2_CAP_SEG_IPV4_SCTP | 525 VIRTCHNL2_CAP_SEG_IPV6_TCP | 526 VIRTCHNL2_CAP_SEG_IPV6_UDP | 527 VIRTCHNL2_CAP_SEG_IPV6_SCTP | 528 VIRTCHNL2_CAP_SEG_TX_SINGLE_TUNNEL); 529 530 caps.rss_caps = 531 cpu_to_le64(VIRTCHNL2_FLOW_IPV4_TCP | 532 VIRTCHNL2_FLOW_IPV4_UDP | 533 VIRTCHNL2_FLOW_IPV4_SCTP | 534 VIRTCHNL2_FLOW_IPV4_OTHER | 535 VIRTCHNL2_FLOW_IPV6_TCP | 536 VIRTCHNL2_FLOW_IPV6_UDP | 537 VIRTCHNL2_FLOW_IPV6_SCTP | 538 VIRTCHNL2_FLOW_IPV6_OTHER); 539 540 caps.hsplit_caps = 541 cpu_to_le32(VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V4 | 542 VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V6); 543 544 caps.rsc_caps = 545 cpu_to_le32(VIRTCHNL2_CAP_RSC_IPV4_TCP | 546 VIRTCHNL2_CAP_RSC_IPV6_TCP); 547 548 caps.other_caps = 549 cpu_to_le64(VIRTCHNL2_CAP_SRIOV | 550 VIRTCHNL2_CAP_RDMA | 551 VIRTCHNL2_CAP_LAN_MEMORY_REGIONS | 552 VIRTCHNL2_CAP_MACFILTER | 553 VIRTCHNL2_CAP_SPLITQ_QSCHED | 554 VIRTCHNL2_CAP_PROMISC | 555 VIRTCHNL2_CAP_LOOPBACK | 556 VIRTCHNL2_CAP_PTP); 557 558 err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); 559 if (err) 560 return err; 561 562 if (xn_params.recv_mem.iov_len < sizeof(adapter->caps)) { 563 err = -EIO; 564 goto free_rx_buf; 565 } 566 567 memcpy(&adapter->caps, xn_params.recv_mem.iov_base, 568 sizeof(adapter->caps)); 569 570 free_rx_buf: 571 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 572 573 return err; 574 } 575 576 /** 577 * idpf_mmio_region_non_static - Check if region is not static 578 * @mmio_info: PCI resources info 579 * @reg: region to check 580 * 581 * Return: %true if region can be received though virtchnl command, 582 * %false if region is related to mailbox or resetting 583 */ 584 bool idpf_mmio_region_non_static(struct libie_mmio_info *mmio_info, 585 struct libie_pci_mmio_region *reg) 586 { 587 struct idpf_adapter *adapter = 588 container_of(mmio_info, struct idpf_adapter, 589 ctlq_ctx.mmio_info); 590 591 for (uint i = 0; i < IDPF_MMIO_REG_NUM_STATIC; i++) { 592 if (reg->bar_idx == 0 && 593 reg->offset == adapter->dev_ops.static_reg_info[i].start) 594 return false; 595 } 596 597 return true; 598 } 599 600 /** 601 * idpf_decfg_lan_memory_regions - Unmap non-static memory regions 602 * @adapter: Driver specific private structure 603 */ 604 static void idpf_decfg_lan_memory_regions(struct idpf_adapter *adapter) 605 { 606 libie_pci_unmap_fltr_regs(&adapter->ctlq_ctx.mmio_info, 607 idpf_mmio_region_non_static); 608 } 609 610 /** 611 * idpf_cfg_lan_memory_regions - Get (via virtchnl) and map LAN memory regions 612 * @adapter: Driver specific private struct 613 * 614 * Return: 0 on success or error code on failure. 615 */ 616 static int idpf_cfg_lan_memory_regions(struct idpf_adapter *adapter) 617 { 618 struct virtchnl2_get_lan_memory_regions *send_regions, *rcvd_regions; 619 struct libie_ctlq_xn_send_params xn_params = { 620 .chnl_opcode = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS, 621 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 622 }; 623 size_t send_sz, reply_sz, size; 624 int num_regions; 625 int err = 0; 626 627 send_sz = sizeof(struct virtchnl2_get_lan_memory_regions) + 628 sizeof(struct virtchnl2_mem_region); 629 send_regions = kzalloc(send_sz, GFP_KERNEL); 630 if (!send_regions) 631 return -ENOMEM; 632 633 send_regions->num_memory_regions = cpu_to_le16(1); 634 err = idpf_send_mb_msg_kfree(adapter, &xn_params, send_regions, 635 send_sz); 636 if (err) 637 return err; 638 639 rcvd_regions = xn_params.recv_mem.iov_base; 640 reply_sz = xn_params.recv_mem.iov_len; 641 if (reply_sz < sizeof(*rcvd_regions)) { 642 err = -EIO; 643 goto rel_rx_buf; 644 } 645 num_regions = le16_to_cpu(rcvd_regions->num_memory_regions); 646 size = struct_size(rcvd_regions, mem_reg, num_regions); 647 if (reply_sz < size) { 648 err = -EIO; 649 goto rel_rx_buf; 650 } 651 652 for (int i = 0; i < num_regions; i++) { 653 struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; 654 resource_size_t offset, len; 655 656 offset = le64_to_cpu(rcvd_regions->mem_reg[i].start_offset); 657 len = le64_to_cpu(rcvd_regions->mem_reg[i].size); 658 if (len && !libie_pci_map_mmio_region(mmio, offset, len)) { 659 idpf_decfg_lan_memory_regions(adapter); 660 err = -EIO; 661 goto rel_rx_buf; 662 } 663 } 664 665 rel_rx_buf: 666 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 667 668 return err; 669 } 670 671 /** 672 * idpf_map_remaining_mmio_regs - map MMIO regions outside mbx and rstat 673 * @adapter: Driver specific private structure 674 * 675 * Called when idpf_cfg_lan_memory_regions is not supported. This will 676 * calculate the offsets and sizes for the regions before, in between, and 677 * after the mailbox and rstat MMIO mappings, and map those ranges. 678 * 679 * Return: 0 on success or error code on failure. 680 */ 681 static int idpf_map_remaining_mmio_regs(struct idpf_adapter *adapter) 682 { 683 struct resource *rstat_reg = &adapter->dev_ops.static_reg_info[1]; 684 struct resource *mbx_reg = &adapter->dev_ops.static_reg_info[0]; 685 struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; 686 resource_size_t reg_start, size; 687 bool ok = true; 688 689 /* Region preceding mailbox */ 690 size = mbx_reg->start; 691 ok &= !size || libie_pci_map_mmio_region(mmio, 0, size); 692 693 /* Region between mailbox and rstat */ 694 reg_start = mbx_reg->end + 1; 695 size = rstat_reg->start - reg_start; 696 ok &= !size || libie_pci_map_mmio_region(mmio, reg_start, size); 697 698 /* Region after rstat */ 699 reg_start = rstat_reg->end + 1; 700 size = pci_resource_len(adapter->pdev, 0) - reg_start; 701 ok &= !size || libie_pci_map_mmio_region(mmio, reg_start, size); 702 703 if (!ok) { 704 idpf_decfg_lan_memory_regions(adapter); 705 return -ENOMEM; 706 } 707 708 return 0; 709 } 710 711 /** 712 * idpf_add_del_fsteer_filters - Send virtchnl add/del Flow Steering message 713 * @adapter: adapter info struct 714 * @rule: Flow steering rule to add/delete 715 * @opcode: VIRTCHNL2_OP_ADD_FLOW_RULE to add filter, or 716 * VIRTCHNL2_OP_DEL_FLOW_RULE to delete. All other values are invalid. 717 * 718 * Send ADD/DELETE flow steering virtchnl message and receive the result. 719 * 720 * Return: 0 on success, negative on failure. 721 */ 722 int idpf_add_del_fsteer_filters(struct idpf_adapter *adapter, 723 struct virtchnl2_flow_rule_add_del *rule, 724 enum virtchnl2_op opcode) 725 { 726 struct libie_ctlq_xn_send_params xn_params = { 727 .chnl_opcode = opcode, 728 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 729 }; 730 struct virtchnl2_flow_rule_add_del *rx_rule; 731 int rule_count = le32_to_cpu(rule->count); 732 size_t send_sz; 733 int err; 734 735 if (opcode != VIRTCHNL2_OP_ADD_FLOW_RULE && 736 opcode != VIRTCHNL2_OP_DEL_FLOW_RULE) { 737 kfree(rule); 738 return -EINVAL; 739 } 740 741 send_sz = struct_size(rule, rule_info, rule_count); 742 err = idpf_send_mb_msg_kfree(adapter, &xn_params, rule, send_sz); 743 if (err) 744 return err; 745 746 if (xn_params.recv_mem.iov_len < send_sz) { 747 err = -EIO; 748 goto rel_rx; 749 } 750 751 rx_rule = xn_params.recv_mem.iov_base; 752 for (int i = 0; i < rule_count; i++) { 753 if (rx_rule->rule_info[i].status != 754 cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) { 755 err = -EIO; 756 goto rel_rx; 757 } 758 } 759 760 rel_rx: 761 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 762 return err; 763 } 764 765 /** 766 * idpf_vport_alloc_max_qs - Allocate max queues for a vport 767 * @adapter: Driver specific private structure 768 * @max_q: vport max queue structure 769 */ 770 int idpf_vport_alloc_max_qs(struct idpf_adapter *adapter, 771 struct idpf_vport_max_q *max_q) 772 { 773 struct idpf_avail_queue_info *avail_queues = &adapter->avail_queues; 774 struct virtchnl2_get_capabilities *caps = &adapter->caps; 775 u16 default_vports = idpf_get_default_vports(adapter); 776 u32 max_rx_q, max_tx_q, max_buf_q, max_compl_q; 777 778 mutex_lock(&adapter->queue_lock); 779 780 /* Caps are device-wide. Give each vport an equal piece */ 781 max_rx_q = le16_to_cpu(caps->max_rx_q) / default_vports; 782 max_tx_q = le16_to_cpu(caps->max_tx_q) / default_vports; 783 max_buf_q = le16_to_cpu(caps->max_rx_bufq) / default_vports; 784 max_compl_q = le16_to_cpu(caps->max_tx_complq) / default_vports; 785 786 if (adapter->num_alloc_vports >= default_vports) { 787 max_rx_q = IDPF_MIN_Q; 788 max_tx_q = IDPF_MIN_Q; 789 } 790 791 /* 792 * Harmonize the numbers. The current implementation always creates 793 * `IDPF_MAX_BUFQS_PER_RXQ_GRP` buffer queues for each Rx queue and 794 * one completion queue for each Tx queue for best performance. 795 * If less buffer or completion queues is available, cap the number 796 * of the corresponding Rx/Tx queues. 797 */ 798 max_rx_q = min(max_rx_q, max_buf_q / IDPF_MAX_BUFQS_PER_RXQ_GRP); 799 max_tx_q = min(max_tx_q, max_compl_q); 800 801 max_q->max_rxq = max_rx_q; 802 max_q->max_txq = max_tx_q; 803 max_q->max_bufq = max_rx_q * IDPF_MAX_BUFQS_PER_RXQ_GRP; 804 max_q->max_complq = max_tx_q; 805 806 if (avail_queues->avail_rxq < max_q->max_rxq || 807 avail_queues->avail_txq < max_q->max_txq || 808 avail_queues->avail_bufq < max_q->max_bufq || 809 avail_queues->avail_complq < max_q->max_complq) { 810 mutex_unlock(&adapter->queue_lock); 811 812 return -EINVAL; 813 } 814 815 avail_queues->avail_rxq -= max_q->max_rxq; 816 avail_queues->avail_txq -= max_q->max_txq; 817 avail_queues->avail_bufq -= max_q->max_bufq; 818 avail_queues->avail_complq -= max_q->max_complq; 819 820 mutex_unlock(&adapter->queue_lock); 821 822 return 0; 823 } 824 825 /** 826 * idpf_vport_dealloc_max_qs - Deallocate max queues of a vport 827 * @adapter: Driver specific private structure 828 * @max_q: vport max queue structure 829 */ 830 void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter, 831 struct idpf_vport_max_q *max_q) 832 { 833 struct idpf_avail_queue_info *avail_queues; 834 835 mutex_lock(&adapter->queue_lock); 836 avail_queues = &adapter->avail_queues; 837 838 avail_queues->avail_rxq += max_q->max_rxq; 839 avail_queues->avail_txq += max_q->max_txq; 840 avail_queues->avail_bufq += max_q->max_bufq; 841 avail_queues->avail_complq += max_q->max_complq; 842 843 mutex_unlock(&adapter->queue_lock); 844 } 845 846 /** 847 * idpf_init_avail_queues - Initialize available queues on the device 848 * @adapter: Driver specific private structure 849 */ 850 static void idpf_init_avail_queues(struct idpf_adapter *adapter) 851 { 852 struct idpf_avail_queue_info *avail_queues = &adapter->avail_queues; 853 struct virtchnl2_get_capabilities *caps = &adapter->caps; 854 855 avail_queues->avail_rxq = le16_to_cpu(caps->max_rx_q); 856 avail_queues->avail_txq = le16_to_cpu(caps->max_tx_q); 857 avail_queues->avail_bufq = le16_to_cpu(caps->max_rx_bufq); 858 avail_queues->avail_complq = le16_to_cpu(caps->max_tx_complq); 859 } 860 861 /** 862 * idpf_vport_init_queue_reg_chunks - initialize queue register chunks 863 * @vport_config: persistent vport structure to store the queue register info 864 * @schunks: source chunks to copy data from 865 * 866 * Return: 0 on success, negative on failure. 867 */ 868 static int 869 idpf_vport_init_queue_reg_chunks(struct idpf_vport_config *vport_config, 870 struct virtchnl2_queue_reg_chunks *schunks) 871 { 872 struct idpf_queue_id_reg_info *q_info = &vport_config->qid_reg_info; 873 u16 num_chunks = le16_to_cpu(schunks->num_chunks); 874 875 kfree(q_info->queue_chunks); 876 877 q_info->queue_chunks = kzalloc_objs(*q_info->queue_chunks, num_chunks); 878 if (!q_info->queue_chunks) { 879 q_info->num_chunks = 0; 880 return -ENOMEM; 881 } 882 883 q_info->num_chunks = num_chunks; 884 885 for (u16 i = 0; i < num_chunks; i++) { 886 struct idpf_queue_id_reg_chunk *dchunk = &q_info->queue_chunks[i]; 887 struct virtchnl2_queue_reg_chunk *schunk = &schunks->chunks[i]; 888 889 dchunk->qtail_reg_start = le64_to_cpu(schunk->qtail_reg_start); 890 dchunk->qtail_reg_spacing = le32_to_cpu(schunk->qtail_reg_spacing); 891 dchunk->type = le32_to_cpu(schunk->type); 892 dchunk->start_queue_id = le32_to_cpu(schunk->start_queue_id); 893 dchunk->num_queues = le32_to_cpu(schunk->num_queues); 894 } 895 896 return 0; 897 } 898 899 /** 900 * idpf_get_reg_intr_vecs - Get vector queue register offset 901 * @adapter: adapter structure to get the vector chunks 902 * @reg_vals: Register offsets to store in 903 * @num_vecs: number of entries the @reg_vals array can hold 904 * 905 * Return: number of registers that got populated 906 */ 907 int idpf_get_reg_intr_vecs(struct idpf_adapter *adapter, 908 struct idpf_vec_regs *reg_vals, int num_vecs) 909 { 910 struct virtchnl2_vector_chunks *chunks; 911 struct idpf_vec_regs reg_val; 912 u16 num_vchunks, num_vec; 913 int num_regs = 0, i, j; 914 915 chunks = &adapter->req_vec_chunks->vchunks; 916 num_vchunks = le16_to_cpu(chunks->num_vchunks); 917 918 for (j = 0; j < num_vchunks; j++) { 919 struct virtchnl2_vector_chunk *chunk; 920 u32 dynctl_reg_spacing; 921 u32 itrn_reg_spacing; 922 923 chunk = &chunks->vchunks[j]; 924 num_vec = le16_to_cpu(chunk->num_vectors); 925 reg_val.dyn_ctl_reg = le32_to_cpu(chunk->dynctl_reg_start); 926 reg_val.itrn_reg = le32_to_cpu(chunk->itrn_reg_start); 927 reg_val.itrn_index_spacing = le32_to_cpu(chunk->itrn_index_spacing); 928 929 dynctl_reg_spacing = le32_to_cpu(chunk->dynctl_reg_spacing); 930 itrn_reg_spacing = le32_to_cpu(chunk->itrn_reg_spacing); 931 932 for (i = 0; i < num_vec && num_regs < num_vecs; i++) { 933 reg_vals[num_regs].dyn_ctl_reg = reg_val.dyn_ctl_reg; 934 reg_vals[num_regs].itrn_reg = reg_val.itrn_reg; 935 reg_vals[num_regs].itrn_index_spacing = 936 reg_val.itrn_index_spacing; 937 938 reg_val.dyn_ctl_reg += dynctl_reg_spacing; 939 reg_val.itrn_reg += itrn_reg_spacing; 940 num_regs++; 941 } 942 } 943 944 return num_regs; 945 } 946 947 /** 948 * idpf_vport_get_q_reg - Get the queue registers for the vport 949 * @reg_vals: register values needing to be set 950 * @num_regs: amount we expect to fill 951 * @q_type: queue model 952 * @chunks: queue regs received over mailbox 953 * 954 * This function parses the queue register offsets from the queue register 955 * chunk information, with a specific queue type and stores it into the array 956 * passed as an argument. It returns the actual number of queue registers that 957 * are filled. 958 */ 959 static int idpf_vport_get_q_reg(u32 *reg_vals, int num_regs, u32 q_type, 960 struct idpf_queue_id_reg_info *chunks) 961 { 962 u16 num_chunks = chunks->num_chunks; 963 int reg_filled = 0, i; 964 u32 reg_val; 965 966 while (num_chunks--) { 967 struct idpf_queue_id_reg_chunk *chunk; 968 u16 num_q; 969 970 chunk = &chunks->queue_chunks[num_chunks]; 971 if (chunk->type != q_type) 972 continue; 973 974 num_q = chunk->num_queues; 975 reg_val = chunk->qtail_reg_start; 976 for (i = 0; i < num_q && reg_filled < num_regs ; i++) { 977 reg_vals[reg_filled++] = reg_val; 978 reg_val += chunk->qtail_reg_spacing; 979 } 980 } 981 982 return reg_filled; 983 } 984 985 /** 986 * __idpf_queue_reg_init - initialize queue registers 987 * @vport: virtual port structure 988 * @rsrc: pointer to queue and vector resources 989 * @reg_vals: registers we are initializing 990 * @num_regs: how many registers there are in total 991 * @q_type: queue model 992 * 993 * Return number of queues that are initialized 994 */ 995 static int __idpf_queue_reg_init(struct idpf_vport *vport, 996 struct idpf_q_vec_rsrc *rsrc, u32 *reg_vals, 997 int num_regs, u32 q_type) 998 { 999 struct libie_mmio_info *mmio = &vport->adapter->ctlq_ctx.mmio_info; 1000 int i, j, k = 0; 1001 1002 switch (q_type) { 1003 case VIRTCHNL2_QUEUE_TYPE_TX: 1004 for (i = 0; i < rsrc->num_txq_grp; i++) { 1005 struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 1006 1007 for (j = 0; j < tx_qgrp->num_txq && k < num_regs; j++, k++) 1008 tx_qgrp->txqs[j]->tail = 1009 libie_pci_get_mmio_addr(mmio, 1010 reg_vals[k]); 1011 } 1012 break; 1013 case VIRTCHNL2_QUEUE_TYPE_RX: 1014 for (i = 0; i < rsrc->num_rxq_grp; i++) { 1015 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1016 u16 num_rxq = rx_qgrp->singleq.num_rxq; 1017 1018 for (j = 0; j < num_rxq && k < num_regs; j++, k++) { 1019 struct idpf_rx_queue *q; 1020 1021 q = rx_qgrp->singleq.rxqs[j]; 1022 q->tail = libie_pci_get_mmio_addr(mmio, 1023 reg_vals[k]); 1024 } 1025 } 1026 break; 1027 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1028 for (i = 0; i < rsrc->num_rxq_grp; i++) { 1029 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1030 u8 num_bufqs = rsrc->num_bufqs_per_qgrp; 1031 1032 for (j = 0; j < num_bufqs && k < num_regs; j++, k++) { 1033 struct idpf_buf_queue *q; 1034 1035 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 1036 q->tail = libie_pci_get_mmio_addr(mmio, 1037 reg_vals[k]); 1038 } 1039 } 1040 break; 1041 default: 1042 break; 1043 } 1044 1045 return k; 1046 } 1047 1048 /** 1049 * idpf_queue_reg_init - initialize queue registers 1050 * @vport: virtual port structure 1051 * @rsrc: pointer to queue and vector resources 1052 * @chunks: queue registers received over mailbox 1053 * 1054 * Return: 0 on success, negative on failure 1055 */ 1056 int idpf_queue_reg_init(struct idpf_vport *vport, 1057 struct idpf_q_vec_rsrc *rsrc, 1058 struct idpf_queue_id_reg_info *chunks) 1059 { 1060 int num_regs, ret = 0; 1061 u32 *reg_vals; 1062 1063 /* We may never deal with more than 256 same type of queues */ 1064 reg_vals = kzalloc(sizeof(void *) * IDPF_LARGE_MAX_Q, GFP_KERNEL); 1065 if (!reg_vals) 1066 return -ENOMEM; 1067 1068 /* Initialize Tx queue tail register address */ 1069 num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, 1070 VIRTCHNL2_QUEUE_TYPE_TX, 1071 chunks); 1072 if (num_regs < rsrc->num_txq) { 1073 ret = -EINVAL; 1074 goto free_reg_vals; 1075 } 1076 1077 num_regs = __idpf_queue_reg_init(vport, rsrc, reg_vals, num_regs, 1078 VIRTCHNL2_QUEUE_TYPE_TX); 1079 if (num_regs < rsrc->num_txq) { 1080 ret = -EINVAL; 1081 goto free_reg_vals; 1082 } 1083 1084 /* Initialize Rx/buffer queue tail register address based on Rx queue 1085 * model 1086 */ 1087 if (idpf_is_queue_model_split(rsrc->rxq_model)) { 1088 num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, 1089 VIRTCHNL2_QUEUE_TYPE_RX_BUFFER, 1090 chunks); 1091 if (num_regs < rsrc->num_bufq) { 1092 ret = -EINVAL; 1093 goto free_reg_vals; 1094 } 1095 1096 num_regs = __idpf_queue_reg_init(vport, rsrc, reg_vals, num_regs, 1097 VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 1098 if (num_regs < rsrc->num_bufq) { 1099 ret = -EINVAL; 1100 goto free_reg_vals; 1101 } 1102 } else { 1103 num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, 1104 VIRTCHNL2_QUEUE_TYPE_RX, 1105 chunks); 1106 if (num_regs < rsrc->num_rxq) { 1107 ret = -EINVAL; 1108 goto free_reg_vals; 1109 } 1110 1111 num_regs = __idpf_queue_reg_init(vport, rsrc, reg_vals, num_regs, 1112 VIRTCHNL2_QUEUE_TYPE_RX); 1113 if (num_regs < rsrc->num_rxq) { 1114 ret = -EINVAL; 1115 goto free_reg_vals; 1116 } 1117 } 1118 1119 free_reg_vals: 1120 kfree(reg_vals); 1121 1122 return ret; 1123 } 1124 1125 /** 1126 * idpf_send_create_vport_msg - Send virtchnl create vport message 1127 * @adapter: Driver specific private structure 1128 * @max_q: vport max queue info 1129 * 1130 * send virtchnl creae vport message 1131 * 1132 * Returns 0 on success, negative on failure 1133 */ 1134 int idpf_send_create_vport_msg(struct idpf_adapter *adapter, 1135 struct idpf_vport_max_q *max_q) 1136 { 1137 struct libie_ctlq_xn_send_params xn_params = { 1138 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1139 .chnl_opcode = VIRTCHNL2_OP_CREATE_VPORT, 1140 }; 1141 struct virtchnl2_create_vport *vport_msg; 1142 u16 idx = adapter->next_vport; 1143 int err, buf_size; 1144 1145 buf_size = sizeof(struct virtchnl2_create_vport); 1146 vport_msg = kzalloc(buf_size, GFP_KERNEL); 1147 if (!vport_msg) 1148 return -ENOMEM; 1149 1150 vport_msg->vport_type = cpu_to_le16(VIRTCHNL2_VPORT_TYPE_DEFAULT); 1151 vport_msg->vport_index = cpu_to_le16(idx); 1152 1153 if (adapter->req_tx_splitq || !IS_ENABLED(CONFIG_IDPF_SINGLEQ)) 1154 vport_msg->txq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SPLIT); 1155 else 1156 vport_msg->txq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SINGLE); 1157 1158 if (adapter->req_rx_splitq || !IS_ENABLED(CONFIG_IDPF_SINGLEQ)) 1159 vport_msg->rxq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SPLIT); 1160 else 1161 vport_msg->rxq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SINGLE); 1162 1163 err = idpf_vport_calc_total_qs(adapter, idx, vport_msg, max_q); 1164 if (err) { 1165 dev_err(&adapter->pdev->dev, "Enough queues are not available"); 1166 goto rel_buf; 1167 } 1168 1169 if (!adapter->vport_params_recvd[idx]) { 1170 adapter->vport_params_recvd[idx] = 1171 kzalloc(LIBIE_CTLQ_MAX_BUF_LEN, GFP_KERNEL); 1172 if (!adapter->vport_params_recvd[idx]) { 1173 err = -ENOMEM; 1174 goto rel_buf; 1175 } 1176 } 1177 1178 err = idpf_send_mb_msg_kfree(adapter, &xn_params, vport_msg, 1179 sizeof(*vport_msg)); 1180 if (err) { 1181 kfree(adapter->vport_params_recvd[idx]); 1182 adapter->vport_params_recvd[idx] = NULL; 1183 return err; 1184 } 1185 1186 memcpy(adapter->vport_params_recvd[idx], xn_params.recv_mem.iov_base, 1187 xn_params.recv_mem.iov_len); 1188 1189 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1190 1191 return 0; 1192 1193 rel_buf: 1194 kfree(vport_msg); 1195 1196 return err; 1197 } 1198 1199 /** 1200 * idpf_check_supported_desc_ids - Verify we have required descriptor support 1201 * @vport: virtual port structure 1202 * 1203 * Return 0 on success, error on failure 1204 */ 1205 int idpf_check_supported_desc_ids(struct idpf_vport *vport) 1206 { 1207 struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; 1208 struct idpf_adapter *adapter = vport->adapter; 1209 struct virtchnl2_create_vport *vport_msg; 1210 u64 rx_desc_ids, tx_desc_ids; 1211 1212 vport_msg = adapter->vport_params_recvd[vport->idx]; 1213 1214 if (!IS_ENABLED(CONFIG_IDPF_SINGLEQ) && 1215 (vport_msg->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE || 1216 vport_msg->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE)) { 1217 pci_err(adapter->pdev, "singleq mode requested, but not compiled-in\n"); 1218 return -EOPNOTSUPP; 1219 } 1220 1221 rx_desc_ids = le64_to_cpu(vport_msg->rx_desc_ids); 1222 tx_desc_ids = le64_to_cpu(vport_msg->tx_desc_ids); 1223 1224 if (idpf_is_queue_model_split(rsrc->rxq_model)) { 1225 if (!(rx_desc_ids & VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M)) { 1226 dev_info(&adapter->pdev->dev, "Minimum RX descriptor support not provided, using the default\n"); 1227 vport_msg->rx_desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1228 } 1229 } else { 1230 if (!(rx_desc_ids & VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M)) 1231 rsrc->base_rxd = true; 1232 } 1233 1234 if (!idpf_is_queue_model_split(rsrc->txq_model)) 1235 return 0; 1236 1237 if ((tx_desc_ids & MIN_SUPPORT_TXDID) != MIN_SUPPORT_TXDID) { 1238 dev_info(&adapter->pdev->dev, "Minimum TX descriptor support not provided, using the default\n"); 1239 vport_msg->tx_desc_ids = cpu_to_le64(MIN_SUPPORT_TXDID); 1240 } 1241 1242 return 0; 1243 } 1244 1245 /** 1246 * idpf_send_destroy_vport_msg - Send virtchnl destroy vport message 1247 * @adapter: adapter pointer used to send virtchnl message 1248 * @vport_id: vport identifier used while preparing the virtchnl message 1249 * 1250 * Return: 0 on success, negative on failure. 1251 */ 1252 int idpf_send_destroy_vport_msg(struct idpf_adapter *adapter, u32 vport_id) 1253 { 1254 struct libie_ctlq_xn_send_params xn_params = { 1255 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1256 .chnl_opcode = VIRTCHNL2_OP_DESTROY_VPORT, 1257 }; 1258 struct virtchnl2_vport v_id; 1259 int err; 1260 1261 v_id.vport_id = cpu_to_le32(vport_id); 1262 1263 err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); 1264 if (err) 1265 return err; 1266 1267 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1268 1269 return 0; 1270 } 1271 1272 /** 1273 * idpf_send_enable_vport_msg - Send virtchnl enable vport message 1274 * @adapter: adapter pointer used to send virtchnl message 1275 * @vport_id: vport identifier used while preparing the virtchnl message 1276 * 1277 * Return: 0 on success, negative on failure. 1278 */ 1279 int idpf_send_enable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) 1280 { 1281 struct libie_ctlq_xn_send_params xn_params = { 1282 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1283 .chnl_opcode = VIRTCHNL2_OP_ENABLE_VPORT, 1284 }; 1285 struct virtchnl2_vport v_id; 1286 int err; 1287 1288 v_id.vport_id = cpu_to_le32(vport_id); 1289 1290 err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); 1291 if (err) 1292 return err; 1293 1294 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1295 1296 return 0; 1297 } 1298 1299 /** 1300 * idpf_send_disable_vport_msg - Send virtchnl disable vport message 1301 * @adapter: adapter pointer used to send virtchnl message 1302 * @vport_id: vport identifier used while preparing the virtchnl message 1303 * 1304 * Return: 0 on success, negative on failure. 1305 */ 1306 int idpf_send_disable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) 1307 { 1308 struct libie_ctlq_xn_send_params xn_params = { 1309 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1310 .chnl_opcode = VIRTCHNL2_OP_DISABLE_VPORT, 1311 }; 1312 struct virtchnl2_vport v_id; 1313 int err; 1314 1315 v_id.vport_id = cpu_to_le32(vport_id); 1316 1317 err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); 1318 if (err) 1319 return err; 1320 1321 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1322 1323 return 0; 1324 } 1325 1326 /** 1327 * idpf_fill_txq_config_chunk - fill chunk describing the Tx queue 1328 * @rsrc: pointer to queue and vector resources 1329 * @q: Tx queue to be inserted into VC chunk 1330 * @qi: pointer to the buffer containing the VC chunk 1331 */ 1332 static void idpf_fill_txq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1333 const struct idpf_tx_queue *q, 1334 struct virtchnl2_txq_info *qi) 1335 { 1336 u32 val; 1337 1338 qi->queue_id = cpu_to_le32(q->q_id); 1339 qi->model = cpu_to_le16(rsrc->txq_model); 1340 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 1341 qi->ring_len = cpu_to_le16(q->desc_count); 1342 qi->dma_ring_addr = cpu_to_le64(q->dma); 1343 qi->relative_queue_id = cpu_to_le16(q->rel_q_id); 1344 1345 if (!idpf_is_queue_model_split(rsrc->txq_model)) { 1346 qi->sched_mode = cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); 1347 return; 1348 } 1349 1350 if (idpf_queue_has(XDP, q)) 1351 val = q->complq->q_id; 1352 else 1353 val = q->txq_grp->complq->q_id; 1354 1355 qi->tx_compl_queue_id = cpu_to_le16(val); 1356 1357 if (idpf_queue_has(FLOW_SCH_EN, q)) 1358 val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1359 else 1360 val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1361 1362 qi->sched_mode = cpu_to_le16(val); 1363 } 1364 1365 /** 1366 * idpf_fill_complq_config_chunk - fill chunk describing the completion queue 1367 * @rsrc: pointer to queue and vector resources 1368 * @q: completion queue to be inserted into VC chunk 1369 * @qi: pointer to the buffer containing the VC chunk 1370 */ 1371 static void idpf_fill_complq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1372 const struct idpf_compl_queue *q, 1373 struct virtchnl2_txq_info *qi) 1374 { 1375 u32 val; 1376 1377 qi->queue_id = cpu_to_le32(q->q_id); 1378 qi->model = cpu_to_le16(rsrc->txq_model); 1379 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 1380 qi->ring_len = cpu_to_le16(q->desc_count); 1381 qi->dma_ring_addr = cpu_to_le64(q->dma); 1382 1383 if (idpf_queue_has(FLOW_SCH_EN, q)) 1384 val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1385 else 1386 val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1387 1388 qi->sched_mode = cpu_to_le16(val); 1389 } 1390 1391 /** 1392 * idpf_prepare_cfg_txqs_msg - prepare message to configure selected Tx queues 1393 * @vport_id: ID of virtual port queues are associated with 1394 * @buf: buffer containing the message 1395 * @pos: pointer to the first chunk describing the tx queue 1396 * @num_chunks: number of chunks in the message 1397 * 1398 * Helper function for preparing the message describing configuration of 1399 * Tx queues. 1400 * 1401 * Return: the total size of the prepared message. 1402 */ 1403 static u32 idpf_prepare_cfg_txqs_msg(u32 vport_id, void *buf, const void *pos, 1404 u32 num_chunks) 1405 { 1406 struct virtchnl2_config_tx_queues *ctq = buf; 1407 1408 ctq->vport_id = cpu_to_le32(vport_id); 1409 ctq->num_qinfo = cpu_to_le16(num_chunks); 1410 memcpy(ctq->qinfo, pos, num_chunks * sizeof(*ctq->qinfo)); 1411 1412 return struct_size(ctq, qinfo, num_chunks); 1413 } 1414 1415 /** 1416 * idpf_send_config_tx_queue_set_msg - send virtchnl config Tx queues 1417 * message for selected queues 1418 * @qs: set of the Tx queues to configure 1419 * 1420 * Send config queues virtchnl message for queues contained in the @qs array. 1421 * The @qs array can contain Tx queues (or completion queues) only. 1422 * 1423 * Return: 0 on success, -errno on failure. 1424 */ 1425 static int idpf_send_config_tx_queue_set_msg(const struct idpf_queue_set *qs) 1426 { 1427 struct virtchnl2_txq_info *qi __free(kfree) = NULL; 1428 struct idpf_chunked_msg_params params = { 1429 .vport_id = qs->vport_id, 1430 .vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES, 1431 .prepare_msg = idpf_prepare_cfg_txqs_msg, 1432 .config_sz = sizeof(struct virtchnl2_config_tx_queues), 1433 .chunk_sz = sizeof(*qi), 1434 }; 1435 1436 qi = kzalloc_objs(*qi, qs->num); 1437 if (!qi) 1438 return -ENOMEM; 1439 1440 params.chunks = qi; 1441 1442 for (u32 i = 0; i < qs->num; i++) { 1443 if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX) 1444 idpf_fill_txq_config_chunk(qs->qv_rsrc, qs->qs[i].txq, 1445 &qi[params.num_chunks++]); 1446 else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION) 1447 idpf_fill_complq_config_chunk(qs->qv_rsrc, 1448 qs->qs[i].complq, 1449 &qi[params.num_chunks++]); 1450 } 1451 1452 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1453 } 1454 1455 /** 1456 * idpf_send_config_tx_queues_msg - send virtchnl config Tx queues message 1457 * @adapter: adapter pointer used to send virtchnl message 1458 * @rsrc: pointer to queue and vector resources 1459 * @vport_id: vport identifier used while preparing the virtchnl message 1460 * 1461 * Return: 0 on success, -errno on failure. 1462 */ 1463 static int idpf_send_config_tx_queues_msg(struct idpf_adapter *adapter, 1464 struct idpf_q_vec_rsrc *rsrc, 1465 u32 vport_id) 1466 { 1467 struct idpf_queue_set *qs __free(kfree) = NULL; 1468 u32 totqs = rsrc->num_txq + rsrc->num_complq; 1469 u32 k = 0; 1470 1471 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, totqs); 1472 if (!qs) 1473 return -ENOMEM; 1474 1475 /* Populate the queue info buffer with all queue context info */ 1476 for (u32 i = 0; i < rsrc->num_txq_grp; i++) { 1477 const struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 1478 1479 for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 1480 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 1481 qs->qs[k++].txq = tx_qgrp->txqs[j]; 1482 } 1483 1484 if (idpf_is_queue_model_split(rsrc->txq_model)) { 1485 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1486 qs->qs[k++].complq = tx_qgrp->complq; 1487 } 1488 } 1489 1490 /* Make sure accounting agrees */ 1491 if (k != totqs) 1492 return -EINVAL; 1493 1494 return idpf_send_config_tx_queue_set_msg(qs); 1495 } 1496 1497 /** 1498 * idpf_fill_rxq_config_chunk - fill chunk describing the Rx queue 1499 * @rsrc: pointer to queue and vector resources 1500 * @q: Rx queue to be inserted into VC chunk 1501 * @qi: pointer to the buffer containing the VC chunk 1502 */ 1503 static void idpf_fill_rxq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1504 struct idpf_rx_queue *q, 1505 struct virtchnl2_rxq_info *qi) 1506 { 1507 const struct idpf_bufq_set *sets; 1508 1509 qi->queue_id = cpu_to_le32(q->q_id); 1510 qi->model = cpu_to_le16(rsrc->rxq_model); 1511 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 1512 qi->ring_len = cpu_to_le16(q->desc_count); 1513 qi->dma_ring_addr = cpu_to_le64(q->dma); 1514 qi->max_pkt_size = cpu_to_le32(q->rx_max_pkt_size); 1515 qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); 1516 qi->qflags = cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); 1517 if (idpf_queue_has(RSC_EN, q)) 1518 qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1519 1520 if (!idpf_is_queue_model_split(rsrc->rxq_model)) { 1521 qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1522 qi->desc_ids = cpu_to_le64(q->rxdids); 1523 1524 return; 1525 } 1526 1527 sets = q->bufq_sets; 1528 1529 /* 1530 * In splitq mode, RxQ buffer size should be set to that of the first 1531 * buffer queue associated with this RxQ. 1532 */ 1533 q->rx_buf_size = sets[0].bufq.rx_buf_size; 1534 qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1535 1536 qi->rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); 1537 if (rsrc->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { 1538 qi->bufq2_ena = IDPF_BUFQ2_ENA; 1539 qi->rx_bufq2_id = cpu_to_le16(sets[1].bufq.q_id); 1540 } 1541 1542 q->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; 1543 1544 if (idpf_queue_has(HSPLIT_EN, q)) { 1545 qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1546 qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); 1547 } 1548 1549 qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1550 } 1551 1552 /** 1553 * idpf_fill_bufq_config_chunk - fill chunk describing the buffer queue 1554 * @rsrc: pointer to queue and vector resources 1555 * @q: buffer queue to be inserted into VC chunk 1556 * @qi: pointer to the buffer containing the VC chunk 1557 */ 1558 static void idpf_fill_bufq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1559 const struct idpf_buf_queue *q, 1560 struct virtchnl2_rxq_info *qi) 1561 { 1562 qi->queue_id = cpu_to_le32(q->q_id); 1563 qi->model = cpu_to_le16(rsrc->rxq_model); 1564 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 1565 qi->ring_len = cpu_to_le16(q->desc_count); 1566 qi->dma_ring_addr = cpu_to_le64(q->dma); 1567 qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1568 qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); 1569 qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1570 qi->buffer_notif_stride = IDPF_RX_BUF_STRIDE; 1571 if (idpf_queue_has(RSC_EN, q)) 1572 qi->qflags = cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1573 1574 if (idpf_queue_has(HSPLIT_EN, q)) { 1575 qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1576 qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); 1577 } 1578 } 1579 1580 /** 1581 * idpf_prepare_cfg_rxqs_msg - prepare message to configure selected Rx queues 1582 * @vport_id: ID of virtual port queues are associated with 1583 * @buf: buffer containing the message 1584 * @pos: pointer to the first chunk describing the rx queue 1585 * @num_chunks: number of chunks in the message 1586 * 1587 * Helper function for preparing the message describing configuration of 1588 * Rx queues. 1589 * 1590 * Return: the total size of the prepared message. 1591 */ 1592 static u32 idpf_prepare_cfg_rxqs_msg(u32 vport_id, void *buf, const void *pos, 1593 u32 num_chunks) 1594 { 1595 struct virtchnl2_config_rx_queues *crq = buf; 1596 1597 crq->vport_id = cpu_to_le32(vport_id); 1598 crq->num_qinfo = cpu_to_le16(num_chunks); 1599 memcpy(crq->qinfo, pos, num_chunks * sizeof(*crq->qinfo)); 1600 1601 return struct_size(crq, qinfo, num_chunks); 1602 } 1603 1604 /** 1605 * idpf_send_config_rx_queue_set_msg - send virtchnl config Rx queues message 1606 * for selected queues. 1607 * @qs: set of the Rx queues to configure 1608 * 1609 * Send config queues virtchnl message for queues contained in the @qs array. 1610 * The @qs array can contain Rx queues (or buffer queues) only. 1611 * 1612 * Return: 0 on success, -errno on failure. 1613 */ 1614 static int idpf_send_config_rx_queue_set_msg(const struct idpf_queue_set *qs) 1615 { 1616 struct virtchnl2_rxq_info *qi __free(kfree) = NULL; 1617 struct idpf_chunked_msg_params params = { 1618 .vport_id = qs->vport_id, 1619 .vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES, 1620 .prepare_msg = idpf_prepare_cfg_rxqs_msg, 1621 .config_sz = sizeof(struct virtchnl2_config_rx_queues), 1622 .chunk_sz = sizeof(*qi), 1623 }; 1624 1625 qi = kzalloc_objs(*qi, qs->num); 1626 if (!qi) 1627 return -ENOMEM; 1628 1629 params.chunks = qi; 1630 1631 for (u32 i = 0; i < qs->num; i++) { 1632 if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX) 1633 idpf_fill_rxq_config_chunk(qs->qv_rsrc, qs->qs[i].rxq, 1634 &qi[params.num_chunks++]); 1635 else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX_BUFFER) 1636 idpf_fill_bufq_config_chunk(qs->qv_rsrc, qs->qs[i].bufq, 1637 &qi[params.num_chunks++]); 1638 } 1639 1640 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1641 } 1642 1643 /** 1644 * idpf_send_config_rx_queues_msg - send virtchnl config Rx queues message 1645 * @adapter: adapter pointer used to send virtchnl message 1646 * @rsrc: pointer to queue and vector resources 1647 * @vport_id: vport identifier used while preparing the virtchnl message 1648 * 1649 * Return: 0 on success, -errno on failure. 1650 */ 1651 static int idpf_send_config_rx_queues_msg(struct idpf_adapter *adapter, 1652 struct idpf_q_vec_rsrc *rsrc, 1653 u32 vport_id) 1654 { 1655 bool splitq = idpf_is_queue_model_split(rsrc->rxq_model); 1656 struct idpf_queue_set *qs __free(kfree) = NULL; 1657 u32 totqs = rsrc->num_rxq + rsrc->num_bufq; 1658 u32 k = 0; 1659 1660 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, totqs); 1661 if (!qs) 1662 return -ENOMEM; 1663 1664 /* Populate the queue info buffer with all queue context info */ 1665 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 1666 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1667 u32 num_rxq; 1668 1669 if (!splitq) { 1670 num_rxq = rx_qgrp->singleq.num_rxq; 1671 goto rxq; 1672 } 1673 1674 for (u32 j = 0; j < rsrc->num_bufqs_per_qgrp; j++) { 1675 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1676 qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 1677 } 1678 1679 num_rxq = rx_qgrp->splitq.num_rxq_sets; 1680 1681 rxq: 1682 for (u32 j = 0; j < num_rxq; j++) { 1683 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 1684 1685 if (splitq) 1686 qs->qs[k++].rxq = 1687 &rx_qgrp->splitq.rxq_sets[j]->rxq; 1688 else 1689 qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 1690 } 1691 } 1692 1693 /* Make sure accounting agrees */ 1694 if (k != totqs) 1695 return -EINVAL; 1696 1697 return idpf_send_config_rx_queue_set_msg(qs); 1698 } 1699 1700 /** 1701 * idpf_prepare_ena_dis_qs_msg - prepare message to enable/disable selected 1702 * queues 1703 * @vport_id: ID of virtual port queues are associated with 1704 * @buf: buffer containing the message 1705 * @pos: pointer to the first chunk describing the queue 1706 * @num_chunks: number of chunks in the message 1707 * 1708 * Helper function for preparing the message describing queues to be enabled 1709 * or disabled. 1710 * 1711 * Return: the total size of the prepared message. 1712 */ 1713 static u32 idpf_prepare_ena_dis_qs_msg(u32 vport_id, void *buf, const void *pos, 1714 u32 num_chunks) 1715 { 1716 struct virtchnl2_del_ena_dis_queues *eq = buf; 1717 1718 eq->vport_id = cpu_to_le32(vport_id); 1719 eq->chunks.num_chunks = cpu_to_le16(num_chunks); 1720 memcpy(eq->chunks.chunks, pos, 1721 num_chunks * sizeof(*eq->chunks.chunks)); 1722 1723 return struct_size(eq, chunks.chunks, num_chunks); 1724 } 1725 1726 /** 1727 * idpf_send_ena_dis_queue_set_msg - send virtchnl enable or disable queues 1728 * message for selected queues 1729 * @qs: set of the queues to enable or disable 1730 * @en: whether to enable or disable queues 1731 * 1732 * Send enable or disable queues virtchnl message for queues contained 1733 * in the @qs array. 1734 * The @qs array can contain pointers to both Rx and Tx queues. 1735 * 1736 * Return: 0 on success, -errno on failure. 1737 */ 1738 static int idpf_send_ena_dis_queue_set_msg(const struct idpf_queue_set *qs, 1739 bool en) 1740 { 1741 struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; 1742 struct idpf_chunked_msg_params params = { 1743 .vport_id = qs->vport_id, 1744 .vc_op = en ? VIRTCHNL2_OP_ENABLE_QUEUES : 1745 VIRTCHNL2_OP_DISABLE_QUEUES, 1746 .prepare_msg = idpf_prepare_ena_dis_qs_msg, 1747 .config_sz = sizeof(struct virtchnl2_del_ena_dis_queues), 1748 .chunk_sz = sizeof(*qc), 1749 .num_chunks = qs->num, 1750 }; 1751 1752 qc = kzalloc_objs(*qc, qs->num); 1753 if (!qc) 1754 return -ENOMEM; 1755 1756 params.chunks = qc; 1757 1758 for (u32 i = 0; i < qs->num; i++) { 1759 const struct idpf_queue_ptr *q = &qs->qs[i]; 1760 u32 qid; 1761 1762 qc[i].type = cpu_to_le32(q->type); 1763 qc[i].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 1764 1765 switch (q->type) { 1766 case VIRTCHNL2_QUEUE_TYPE_RX: 1767 qid = q->rxq->q_id; 1768 break; 1769 case VIRTCHNL2_QUEUE_TYPE_TX: 1770 qid = q->txq->q_id; 1771 break; 1772 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1773 qid = q->bufq->q_id; 1774 break; 1775 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 1776 qid = q->complq->q_id; 1777 break; 1778 default: 1779 return -EINVAL; 1780 } 1781 1782 qc[i].start_queue_id = cpu_to_le32(qid); 1783 } 1784 1785 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1786 } 1787 1788 /** 1789 * idpf_send_ena_dis_queues_msg - send virtchnl enable or disable queues 1790 * message 1791 * @adapter: adapter pointer used to send virtchnl message 1792 * @rsrc: pointer to queue and vector resources 1793 * @vport_id: vport identifier used while preparing the virtchnl message 1794 * @en: whether to enable or disable queues 1795 * 1796 * Return: 0 on success, -errno on failure. 1797 */ 1798 static int idpf_send_ena_dis_queues_msg(struct idpf_adapter *adapter, 1799 struct idpf_q_vec_rsrc *rsrc, 1800 u32 vport_id, bool en) 1801 { 1802 struct idpf_queue_set *qs __free(kfree) = NULL; 1803 u32 num_txq, num_q, k = 0; 1804 bool split; 1805 1806 num_txq = rsrc->num_txq + rsrc->num_complq; 1807 num_q = num_txq + rsrc->num_rxq + rsrc->num_bufq; 1808 1809 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, num_q); 1810 if (!qs) 1811 return -ENOMEM; 1812 1813 split = idpf_is_queue_model_split(rsrc->txq_model); 1814 1815 for (u32 i = 0; i < rsrc->num_txq_grp; i++) { 1816 const struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 1817 1818 for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 1819 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 1820 qs->qs[k++].txq = tx_qgrp->txqs[j]; 1821 } 1822 1823 if (!split) 1824 continue; 1825 1826 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1827 qs->qs[k++].complq = tx_qgrp->complq; 1828 } 1829 1830 if (k != num_txq) 1831 return -EINVAL; 1832 1833 split = idpf_is_queue_model_split(rsrc->rxq_model); 1834 1835 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 1836 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1837 u32 num_rxq; 1838 1839 if (split) 1840 num_rxq = rx_qgrp->splitq.num_rxq_sets; 1841 else 1842 num_rxq = rx_qgrp->singleq.num_rxq; 1843 1844 for (u32 j = 0; j < num_rxq; j++) { 1845 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 1846 1847 if (split) 1848 qs->qs[k++].rxq = 1849 &rx_qgrp->splitq.rxq_sets[j]->rxq; 1850 else 1851 qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 1852 } 1853 1854 if (!split) 1855 continue; 1856 1857 for (u32 j = 0; j < rsrc->num_bufqs_per_qgrp; j++) { 1858 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1859 qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 1860 } 1861 } 1862 1863 if (k != num_q) 1864 return -EINVAL; 1865 1866 return idpf_send_ena_dis_queue_set_msg(qs, en); 1867 } 1868 1869 /** 1870 * idpf_prep_map_unmap_queue_set_vector_msg - prepare message to map or unmap 1871 * queue set to the interrupt vector 1872 * @vport_id: ID of virtual port queues are associated with 1873 * @buf: buffer containing the message 1874 * @pos: pointer to the first chunk describing the vector mapping 1875 * @num_chunks: number of chunks in the message 1876 * 1877 * Helper function for preparing the message describing mapping queues to 1878 * q_vectors. 1879 * 1880 * Return: the total size of the prepared message. 1881 */ 1882 static u32 1883 idpf_prep_map_unmap_queue_set_vector_msg(u32 vport_id, void *buf, 1884 const void *pos, u32 num_chunks) 1885 { 1886 struct virtchnl2_queue_vector_maps *vqvm = buf; 1887 1888 vqvm->vport_id = cpu_to_le32(vport_id); 1889 vqvm->num_qv_maps = cpu_to_le16(num_chunks); 1890 memcpy(vqvm->qv_maps, pos, num_chunks * sizeof(*vqvm->qv_maps)); 1891 1892 return struct_size(vqvm, qv_maps, num_chunks); 1893 } 1894 1895 /** 1896 * idpf_send_map_unmap_queue_set_vector_msg - send virtchnl map or unmap 1897 * queue set vector message 1898 * @qs: set of the queues to map or unmap 1899 * @map: true for map and false for unmap 1900 * 1901 * Return: 0 on success, -errno on failure. 1902 */ 1903 static int 1904 idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, 1905 bool map) 1906 { 1907 struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; 1908 struct idpf_chunked_msg_params params = { 1909 .vport_id = qs->vport_id, 1910 .vc_op = map ? VIRTCHNL2_OP_MAP_QUEUE_VECTOR : 1911 VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR, 1912 .prepare_msg = idpf_prep_map_unmap_queue_set_vector_msg, 1913 .config_sz = sizeof(struct virtchnl2_queue_vector_maps), 1914 .chunk_sz = sizeof(*vqv), 1915 .num_chunks = qs->num, 1916 }; 1917 bool split; 1918 1919 vqv = kzalloc_objs(*vqv, qs->num); 1920 if (!vqv) 1921 return -ENOMEM; 1922 1923 params.chunks = vqv; 1924 1925 split = idpf_is_queue_model_split(qs->qv_rsrc->txq_model); 1926 1927 for (u32 i = 0; i < qs->num; i++) { 1928 const struct idpf_queue_ptr *q = &qs->qs[i]; 1929 const struct idpf_q_vector *vec; 1930 u32 qid, v_idx, itr_idx; 1931 1932 vqv[i].queue_type = cpu_to_le32(q->type); 1933 1934 switch (q->type) { 1935 case VIRTCHNL2_QUEUE_TYPE_RX: 1936 qid = q->rxq->q_id; 1937 1938 if (idpf_queue_has(NOIRQ, q->rxq)) 1939 vec = NULL; 1940 else 1941 vec = q->rxq->q_vector; 1942 1943 if (vec) { 1944 v_idx = vec->v_idx; 1945 itr_idx = vec->rx_itr_idx; 1946 } else { 1947 v_idx = qs->qv_rsrc->noirq_v_idx; 1948 itr_idx = VIRTCHNL2_ITR_IDX_0; 1949 } 1950 break; 1951 case VIRTCHNL2_QUEUE_TYPE_TX: 1952 qid = q->txq->q_id; 1953 1954 if (idpf_queue_has(NOIRQ, q->txq)) 1955 vec = NULL; 1956 else if (idpf_queue_has(XDP, q->txq)) 1957 vec = q->txq->complq->q_vector; 1958 else if (split) 1959 vec = q->txq->txq_grp->complq->q_vector; 1960 else 1961 vec = q->txq->q_vector; 1962 1963 if (vec) { 1964 v_idx = vec->v_idx; 1965 itr_idx = vec->tx_itr_idx; 1966 } else { 1967 v_idx = qs->qv_rsrc->noirq_v_idx; 1968 itr_idx = VIRTCHNL2_ITR_IDX_1; 1969 } 1970 break; 1971 default: 1972 return -EINVAL; 1973 } 1974 1975 vqv[i].queue_id = cpu_to_le32(qid); 1976 vqv[i].vector_id = cpu_to_le16(v_idx); 1977 vqv[i].itr_idx = cpu_to_le32(itr_idx); 1978 } 1979 1980 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1981 } 1982 1983 /** 1984 * idpf_send_map_unmap_queue_vector_msg - send virtchnl map or unmap queue 1985 * vector message 1986 * @adapter: adapter pointer used to send virtchnl message 1987 * @rsrc: pointer to queue and vector resources 1988 * @vport_id: vport identifier used while preparing the virtchnl message 1989 * @map: true for map and false for unmap 1990 * 1991 * Return: 0 on success, -errno on failure. 1992 */ 1993 int idpf_send_map_unmap_queue_vector_msg(struct idpf_adapter *adapter, 1994 struct idpf_q_vec_rsrc *rsrc, 1995 u32 vport_id, bool map) 1996 { 1997 struct idpf_queue_set *qs __free(kfree) = NULL; 1998 u32 num_q = rsrc->num_txq + rsrc->num_rxq; 1999 u32 k = 0; 2000 2001 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, num_q); 2002 if (!qs) 2003 return -ENOMEM; 2004 2005 for (u32 i = 0; i < rsrc->num_txq_grp; i++) { 2006 const struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 2007 2008 for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 2009 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 2010 qs->qs[k++].txq = tx_qgrp->txqs[j]; 2011 } 2012 } 2013 2014 if (k != rsrc->num_txq) 2015 return -EINVAL; 2016 2017 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 2018 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 2019 u32 num_rxq; 2020 2021 if (idpf_is_queue_model_split(rsrc->rxq_model)) 2022 num_rxq = rx_qgrp->splitq.num_rxq_sets; 2023 else 2024 num_rxq = rx_qgrp->singleq.num_rxq; 2025 2026 for (u32 j = 0; j < num_rxq; j++) { 2027 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 2028 2029 if (idpf_is_queue_model_split(rsrc->rxq_model)) 2030 qs->qs[k++].rxq = 2031 &rx_qgrp->splitq.rxq_sets[j]->rxq; 2032 else 2033 qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 2034 } 2035 } 2036 2037 if (k != num_q) 2038 return -EINVAL; 2039 2040 return idpf_send_map_unmap_queue_set_vector_msg(qs, map); 2041 } 2042 2043 /** 2044 * idpf_send_enable_queue_set_msg - send enable queues virtchnl message for 2045 * selected queues 2046 * @qs: set of the queues 2047 * 2048 * Send enable queues virtchnl message for queues contained in the @qs array. 2049 * 2050 * Return: 0 on success, -errno on failure. 2051 */ 2052 int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs) 2053 { 2054 return idpf_send_ena_dis_queue_set_msg(qs, true); 2055 } 2056 2057 /** 2058 * idpf_send_disable_queue_set_msg - send disable queues virtchnl message for 2059 * selected queues 2060 * @qs: set of the queues 2061 * 2062 * Return: 0 on success, -errno on failure. 2063 */ 2064 int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs) 2065 { 2066 int err; 2067 2068 err = idpf_send_ena_dis_queue_set_msg(qs, false); 2069 if (err) 2070 return err; 2071 2072 return idpf_wait_for_marker_event_set(qs); 2073 } 2074 2075 /** 2076 * idpf_send_config_queue_set_msg - send virtchnl config queues message for 2077 * selected queues 2078 * @qs: set of the queues 2079 * 2080 * Send config queues virtchnl message for queues contained in the @qs array. 2081 * The @qs array can contain both Rx or Tx queues. 2082 * 2083 * Return: 0 on success, -errno on failure. 2084 */ 2085 int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs) 2086 { 2087 int err; 2088 2089 err = idpf_send_config_tx_queue_set_msg(qs); 2090 if (err) 2091 return err; 2092 2093 return idpf_send_config_rx_queue_set_msg(qs); 2094 } 2095 2096 /** 2097 * idpf_send_enable_queues_msg - send enable queues virtchnl message 2098 * @vport: Virtual port private data structure 2099 * 2100 * Will send enable queues virtchnl message. Returns 0 on success, negative on 2101 * failure. 2102 */ 2103 int idpf_send_enable_queues_msg(struct idpf_vport *vport) 2104 { 2105 return idpf_send_ena_dis_queues_msg(vport->adapter, 2106 &vport->dflt_qv_rsrc, 2107 vport->vport_id, true); 2108 } 2109 2110 /** 2111 * idpf_send_disable_queues_msg - send disable queues virtchnl message 2112 * @vport: Virtual port private data structure 2113 * 2114 * Will send disable queues virtchnl message. Returns 0 on success, negative 2115 * on failure. 2116 */ 2117 int idpf_send_disable_queues_msg(struct idpf_vport *vport) 2118 { 2119 int err; 2120 2121 err = idpf_send_ena_dis_queues_msg(vport->adapter, 2122 &vport->dflt_qv_rsrc, 2123 vport->vport_id, false); 2124 if (err) 2125 return err; 2126 2127 return idpf_wait_for_marker_event(vport); 2128 } 2129 2130 /** 2131 * idpf_convert_reg_to_queue_chunks - Copy queue chunk information to the right 2132 * structure 2133 * @dchunks: Destination chunks to store data to 2134 * @schunks: Source chunks to copy data from 2135 * @num_chunks: number of chunks to copy 2136 */ 2137 static void idpf_convert_reg_to_queue_chunks(struct virtchnl2_queue_chunk *dchunks, 2138 struct idpf_queue_id_reg_chunk *schunks, 2139 u16 num_chunks) 2140 { 2141 u16 i; 2142 2143 for (i = 0; i < num_chunks; i++) { 2144 dchunks[i].type = cpu_to_le32(schunks[i].type); 2145 dchunks[i].start_queue_id = cpu_to_le32(schunks[i].start_queue_id); 2146 dchunks[i].num_queues = cpu_to_le32(schunks[i].num_queues); 2147 } 2148 } 2149 2150 /** 2151 * idpf_send_delete_queues_msg - send delete queues virtchnl message 2152 * @adapter: adapter pointer used to send virtchnl message 2153 * @chunks: queue ids received over mailbox 2154 * @vport_id: vport identifier used while preparing the virtchnl message 2155 * 2156 * Return: 0 on success, negative on failure. 2157 */ 2158 int idpf_send_delete_queues_msg(struct idpf_adapter *adapter, 2159 struct idpf_queue_id_reg_info *chunks, 2160 u32 vport_id) 2161 { 2162 struct libie_ctlq_xn_send_params xn_params = { 2163 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2164 .chnl_opcode = VIRTCHNL2_OP_DEL_QUEUES, 2165 }; 2166 struct virtchnl2_del_ena_dis_queues *eq; 2167 ssize_t buf_size; 2168 u16 num_chunks; 2169 int err; 2170 2171 num_chunks = chunks->num_chunks; 2172 buf_size = struct_size(eq, chunks.chunks, num_chunks); 2173 2174 eq = kzalloc(buf_size, GFP_KERNEL); 2175 if (!eq) 2176 return -ENOMEM; 2177 2178 eq->vport_id = cpu_to_le32(vport_id); 2179 eq->chunks.num_chunks = cpu_to_le16(num_chunks); 2180 2181 idpf_convert_reg_to_queue_chunks(eq->chunks.chunks, chunks->queue_chunks, 2182 num_chunks); 2183 2184 err = idpf_send_mb_msg_kfree(adapter, &xn_params, eq, buf_size); 2185 if (err) 2186 return err; 2187 2188 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2189 2190 return 0; 2191 } 2192 2193 /** 2194 * idpf_send_config_queues_msg - Send config queues virtchnl message 2195 * @adapter: adapter pointer used to send virtchnl message 2196 * @rsrc: pointer to queue and vector resources 2197 * @vport_id: vport identifier used while preparing the virtchnl message 2198 * 2199 * Return: 0 on success, negative on failure. 2200 */ 2201 int idpf_send_config_queues_msg(struct idpf_adapter *adapter, 2202 struct idpf_q_vec_rsrc *rsrc, 2203 u32 vport_id) 2204 { 2205 int err; 2206 2207 err = idpf_send_config_tx_queues_msg(adapter, rsrc, vport_id); 2208 if (err) 2209 return err; 2210 2211 return idpf_send_config_rx_queues_msg(adapter, rsrc, vport_id); 2212 } 2213 2214 /** 2215 * idpf_send_add_queues_msg - Send virtchnl add queues message 2216 * @adapter: adapter pointer used to send virtchnl message 2217 * @vport_config: vport persistent structure to store the queue chunk info 2218 * @rsrc: pointer to queue and vector resources 2219 * @vport_id: vport identifier used while preparing the virtchnl message 2220 * 2221 * Return: 0 on success, negative on failure. 2222 */ 2223 int idpf_send_add_queues_msg(struct idpf_adapter *adapter, 2224 struct idpf_vport_config *vport_config, 2225 struct idpf_q_vec_rsrc *rsrc, 2226 u32 vport_id) 2227 { 2228 struct libie_ctlq_xn_send_params xn_params = { 2229 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2230 .chnl_opcode = VIRTCHNL2_OP_ADD_QUEUES, 2231 }; 2232 struct virtchnl2_add_queues *vc_msg; 2233 struct virtchnl2_add_queues aq = {}; 2234 size_t size; 2235 int err; 2236 2237 aq.vport_id = cpu_to_le32(vport_id); 2238 aq.num_tx_q = cpu_to_le16(rsrc->num_txq); 2239 aq.num_tx_complq = cpu_to_le16(rsrc->num_complq); 2240 aq.num_rx_q = cpu_to_le16(rsrc->num_rxq); 2241 aq.num_rx_bufq = cpu_to_le16(rsrc->num_bufq); 2242 2243 err = idpf_send_mb_msg_stack(adapter, &xn_params, &aq); 2244 if (err) 2245 return err; 2246 2247 vc_msg = xn_params.recv_mem.iov_base; 2248 if (xn_params.recv_mem.iov_len < sizeof(*vc_msg)) { 2249 err = -EIO; 2250 goto free_rx_buf; 2251 } 2252 2253 /* compare vc_msg num queues with vport num queues */ 2254 if (le16_to_cpu(vc_msg->num_tx_q) != rsrc->num_txq || 2255 le16_to_cpu(vc_msg->num_rx_q) != rsrc->num_rxq || 2256 le16_to_cpu(vc_msg->num_tx_complq) != rsrc->num_complq || 2257 le16_to_cpu(vc_msg->num_rx_bufq) != rsrc->num_bufq) { 2258 err = -EINVAL; 2259 goto free_rx_buf; 2260 } 2261 2262 size = struct_size(vc_msg, chunks.chunks, 2263 le16_to_cpu(vc_msg->chunks.num_chunks)); 2264 if (xn_params.recv_mem.iov_len < size) { 2265 err = -EIO; 2266 goto free_rx_buf; 2267 } 2268 2269 err = idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks); 2270 2271 free_rx_buf: 2272 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2273 2274 return err; 2275 } 2276 2277 /** 2278 * idpf_send_alloc_vectors_msg - Send virtchnl alloc vectors message 2279 * @adapter: Driver specific private structure 2280 * @num_vectors: number of vectors to be allocated 2281 * 2282 * Returns 0 on success, negative on failure. 2283 */ 2284 int idpf_send_alloc_vectors_msg(struct idpf_adapter *adapter, u16 num_vectors) 2285 { 2286 struct libie_ctlq_xn_send_params xn_params = { 2287 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2288 .chnl_opcode = VIRTCHNL2_OP_ALLOC_VECTORS, 2289 }; 2290 struct virtchnl2_alloc_vectors *rcvd_vec; 2291 struct virtchnl2_alloc_vectors ac = {}; 2292 u16 num_vchunks; 2293 int size, err; 2294 2295 ac.num_vectors = cpu_to_le16(num_vectors); 2296 2297 err = idpf_send_mb_msg_stack(adapter, &xn_params, &ac); 2298 if (err) 2299 return err; 2300 2301 rcvd_vec = xn_params.recv_mem.iov_base; 2302 if (xn_params.recv_mem.iov_len < sizeof(*rcvd_vec)) { 2303 err = -EIO; 2304 goto free_rx_buf; 2305 } 2306 2307 num_vchunks = le16_to_cpu(rcvd_vec->vchunks.num_vchunks); 2308 size = struct_size(rcvd_vec, vchunks.vchunks, num_vchunks); 2309 if (xn_params.recv_mem.iov_len < size) { 2310 err = -EIO; 2311 goto free_rx_buf; 2312 } 2313 2314 kfree(adapter->req_vec_chunks); 2315 adapter->req_vec_chunks = kmemdup(rcvd_vec, size, GFP_KERNEL); 2316 if (!adapter->req_vec_chunks) { 2317 err = -ENOMEM; 2318 goto free_rx_buf; 2319 } 2320 2321 if (le16_to_cpu(adapter->req_vec_chunks->num_vectors) < num_vectors) { 2322 kfree(adapter->req_vec_chunks); 2323 adapter->req_vec_chunks = NULL; 2324 err = -EINVAL; 2325 } 2326 2327 free_rx_buf: 2328 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2329 2330 return err; 2331 } 2332 2333 /** 2334 * idpf_send_dealloc_vectors_msg - Send virtchnl de allocate vectors message 2335 * @adapter: Driver specific private structure 2336 * 2337 * Returns 0 on success, negative on failure. 2338 */ 2339 int idpf_send_dealloc_vectors_msg(struct idpf_adapter *adapter) 2340 { 2341 struct virtchnl2_alloc_vectors *ac = adapter->req_vec_chunks; 2342 struct libie_ctlq_xn_send_params xn_params = { 2343 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2344 .chnl_opcode = VIRTCHNL2_OP_DEALLOC_VECTORS, 2345 }; 2346 struct virtchnl2_vector_chunks *vcs; 2347 int buf_size, err; 2348 2349 buf_size = struct_size(&ac->vchunks, vchunks, 2350 le16_to_cpu(ac->vchunks.num_vchunks)); 2351 vcs = kmemdup(&ac->vchunks, buf_size, GFP_KERNEL); 2352 if (!vcs) 2353 return -ENOMEM; 2354 2355 err = idpf_send_mb_msg_kfree(adapter, &xn_params, vcs, buf_size); 2356 if (err) 2357 return err; 2358 2359 kfree(adapter->req_vec_chunks); 2360 adapter->req_vec_chunks = NULL; 2361 2362 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2363 2364 return 0; 2365 } 2366 2367 /** 2368 * idpf_get_max_vfs - Get max number of vfs supported 2369 * @adapter: Driver specific private structure 2370 * 2371 * Returns max number of VFs 2372 */ 2373 static int idpf_get_max_vfs(struct idpf_adapter *adapter) 2374 { 2375 return le16_to_cpu(adapter->caps.max_sriov_vfs); 2376 } 2377 2378 /** 2379 * idpf_send_set_sriov_vfs_msg - Send virtchnl set sriov vfs message 2380 * @adapter: Driver specific private structure 2381 * @num_vfs: number of virtual functions to be created 2382 * 2383 * Returns 0 on success, negative on failure. 2384 */ 2385 int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs) 2386 { 2387 struct libie_ctlq_xn_send_params xn_params = { 2388 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2389 .chnl_opcode = VIRTCHNL2_OP_SET_SRIOV_VFS, 2390 }; 2391 struct virtchnl2_sriov_vfs_info svi = {}; 2392 int err; 2393 2394 svi.num_vfs = cpu_to_le16(num_vfs); 2395 2396 err = idpf_send_mb_msg_stack(adapter, &xn_params, &svi); 2397 if (err) 2398 return err; 2399 2400 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2401 2402 return 0; 2403 } 2404 2405 /** 2406 * idpf_send_get_stats_msg - Send virtchnl get statistics message 2407 * @np: netdev private structure 2408 * @port_stats: structure to store the vport statistics 2409 * 2410 * Return: 0 on success, negative on failure. 2411 */ 2412 int idpf_send_get_stats_msg(struct idpf_netdev_priv *np, 2413 struct idpf_port_stats *port_stats) 2414 { 2415 struct libie_ctlq_xn_send_params xn_params = { 2416 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2417 .chnl_opcode = VIRTCHNL2_OP_GET_STATS, 2418 }; 2419 struct rtnl_link_stats64 *netstats = &np->netstats; 2420 struct virtchnl2_vport_stats *stats_recv; 2421 struct virtchnl2_vport_stats stats_msg = {}; 2422 int err; 2423 2424 2425 /* Don't send get_stats message if the link is down */ 2426 if (!test_bit(IDPF_VPORT_UP, np->state)) 2427 return 0; 2428 2429 stats_msg.vport_id = cpu_to_le32(np->vport_id); 2430 2431 err = idpf_send_mb_msg_stack(np->adapter, &xn_params, &stats_msg); 2432 if (err) 2433 return err; 2434 2435 if (xn_params.recv_mem.iov_len < sizeof(*stats_recv)) { 2436 err = -EIO; 2437 goto free_rx_buf; 2438 } 2439 2440 stats_recv = xn_params.recv_mem.iov_base; 2441 2442 spin_lock_bh(&np->stats_lock); 2443 2444 netstats->rx_packets = le64_to_cpu(stats_recv->rx_unicast) + 2445 le64_to_cpu(stats_recv->rx_multicast) + 2446 le64_to_cpu(stats_recv->rx_broadcast); 2447 netstats->tx_packets = le64_to_cpu(stats_recv->tx_unicast) + 2448 le64_to_cpu(stats_recv->tx_multicast) + 2449 le64_to_cpu(stats_recv->tx_broadcast); 2450 netstats->rx_bytes = le64_to_cpu(stats_recv->rx_bytes); 2451 netstats->tx_bytes = le64_to_cpu(stats_recv->tx_bytes); 2452 netstats->rx_errors = le64_to_cpu(stats_recv->rx_errors); 2453 netstats->tx_errors = le64_to_cpu(stats_recv->tx_errors); 2454 netstats->rx_dropped = le64_to_cpu(stats_recv->rx_discards); 2455 netstats->tx_dropped = le64_to_cpu(stats_recv->tx_discards); 2456 2457 port_stats->vport_stats = *stats_recv; 2458 2459 spin_unlock_bh(&np->stats_lock); 2460 2461 free_rx_buf: 2462 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2463 2464 return err; 2465 } 2466 2467 /** 2468 * idpf_send_set_rss_lut_msg - Send virtchnl set RSS lut message 2469 * @adapter: adapter pointer used to send virtchnl message 2470 * @rss_data: pointer to RSS key and lut info 2471 * @vport_id: vport identifier used while preparing the virtchnl message 2472 * 2473 * When rxhash is disabled, RSS LUT will be configured with zeros. If rxhash 2474 * is enabled, the LUT values stored in driver's soft copy will be used to setup 2475 * the HW. 2476 * 2477 * Return: 0 on success, negative on failure. 2478 */ 2479 int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter, 2480 struct idpf_rss_data *rss_data, u32 vport_id) 2481 { 2482 struct libie_ctlq_xn_send_params xn_params = { 2483 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2484 .chnl_opcode = VIRTCHNL2_OP_SET_RSS_LUT, 2485 }; 2486 struct virtchnl2_rss_lut *rl; 2487 struct idpf_vport *vport; 2488 int buf_size, i, err; 2489 bool rxhash_ena; 2490 2491 vport = idpf_vid_to_vport(adapter, vport_id); 2492 if (!vport) 2493 return -EINVAL; 2494 2495 rxhash_ena = idpf_is_feature_ena(vport, NETIF_F_RXHASH); 2496 2497 buf_size = struct_size(rl, lut, rss_data->rss_lut_size); 2498 rl = kzalloc(buf_size, GFP_KERNEL); 2499 if (!rl) 2500 return -ENOMEM; 2501 2502 rl->vport_id = cpu_to_le32(vport_id); 2503 rl->lut_entries = cpu_to_le16(rss_data->rss_lut_size); 2504 for (i = 0; i < rss_data->rss_lut_size; i++) 2505 rl->lut[i] = rxhash_ena ? cpu_to_le32(rss_data->rss_lut[i]) : 0; 2506 2507 err = idpf_send_mb_msg_kfree(adapter, &xn_params, rl, buf_size); 2508 if (err) 2509 return err; 2510 2511 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2512 2513 return err; 2514 } 2515 2516 /** 2517 * idpf_send_set_rss_key_msg - Send virtchnl set RSS key message 2518 * @adapter: adapter pointer used to send virtchnl message 2519 * @rss_data: pointer to RSS key and lut info 2520 * @vport_id: vport identifier used while preparing the virtchnl message 2521 * 2522 * Return: 0 on success, negative on failure 2523 */ 2524 int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter, 2525 struct idpf_rss_data *rss_data, u32 vport_id) 2526 { 2527 struct libie_ctlq_xn_send_params xn_params = { 2528 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2529 .chnl_opcode = VIRTCHNL2_OP_SET_RSS_KEY, 2530 }; 2531 struct virtchnl2_rss_key *rk; 2532 int i, buf_size, err; 2533 2534 buf_size = struct_size(rk, key_flex, rss_data->rss_key_size); 2535 rk = kzalloc(buf_size, GFP_KERNEL); 2536 if (!rk) 2537 return -ENOMEM; 2538 2539 rk->vport_id = cpu_to_le32(vport_id); 2540 rk->key_len = cpu_to_le16(rss_data->rss_key_size); 2541 for (i = 0; i < rss_data->rss_key_size; i++) 2542 rk->key_flex[i] = rss_data->rss_key[i]; 2543 2544 err = idpf_send_mb_msg_kfree(adapter, &xn_params, rk, buf_size); 2545 if (err) 2546 return err; 2547 2548 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2549 2550 return err; 2551 } 2552 2553 /** 2554 * idpf_fill_ptype_lookup - Fill L3 specific fields in ptype lookup table 2555 * @ptype: ptype lookup table 2556 * @pstate: state machine for ptype lookup table 2557 * @ipv4: ipv4 or ipv6 2558 * @frag: fragmentation allowed 2559 * 2560 */ 2561 static void idpf_fill_ptype_lookup(struct libeth_rx_pt *ptype, 2562 struct idpf_ptype_state *pstate, 2563 bool ipv4, bool frag) 2564 { 2565 if (!pstate->outer_ip || !pstate->outer_frag) { 2566 pstate->outer_ip = true; 2567 2568 if (ipv4) 2569 ptype->outer_ip = LIBETH_RX_PT_OUTER_IPV4; 2570 else 2571 ptype->outer_ip = LIBETH_RX_PT_OUTER_IPV6; 2572 2573 if (frag) { 2574 ptype->outer_frag = LIBETH_RX_PT_FRAG; 2575 pstate->outer_frag = true; 2576 } 2577 } else { 2578 ptype->tunnel_type = LIBETH_RX_PT_TUNNEL_IP_IP; 2579 pstate->tunnel_state = IDPF_PTYPE_TUNNEL_IP; 2580 2581 if (ipv4) 2582 ptype->tunnel_end_prot = LIBETH_RX_PT_TUNNEL_END_IPV4; 2583 else 2584 ptype->tunnel_end_prot = LIBETH_RX_PT_TUNNEL_END_IPV6; 2585 2586 if (frag) 2587 ptype->tunnel_end_frag = LIBETH_RX_PT_FRAG; 2588 } 2589 } 2590 2591 static void idpf_finalize_ptype_lookup(struct libeth_rx_pt *ptype) 2592 { 2593 if (ptype->payload_layer == LIBETH_RX_PT_PAYLOAD_L2 && 2594 ptype->inner_prot) 2595 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_L4; 2596 else if (ptype->payload_layer == LIBETH_RX_PT_PAYLOAD_L2 && 2597 ptype->outer_ip) 2598 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_L3; 2599 else if (ptype->outer_ip == LIBETH_RX_PT_OUTER_L2) 2600 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_L2; 2601 else 2602 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_NONE; 2603 2604 libeth_rx_pt_gen_hash_type(ptype); 2605 } 2606 2607 /** 2608 * idpf_parse_protocol_ids - parse protocol IDs for a given packet type 2609 * @ptype: packet type to parse 2610 * @rx_pt: store the parsed packet type info into 2611 */ 2612 static void idpf_parse_protocol_ids(struct virtchnl2_ptype *ptype, 2613 struct libeth_rx_pt *rx_pt) 2614 { 2615 struct idpf_ptype_state pstate = {}; 2616 2617 for (u32 j = 0; j < ptype->proto_id_count; j++) { 2618 u16 id = le16_to_cpu(ptype->proto_id[j]); 2619 2620 switch (id) { 2621 case VIRTCHNL2_PROTO_HDR_GRE: 2622 if (pstate.tunnel_state == IDPF_PTYPE_TUNNEL_IP) { 2623 rx_pt->tunnel_type = 2624 LIBETH_RX_PT_TUNNEL_IP_GRENAT; 2625 pstate.tunnel_state |= 2626 IDPF_PTYPE_TUNNEL_IP_GRENAT; 2627 } 2628 break; 2629 case VIRTCHNL2_PROTO_HDR_MAC: 2630 rx_pt->outer_ip = LIBETH_RX_PT_OUTER_L2; 2631 if (pstate.tunnel_state == IDPF_TUN_IP_GRE) { 2632 rx_pt->tunnel_type = 2633 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC; 2634 pstate.tunnel_state |= 2635 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC; 2636 } 2637 break; 2638 case VIRTCHNL2_PROTO_HDR_IPV4: 2639 idpf_fill_ptype_lookup(rx_pt, &pstate, true, false); 2640 break; 2641 case VIRTCHNL2_PROTO_HDR_IPV6: 2642 idpf_fill_ptype_lookup(rx_pt, &pstate, false, false); 2643 break; 2644 case VIRTCHNL2_PROTO_HDR_IPV4_FRAG: 2645 idpf_fill_ptype_lookup(rx_pt, &pstate, true, true); 2646 break; 2647 case VIRTCHNL2_PROTO_HDR_IPV6_FRAG: 2648 idpf_fill_ptype_lookup(rx_pt, &pstate, false, true); 2649 break; 2650 case VIRTCHNL2_PROTO_HDR_UDP: 2651 rx_pt->inner_prot = LIBETH_RX_PT_INNER_UDP; 2652 break; 2653 case VIRTCHNL2_PROTO_HDR_TCP: 2654 rx_pt->inner_prot = LIBETH_RX_PT_INNER_TCP; 2655 break; 2656 case VIRTCHNL2_PROTO_HDR_SCTP: 2657 rx_pt->inner_prot = LIBETH_RX_PT_INNER_SCTP; 2658 break; 2659 case VIRTCHNL2_PROTO_HDR_ICMP: 2660 rx_pt->inner_prot = LIBETH_RX_PT_INNER_ICMP; 2661 break; 2662 case VIRTCHNL2_PROTO_HDR_PAY: 2663 rx_pt->payload_layer = LIBETH_RX_PT_PAYLOAD_L2; 2664 break; 2665 case VIRTCHNL2_PROTO_HDR_ICMPV6: 2666 case VIRTCHNL2_PROTO_HDR_IPV6_EH: 2667 case VIRTCHNL2_PROTO_HDR_PRE_MAC: 2668 case VIRTCHNL2_PROTO_HDR_POST_MAC: 2669 case VIRTCHNL2_PROTO_HDR_ETHERTYPE: 2670 case VIRTCHNL2_PROTO_HDR_SVLAN: 2671 case VIRTCHNL2_PROTO_HDR_CVLAN: 2672 case VIRTCHNL2_PROTO_HDR_MPLS: 2673 case VIRTCHNL2_PROTO_HDR_MMPLS: 2674 case VIRTCHNL2_PROTO_HDR_PTP: 2675 case VIRTCHNL2_PROTO_HDR_CTRL: 2676 case VIRTCHNL2_PROTO_HDR_LLDP: 2677 case VIRTCHNL2_PROTO_HDR_ARP: 2678 case VIRTCHNL2_PROTO_HDR_ECP: 2679 case VIRTCHNL2_PROTO_HDR_EAPOL: 2680 case VIRTCHNL2_PROTO_HDR_PPPOD: 2681 case VIRTCHNL2_PROTO_HDR_PPPOE: 2682 case VIRTCHNL2_PROTO_HDR_IGMP: 2683 case VIRTCHNL2_PROTO_HDR_AH: 2684 case VIRTCHNL2_PROTO_HDR_ESP: 2685 case VIRTCHNL2_PROTO_HDR_IKE: 2686 case VIRTCHNL2_PROTO_HDR_NATT_KEEP: 2687 case VIRTCHNL2_PROTO_HDR_L2TPV2: 2688 case VIRTCHNL2_PROTO_HDR_L2TPV2_CONTROL: 2689 case VIRTCHNL2_PROTO_HDR_L2TPV3: 2690 case VIRTCHNL2_PROTO_HDR_GTP: 2691 case VIRTCHNL2_PROTO_HDR_GTP_EH: 2692 case VIRTCHNL2_PROTO_HDR_GTPCV2: 2693 case VIRTCHNL2_PROTO_HDR_GTPC_TEID: 2694 case VIRTCHNL2_PROTO_HDR_GTPU: 2695 case VIRTCHNL2_PROTO_HDR_GTPU_UL: 2696 case VIRTCHNL2_PROTO_HDR_GTPU_DL: 2697 case VIRTCHNL2_PROTO_HDR_ECPRI: 2698 case VIRTCHNL2_PROTO_HDR_VRRP: 2699 case VIRTCHNL2_PROTO_HDR_OSPF: 2700 case VIRTCHNL2_PROTO_HDR_TUN: 2701 case VIRTCHNL2_PROTO_HDR_NVGRE: 2702 case VIRTCHNL2_PROTO_HDR_VXLAN: 2703 case VIRTCHNL2_PROTO_HDR_VXLAN_GPE: 2704 case VIRTCHNL2_PROTO_HDR_GENEVE: 2705 case VIRTCHNL2_PROTO_HDR_NSH: 2706 case VIRTCHNL2_PROTO_HDR_QUIC: 2707 case VIRTCHNL2_PROTO_HDR_PFCP: 2708 case VIRTCHNL2_PROTO_HDR_PFCP_NODE: 2709 case VIRTCHNL2_PROTO_HDR_PFCP_SESSION: 2710 case VIRTCHNL2_PROTO_HDR_RTP: 2711 case VIRTCHNL2_PROTO_HDR_NO_PROTO: 2712 break; 2713 default: 2714 break; 2715 } 2716 } 2717 } 2718 2719 /** 2720 * idpf_send_get_rx_ptype_msg - Send virtchnl for ptype info 2721 * @adapter: driver specific private structure 2722 * 2723 * Return: 0 on success, negative on failure. 2724 */ 2725 static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter) 2726 { 2727 struct libie_ctlq_xn_send_params xn_params = { 2728 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2729 .chnl_opcode = VIRTCHNL2_OP_GET_PTYPE_INFO, 2730 }; 2731 struct virtchnl2_get_ptype_info *get_ptype_info; 2732 struct virtchnl2_get_ptype_info *ptype_info; 2733 int err = 0, max_ptype = IDPF_RX_MAX_PTYPE; 2734 int buf_size = sizeof(*get_ptype_info); 2735 struct libeth_rx_pt *singleq_pt_lkup; 2736 struct libeth_rx_pt *splitq_pt_lkup; 2737 int ptypes_recvd = 0, ptype_offset; 2738 u16 next_ptype_id = 0; 2739 2740 singleq_pt_lkup = kzalloc_objs(*singleq_pt_lkup, IDPF_RX_MAX_BASE_PTYPE); 2741 if (!singleq_pt_lkup) 2742 return -ENOMEM; 2743 2744 splitq_pt_lkup = kzalloc_objs(*splitq_pt_lkup, max_ptype); 2745 if (!splitq_pt_lkup) { 2746 err = -ENOMEM; 2747 goto free_singleq; 2748 } 2749 2750 while (next_ptype_id < max_ptype) { 2751 u16 num_ptypes; 2752 2753 get_ptype_info = kzalloc(buf_size, GFP_KERNEL); 2754 if (!get_ptype_info) { 2755 err = -ENOMEM; 2756 goto free_splitq; 2757 } 2758 2759 get_ptype_info->start_ptype_id = cpu_to_le16(next_ptype_id); 2760 2761 if ((next_ptype_id + IDPF_RX_MAX_PTYPES_PER_BUF) > max_ptype) 2762 num_ptypes = max_ptype - next_ptype_id; 2763 else 2764 num_ptypes = IDPF_RX_MAX_PTYPES_PER_BUF; 2765 2766 get_ptype_info->num_ptypes = cpu_to_le16(num_ptypes); 2767 err = idpf_send_mb_msg_kfree(adapter, &xn_params, 2768 get_ptype_info, buf_size); 2769 if (err) 2770 goto free_splitq; 2771 2772 ptype_info = xn_params.recv_mem.iov_base; 2773 if (xn_params.recv_mem.iov_len < sizeof(*ptype_info)) { 2774 err = -EIO; 2775 goto free_rx_buf; 2776 } 2777 ptypes_recvd += le16_to_cpu(ptype_info->num_ptypes); 2778 if (ptypes_recvd > max_ptype) { 2779 err = -EINVAL; 2780 goto free_rx_buf; 2781 } 2782 2783 next_ptype_id = next_ptype_id + num_ptypes; 2784 ptype_offset = IDPF_RX_PTYPE_HDR_SZ; 2785 2786 for (u16 i = 0; i < le16_to_cpu(ptype_info->num_ptypes); i++) { 2787 struct libeth_rx_pt rx_pt = {}; 2788 struct virtchnl2_ptype *ptype; 2789 u16 pt_10, pt_8; 2790 2791 ptype = (struct virtchnl2_ptype *) 2792 ((u8 *)ptype_info + ptype_offset); 2793 if (xn_params.recv_mem.iov_len < 2794 ptype_offset + sizeof(struct virtchnl2_ptype)) { 2795 err = -EINVAL; 2796 goto free_rx_buf; 2797 } 2798 2799 pt_10 = le16_to_cpu(ptype->ptype_id_10); 2800 pt_8 = ptype->ptype_id_8; 2801 2802 ptype_offset += IDPF_GET_PTYPE_SIZE(ptype); 2803 if (xn_params.recv_mem.iov_len < ptype_offset) { 2804 err = -EINVAL; 2805 goto free_rx_buf; 2806 } 2807 2808 /* 0xFFFF indicates end of ptypes */ 2809 if (pt_10 == IDPF_INVALID_PTYPE_ID) 2810 goto out; 2811 if (pt_10 >= max_ptype) { 2812 err = -EINVAL; 2813 goto free_rx_buf; 2814 } 2815 2816 idpf_parse_protocol_ids(ptype, &rx_pt); 2817 idpf_finalize_ptype_lookup(&rx_pt); 2818 2819 /* For a given protocol ID stack, the ptype value might 2820 * vary between ptype_id_10 and ptype_id_8. So store 2821 * them separately for splitq and singleq. Also skip 2822 * the repeated ptypes in case of singleq. 2823 */ 2824 splitq_pt_lkup[pt_10] = rx_pt; 2825 if (!singleq_pt_lkup[pt_8].outer_ip) 2826 singleq_pt_lkup[pt_8] = rx_pt; 2827 } 2828 2829 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2830 xn_params.recv_mem = (struct kvec) {}; 2831 } 2832 2833 out: 2834 adapter->splitq_pt_lkup = splitq_pt_lkup; 2835 adapter->singleq_pt_lkup = singleq_pt_lkup; 2836 splitq_pt_lkup = NULL; 2837 singleq_pt_lkup = NULL; 2838 free_rx_buf: 2839 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2840 free_splitq: 2841 kfree(splitq_pt_lkup); 2842 free_singleq: 2843 kfree(singleq_pt_lkup); 2844 2845 return err; 2846 } 2847 2848 /** 2849 * idpf_rel_rx_pt_lkup - release RX ptype lookup table 2850 * @adapter: adapter pointer to get the lookup table 2851 */ 2852 static void idpf_rel_rx_pt_lkup(struct idpf_adapter *adapter) 2853 { 2854 kfree(adapter->splitq_pt_lkup); 2855 adapter->splitq_pt_lkup = NULL; 2856 2857 kfree(adapter->singleq_pt_lkup); 2858 adapter->singleq_pt_lkup = NULL; 2859 } 2860 2861 /** 2862 * idpf_send_ena_dis_loopback_msg - Send virtchnl enable/disable loopback 2863 * message 2864 * @adapter: adapter pointer used to send virtchnl message 2865 * @vport_id: vport identifier used while preparing the virtchnl message 2866 * @loopback_ena: flag to enable or disable loopback 2867 * 2868 * Return: 0 on success, negative on failure. 2869 */ 2870 int idpf_send_ena_dis_loopback_msg(struct idpf_adapter *adapter, u32 vport_id, 2871 bool loopback_ena) 2872 { 2873 struct libie_ctlq_xn_send_params xn_params = { 2874 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2875 .chnl_opcode = VIRTCHNL2_OP_LOOPBACK, 2876 }; 2877 struct virtchnl2_loopback loopback; 2878 int err; 2879 2880 loopback.vport_id = cpu_to_le32(vport_id); 2881 loopback.enable = loopback_ena; 2882 2883 err = idpf_send_mb_msg_stack(adapter, &xn_params, &loopback); 2884 if (err) 2885 return err; 2886 2887 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2888 2889 return 0; 2890 } 2891 2892 /** 2893 * idpf_init_dflt_mbx - Setup default mailbox parameters and make request 2894 * @adapter: adapter info struct 2895 * 2896 * Returns 0 on success, negative otherwise 2897 */ 2898 int idpf_init_dflt_mbx(struct idpf_adapter *adapter) 2899 { 2900 struct libie_ctlq_ctx *ctx = &adapter->ctlq_ctx; 2901 struct libie_ctlq_create_info ctlq_info[] = { 2902 { 2903 .type = LIBIE_CTLQ_TYPE_TX, 2904 .id = LIBIE_CTLQ_MBX_ID, 2905 .len = IDPF_DFLT_MBX_Q_LEN, 2906 }, 2907 { 2908 .type = LIBIE_CTLQ_TYPE_RX, 2909 .id = LIBIE_CTLQ_MBX_ID, 2910 .len = IDPF_DFLT_MBX_Q_LEN, 2911 } 2912 }; 2913 struct libie_ctlq_xn_init_params params = { 2914 .num_qs = IDPF_NUM_DFLT_MBX_Q, 2915 .cctlq_info = ctlq_info, 2916 .ctx = ctx, 2917 }; 2918 int err; 2919 2920 adapter->dev_ops.reg_ops.ctlq_reg_init(&ctx->mmio_info, 2921 params.cctlq_info); 2922 2923 err = libie_ctlq_xn_init(¶ms); 2924 if (err) 2925 return err; 2926 2927 adapter->asq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_TX, 2928 LIBIE_CTLQ_MBX_ID); 2929 adapter->arq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_RX, 2930 LIBIE_CTLQ_MBX_ID); 2931 if (!adapter->asq || !adapter->arq) { 2932 adapter->asq = NULL; 2933 adapter->arq = NULL; 2934 libie_ctlq_xn_deinit(params.xnm, ctx); 2935 return -ENOENT; 2936 } 2937 2938 adapter->xnm = params.xnm; 2939 adapter->state = __IDPF_VER_CHECK; 2940 2941 queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); 2942 2943 return 0; 2944 } 2945 2946 /** 2947 * idpf_deinit_dflt_mbx - Free up ctlqs setup 2948 * @adapter: Driver specific private data structure 2949 */ 2950 void idpf_deinit_dflt_mbx(struct idpf_adapter *adapter) 2951 { 2952 idpf_mb_intr_rel_irq(adapter); 2953 cancel_delayed_work_sync(&adapter->mbx_task); 2954 2955 if (adapter->xnm) { 2956 libie_ctlq_xn_shutdown(adapter->xnm); 2957 idpf_mb_clean(adapter->asq, true); 2958 libie_ctlq_xn_deinit(adapter->xnm, &adapter->ctlq_ctx); 2959 } 2960 2961 adapter->arq = NULL; 2962 adapter->asq = NULL; 2963 adapter->xnm = NULL; 2964 } 2965 2966 /** 2967 * idpf_vport_params_buf_rel - Release memory for MailBox resources 2968 * @adapter: Driver specific private data structure 2969 * 2970 * Will release memory to hold the vport parameters received on MailBox 2971 */ 2972 static void idpf_vport_params_buf_rel(struct idpf_adapter *adapter) 2973 { 2974 kfree(adapter->vport_params_recvd); 2975 adapter->vport_params_recvd = NULL; 2976 kfree(adapter->vport_ids); 2977 adapter->vport_ids = NULL; 2978 } 2979 2980 /** 2981 * idpf_vport_params_buf_alloc - Allocate memory for MailBox resources 2982 * @adapter: Driver specific private data structure 2983 * 2984 * Will alloc memory to hold the vport parameters received on MailBox 2985 */ 2986 static int idpf_vport_params_buf_alloc(struct idpf_adapter *adapter) 2987 { 2988 u16 num_max_vports = idpf_get_max_vports(adapter); 2989 2990 adapter->vport_params_recvd = kzalloc_objs(*adapter->vport_params_recvd, 2991 num_max_vports); 2992 if (!adapter->vport_params_recvd) 2993 return -ENOMEM; 2994 2995 adapter->vport_ids = kcalloc(num_max_vports, sizeof(u32), GFP_KERNEL); 2996 if (!adapter->vport_ids) 2997 goto err_mem; 2998 2999 if (adapter->vport_config) 3000 return 0; 3001 3002 adapter->vport_config = kzalloc_objs(*adapter->vport_config, 3003 num_max_vports); 3004 if (!adapter->vport_config) 3005 goto err_mem; 3006 3007 return 0; 3008 3009 err_mem: 3010 idpf_vport_params_buf_rel(adapter); 3011 3012 return -ENOMEM; 3013 } 3014 3015 /** 3016 * idpf_vc_core_init - Initialize state machine and get driver specific 3017 * resources 3018 * @adapter: Driver specific private structure 3019 * 3020 * This function will initialize the state machine and request all necessary 3021 * resources required by the device driver. Once the state machine is 3022 * initialized, allocate memory to store vport specific information and also 3023 * requests required interrupts. 3024 * 3025 * Returns 0 on success, -EAGAIN function will get called again, 3026 * otherwise negative on failure. 3027 */ 3028 int idpf_vc_core_init(struct idpf_adapter *adapter) 3029 { 3030 int task_delay = 30; 3031 u16 num_max_vports; 3032 int err = 0; 3033 3034 while (adapter->state != __IDPF_INIT_SW) { 3035 switch (adapter->state) { 3036 case __IDPF_VER_CHECK: 3037 err = idpf_send_ver_msg(adapter); 3038 switch (err) { 3039 case 0: 3040 /* success, move state machine forward */ 3041 adapter->state = __IDPF_GET_CAPS; 3042 fallthrough; 3043 case -EAGAIN: 3044 goto restart; 3045 default: 3046 /* Something bad happened, try again but only a 3047 * few times. 3048 */ 3049 goto init_failed; 3050 } 3051 case __IDPF_GET_CAPS: 3052 err = idpf_send_get_caps_msg(adapter); 3053 if (err) 3054 goto init_failed; 3055 adapter->state = __IDPF_INIT_SW; 3056 break; 3057 default: 3058 dev_err(&adapter->pdev->dev, "Device is in bad state: %d\n", 3059 adapter->state); 3060 err = -EINVAL; 3061 goto init_failed; 3062 } 3063 break; 3064 restart: 3065 /* Give enough time before proceeding further with 3066 * state machine 3067 */ 3068 msleep(task_delay); 3069 } 3070 3071 if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_LAN_MEMORY_REGIONS)) { 3072 err = idpf_cfg_lan_memory_regions(adapter); 3073 if (err) { 3074 dev_err(&adapter->pdev->dev, "Failed to configure LAN memory regions: %d\n", 3075 err); 3076 return -EINVAL; 3077 } 3078 } else { 3079 /* Fallback to mapping the remaining regions of the entire BAR */ 3080 err = idpf_map_remaining_mmio_regs(adapter); 3081 if (err) { 3082 dev_err(&adapter->pdev->dev, "Failed to configure BAR0 region(s): %d\n", 3083 err); 3084 return err; 3085 } 3086 } 3087 3088 pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter)); 3089 num_max_vports = idpf_get_max_vports(adapter); 3090 adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports); 3091 if (!adapter->vports) { 3092 err = -ENOMEM; 3093 goto decfg_regions; 3094 } 3095 3096 if (!adapter->netdevs) { 3097 adapter->netdevs = kzalloc_objs(struct net_device *, 3098 num_max_vports); 3099 if (!adapter->netdevs) { 3100 err = -ENOMEM; 3101 goto err_netdev_alloc; 3102 } 3103 } 3104 3105 err = idpf_vport_params_buf_alloc(adapter); 3106 if (err) { 3107 dev_err(&adapter->pdev->dev, "Failed to alloc vport params buffer: %d\n", 3108 err); 3109 goto err_netdev_alloc; 3110 } 3111 3112 /* Set max_vports only after vports, netdevs and vport_config buffers 3113 * are allocated to make sure max_vport bound loops don't end up 3114 * crashing, following allocation errors on init. 3115 */ 3116 adapter->max_vports = num_max_vports; 3117 3118 /* Start the mailbox task before requesting vectors. This will ensure 3119 * vector information response from mailbox is handled 3120 */ 3121 queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); 3122 3123 queue_delayed_work(adapter->serv_wq, &adapter->serv_task, 3124 msecs_to_jiffies(5 * (adapter->pdev->devfn & 0x07))); 3125 3126 err = idpf_intr_req(adapter); 3127 if (err) { 3128 dev_err(&adapter->pdev->dev, "failed to enable interrupt vectors: %d\n", 3129 err); 3130 goto err_intr_req; 3131 } 3132 3133 err = idpf_send_get_rx_ptype_msg(adapter); 3134 if (err) { 3135 dev_err(&adapter->pdev->dev, "failed to get RX ptypes: %d\n", 3136 err); 3137 goto intr_rel; 3138 } 3139 3140 err = idpf_ptp_init(adapter); 3141 if (err) 3142 pci_err(adapter->pdev, "PTP init failed, err=%pe\n", 3143 ERR_PTR(err)); 3144 3145 idpf_init_avail_queues(adapter); 3146 3147 /* Skew the delay for init tasks for each function based on fn number 3148 * to prevent every function from making the same call simultaneously. 3149 */ 3150 queue_delayed_work(adapter->init_wq, &adapter->init_task, 3151 msecs_to_jiffies(5 * (adapter->pdev->devfn & 0x07))); 3152 3153 set_bit(IDPF_VC_CORE_INIT, adapter->flags); 3154 3155 return 0; 3156 3157 intr_rel: 3158 idpf_intr_rel(adapter); 3159 err_intr_req: 3160 cancel_delayed_work_sync(&adapter->serv_task); 3161 cancel_delayed_work_sync(&adapter->mbx_task); 3162 idpf_vport_params_buf_rel(adapter); 3163 err_netdev_alloc: 3164 kfree(adapter->vports); 3165 adapter->vports = NULL; 3166 decfg_regions: 3167 idpf_decfg_lan_memory_regions(adapter); 3168 return err; 3169 3170 init_failed: 3171 /* Don't retry if we're trying to go down, just bail. */ 3172 if (test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) 3173 return err; 3174 3175 if (++adapter->mb_wait_count > IDPF_MB_MAX_ERR) { 3176 dev_err(&adapter->pdev->dev, "Failed to establish mailbox communications with hardware\n"); 3177 3178 return -EFAULT; 3179 } 3180 /* If it reached here, it is possible that mailbox queue initialization 3181 * register writes might not have taken effect. Retry to initialize 3182 * the mailbox again 3183 */ 3184 adapter->state = __IDPF_VER_CHECK; 3185 libie_ctlq_xn_shutdown(adapter->xnm); 3186 set_bit(IDPF_HR_DRV_LOAD, adapter->flags); 3187 queue_delayed_work(adapter->vc_event_wq, &adapter->vc_event_task, 3188 msecs_to_jiffies(task_delay)); 3189 3190 return -EAGAIN; 3191 } 3192 3193 /** 3194 * idpf_vc_core_deinit - Device deinit routine 3195 * @adapter: Driver specific private structure 3196 * 3197 */ 3198 void idpf_vc_core_deinit(struct idpf_adapter *adapter) 3199 { 3200 bool remove_in_prog; 3201 3202 if (!test_bit(IDPF_VC_CORE_INIT, adapter->flags)) 3203 return; 3204 3205 /* Avoid transaction timeouts when called during reset */ 3206 remove_in_prog = test_bit(IDPF_REMOVE_IN_PROG, adapter->flags); 3207 if (!remove_in_prog) 3208 libie_ctlq_xn_shutdown(adapter->xnm); 3209 3210 idpf_ptp_release(adapter); 3211 idpf_deinit_task(adapter); 3212 idpf_idc_deinit_core_aux_device(adapter); 3213 idpf_rel_rx_pt_lkup(adapter); 3214 idpf_intr_rel(adapter); 3215 3216 if (remove_in_prog) 3217 libie_ctlq_xn_shutdown(adapter->xnm); 3218 3219 cancel_delayed_work_sync(&adapter->serv_task); 3220 cancel_delayed_work_sync(&adapter->mbx_task); 3221 3222 idpf_vport_params_buf_rel(adapter); 3223 3224 kfree(adapter->vports); 3225 adapter->vports = NULL; 3226 3227 idpf_decfg_lan_memory_regions(adapter); 3228 clear_bit(IDPF_VC_CORE_INIT, adapter->flags); 3229 } 3230 3231 /** 3232 * idpf_vport_alloc_vec_indexes - Get relative vector indexes 3233 * @vport: virtual port data struct 3234 * @rsrc: pointer to queue and vector resources 3235 * 3236 * This function requests the vector information required for the vport and 3237 * stores the vector indexes received from the 'global vector distribution' 3238 * in the vport's queue vectors array. 3239 * 3240 * Return: 0 on success, error on failure 3241 */ 3242 int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport, 3243 struct idpf_q_vec_rsrc *rsrc) 3244 { 3245 struct idpf_vector_info vec_info; 3246 int num_alloc_vecs; 3247 u32 req; 3248 3249 vec_info.num_curr_vecs = rsrc->num_q_vectors; 3250 if (vec_info.num_curr_vecs) 3251 vec_info.num_curr_vecs += IDPF_RESERVED_VECS; 3252 3253 /* XDPSQs are all bound to the NOIRQ vector from IDPF_RESERVED_VECS */ 3254 req = max(rsrc->num_txq - vport->num_xdp_txq, rsrc->num_rxq) + 3255 IDPF_RESERVED_VECS; 3256 vec_info.num_req_vecs = req; 3257 3258 vec_info.default_vport = vport->default_vport; 3259 vec_info.index = vport->idx; 3260 3261 num_alloc_vecs = idpf_req_rel_vector_indexes(vport->adapter, 3262 rsrc->q_vector_idxs, 3263 &vec_info); 3264 if (num_alloc_vecs <= 0) { 3265 dev_err(&vport->adapter->pdev->dev, "Vector distribution failed: %d\n", 3266 num_alloc_vecs); 3267 return -EINVAL; 3268 } 3269 3270 rsrc->num_q_vectors = num_alloc_vecs - IDPF_RESERVED_VECS; 3271 3272 return 0; 3273 } 3274 3275 /** 3276 * idpf_vport_init - Initialize virtual port 3277 * @vport: virtual port to be initialized 3278 * @max_q: vport max queue info 3279 * 3280 * Will initialize vport with the info received through MB earlier 3281 * 3282 * Return: 0 on success, negative on failure. 3283 */ 3284 int idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q) 3285 { 3286 struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; 3287 struct idpf_adapter *adapter = vport->adapter; 3288 struct virtchnl2_create_vport *vport_msg; 3289 struct idpf_vport_config *vport_config; 3290 u16 tx_itr[] = {2, 8, 64, 128, 256}; 3291 u16 rx_itr[] = {2, 8, 32, 96, 128}; 3292 struct idpf_rss_data *rss_data; 3293 u16 idx = vport->idx; 3294 int err; 3295 3296 vport_config = adapter->vport_config[idx]; 3297 rss_data = &vport_config->user_config.rss_data; 3298 vport_msg = adapter->vport_params_recvd[idx]; 3299 3300 err = idpf_vport_init_queue_reg_chunks(vport_config, 3301 &vport_msg->chunks); 3302 if (err) 3303 return err; 3304 3305 vport_config->max_q.max_txq = max_q->max_txq; 3306 vport_config->max_q.max_rxq = max_q->max_rxq; 3307 vport_config->max_q.max_complq = max_q->max_complq; 3308 vport_config->max_q.max_bufq = max_q->max_bufq; 3309 3310 rsrc->txq_model = le16_to_cpu(vport_msg->txq_model); 3311 rsrc->rxq_model = le16_to_cpu(vport_msg->rxq_model); 3312 vport->vport_type = le16_to_cpu(vport_msg->vport_type); 3313 vport->vport_id = le32_to_cpu(vport_msg->vport_id); 3314 3315 rss_data->rss_key_size = min_t(u16, NETDEV_RSS_KEY_LEN, 3316 le16_to_cpu(vport_msg->rss_key_size)); 3317 rss_data->rss_lut_size = le16_to_cpu(vport_msg->rss_lut_size); 3318 3319 ether_addr_copy(vport->default_mac_addr, vport_msg->default_mac_addr); 3320 vport->max_mtu = le16_to_cpu(vport_msg->max_mtu) - LIBETH_RX_LL_LEN; 3321 3322 /* Initialize Tx and Rx profiles for Dynamic Interrupt Moderation */ 3323 memcpy(vport->rx_itr_profile, rx_itr, IDPF_DIM_PROFILE_SLOTS); 3324 memcpy(vport->tx_itr_profile, tx_itr, IDPF_DIM_PROFILE_SLOTS); 3325 3326 idpf_vport_set_hsplit(vport, ETHTOOL_TCP_DATA_SPLIT_ENABLED); 3327 3328 idpf_vport_init_num_qs(vport, vport_msg, rsrc); 3329 idpf_vport_calc_num_q_desc(vport, rsrc); 3330 idpf_vport_calc_num_q_groups(rsrc); 3331 idpf_vport_alloc_vec_indexes(vport, rsrc); 3332 3333 vport->crc_enable = adapter->crc_enable; 3334 3335 if (!(vport_msg->vport_flags & 3336 cpu_to_le16(VIRTCHNL2_VPORT_UPLINK_PORT))) 3337 return 0; 3338 3339 err = idpf_ptp_get_vport_tstamps_caps(vport); 3340 if (err) { 3341 /* Do not error on timestamp failure */ 3342 pci_dbg(vport->adapter->pdev, "Tx timestamping not supported\n"); 3343 return 0; 3344 } 3345 3346 INIT_WORK(&vport->tstamp_task, idpf_tstamp_task); 3347 3348 return 0; 3349 } 3350 3351 /** 3352 * idpf_get_vec_ids - Initialize vector id from Mailbox parameters 3353 * @adapter: adapter structure to get the mailbox vector id 3354 * @vecids: Array of vector ids 3355 * @num_vecids: number of vector ids 3356 * @chunks: vector ids received over mailbox 3357 * 3358 * Will initialize the mailbox vector id which is received from the 3359 * get capabilities and data queue vector ids with ids received as 3360 * mailbox parameters. 3361 * Returns number of ids filled 3362 */ 3363 int idpf_get_vec_ids(struct idpf_adapter *adapter, 3364 u16 *vecids, int num_vecids, 3365 struct virtchnl2_vector_chunks *chunks) 3366 { 3367 u16 num_chunks = le16_to_cpu(chunks->num_vchunks); 3368 int num_vecid_filled = 0; 3369 int i, j; 3370 3371 vecids[num_vecid_filled] = adapter->mb_vector.v_idx; 3372 num_vecid_filled++; 3373 3374 for (j = 0; j < num_chunks; j++) { 3375 struct virtchnl2_vector_chunk *chunk; 3376 u16 start_vecid, num_vec; 3377 3378 chunk = &chunks->vchunks[j]; 3379 num_vec = le16_to_cpu(chunk->num_vectors); 3380 start_vecid = le16_to_cpu(chunk->start_vector_id); 3381 3382 for (i = 0; i < num_vec; i++) { 3383 if ((num_vecid_filled + i) < num_vecids) { 3384 vecids[num_vecid_filled + i] = start_vecid; 3385 start_vecid++; 3386 } else { 3387 break; 3388 } 3389 } 3390 num_vecid_filled = num_vecid_filled + i; 3391 } 3392 3393 return num_vecid_filled; 3394 } 3395 3396 /** 3397 * idpf_vport_get_queue_ids - Initialize queue id from Mailbox parameters 3398 * @qids: Array of queue ids 3399 * @num_qids: number of queue ids 3400 * @q_type: queue model 3401 * @chunks: queue ids received over mailbox 3402 * 3403 * Will initialize all queue ids with ids received as mailbox parameters 3404 * Returns number of ids filled 3405 */ 3406 static int idpf_vport_get_queue_ids(u32 *qids, int num_qids, u16 q_type, 3407 struct idpf_queue_id_reg_info *chunks) 3408 { 3409 u16 num_chunks = chunks->num_chunks; 3410 u32 num_q_id_filled = 0, i; 3411 u32 start_q_id, num_q; 3412 3413 while (num_chunks--) { 3414 struct idpf_queue_id_reg_chunk *chunk; 3415 3416 chunk = &chunks->queue_chunks[num_chunks]; 3417 if (chunk->type != q_type) 3418 continue; 3419 3420 num_q = chunk->num_queues; 3421 start_q_id = chunk->start_queue_id; 3422 3423 for (i = 0; i < num_q; i++) { 3424 if ((num_q_id_filled + i) < num_qids) { 3425 qids[num_q_id_filled + i] = start_q_id; 3426 start_q_id++; 3427 } else { 3428 break; 3429 } 3430 } 3431 num_q_id_filled = num_q_id_filled + i; 3432 } 3433 3434 return num_q_id_filled; 3435 } 3436 3437 /** 3438 * __idpf_vport_queue_ids_init - Initialize queue ids from Mailbox parameters 3439 * @vport: virtual port for which the queues ids are initialized 3440 * @rsrc: pointer to queue and vector resources 3441 * @qids: queue ids 3442 * @num_qids: number of queue ids 3443 * @q_type: type of queue 3444 * 3445 * Will initialize all queue ids with ids received as mailbox 3446 * parameters. Returns number of queue ids initialized. 3447 */ 3448 static int __idpf_vport_queue_ids_init(struct idpf_vport *vport, 3449 struct idpf_q_vec_rsrc *rsrc, 3450 const u32 *qids, 3451 int num_qids, 3452 u32 q_type) 3453 { 3454 int i, j, k = 0; 3455 3456 switch (q_type) { 3457 case VIRTCHNL2_QUEUE_TYPE_TX: 3458 for (i = 0; i < rsrc->num_txq_grp; i++) { 3459 struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 3460 3461 for (j = 0; j < tx_qgrp->num_txq && k < num_qids; j++, k++) 3462 tx_qgrp->txqs[j]->q_id = qids[k]; 3463 } 3464 break; 3465 case VIRTCHNL2_QUEUE_TYPE_RX: 3466 for (i = 0; i < rsrc->num_rxq_grp; i++) { 3467 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 3468 u16 num_rxq; 3469 3470 if (idpf_is_queue_model_split(rsrc->rxq_model)) 3471 num_rxq = rx_qgrp->splitq.num_rxq_sets; 3472 else 3473 num_rxq = rx_qgrp->singleq.num_rxq; 3474 3475 for (j = 0; j < num_rxq && k < num_qids; j++, k++) { 3476 struct idpf_rx_queue *q; 3477 3478 if (idpf_is_queue_model_split(rsrc->rxq_model)) 3479 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 3480 else 3481 q = rx_qgrp->singleq.rxqs[j]; 3482 q->q_id = qids[k]; 3483 } 3484 } 3485 break; 3486 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 3487 for (i = 0; i < rsrc->num_txq_grp && k < num_qids; i++, k++) { 3488 struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 3489 3490 tx_qgrp->complq->q_id = qids[k]; 3491 } 3492 break; 3493 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 3494 for (i = 0; i < rsrc->num_rxq_grp; i++) { 3495 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 3496 u8 num_bufqs = rsrc->num_bufqs_per_qgrp; 3497 3498 for (j = 0; j < num_bufqs && k < num_qids; j++, k++) { 3499 struct idpf_buf_queue *q; 3500 3501 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 3502 q->q_id = qids[k]; 3503 } 3504 } 3505 break; 3506 default: 3507 break; 3508 } 3509 3510 return k; 3511 } 3512 3513 /** 3514 * idpf_vport_queue_ids_init - Initialize queue ids from Mailbox parameters 3515 * @vport: virtual port for which the queues ids are initialized 3516 * @rsrc: pointer to queue and vector resources 3517 * @chunks: queue ids received over mailbox 3518 * 3519 * Will initialize all queue ids with ids received as mailbox parameters. 3520 * 3521 * Return: 0 on success, negative if all the queues are not initialized. 3522 */ 3523 int idpf_vport_queue_ids_init(struct idpf_vport *vport, 3524 struct idpf_q_vec_rsrc *rsrc, 3525 struct idpf_queue_id_reg_info *chunks) 3526 { 3527 int num_ids, err = 0; 3528 u16 q_type; 3529 u32 *qids; 3530 3531 qids = kcalloc(IDPF_MAX_QIDS, sizeof(u32), GFP_KERNEL); 3532 if (!qids) 3533 return -ENOMEM; 3534 3535 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, 3536 VIRTCHNL2_QUEUE_TYPE_TX, 3537 chunks); 3538 if (num_ids < rsrc->num_txq) { 3539 err = -EINVAL; 3540 goto mem_rel; 3541 } 3542 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, num_ids, 3543 VIRTCHNL2_QUEUE_TYPE_TX); 3544 if (num_ids < rsrc->num_txq) { 3545 err = -EINVAL; 3546 goto mem_rel; 3547 } 3548 3549 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, 3550 VIRTCHNL2_QUEUE_TYPE_RX, 3551 chunks); 3552 if (num_ids < rsrc->num_rxq) { 3553 err = -EINVAL; 3554 goto mem_rel; 3555 } 3556 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, num_ids, 3557 VIRTCHNL2_QUEUE_TYPE_RX); 3558 if (num_ids < rsrc->num_rxq) { 3559 err = -EINVAL; 3560 goto mem_rel; 3561 } 3562 3563 if (!idpf_is_queue_model_split(rsrc->txq_model)) 3564 goto check_rxq; 3565 3566 q_type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 3567 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, q_type, chunks); 3568 if (num_ids < rsrc->num_complq) { 3569 err = -EINVAL; 3570 goto mem_rel; 3571 } 3572 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, 3573 num_ids, q_type); 3574 if (num_ids < rsrc->num_complq) { 3575 err = -EINVAL; 3576 goto mem_rel; 3577 } 3578 3579 check_rxq: 3580 if (!idpf_is_queue_model_split(rsrc->rxq_model)) 3581 goto mem_rel; 3582 3583 q_type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 3584 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, q_type, chunks); 3585 if (num_ids < rsrc->num_bufq) { 3586 err = -EINVAL; 3587 goto mem_rel; 3588 } 3589 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, 3590 num_ids, q_type); 3591 if (num_ids < rsrc->num_bufq) 3592 err = -EINVAL; 3593 3594 mem_rel: 3595 kfree(qids); 3596 3597 return err; 3598 } 3599 3600 /** 3601 * idpf_vport_adjust_qs - Adjust to new requested queues 3602 * @vport: virtual port data struct 3603 * @rsrc: pointer to queue and vector resources 3604 * 3605 * Renegotiate queues. Returns 0 on success, negative on failure. 3606 */ 3607 int idpf_vport_adjust_qs(struct idpf_vport *vport, struct idpf_q_vec_rsrc *rsrc) 3608 { 3609 struct virtchnl2_create_vport vport_msg; 3610 int err; 3611 3612 vport_msg.txq_model = cpu_to_le16(rsrc->txq_model); 3613 vport_msg.rxq_model = cpu_to_le16(rsrc->rxq_model); 3614 err = idpf_vport_calc_total_qs(vport->adapter, vport->idx, &vport_msg, 3615 NULL); 3616 if (err) 3617 return err; 3618 3619 idpf_vport_init_num_qs(vport, &vport_msg, rsrc); 3620 idpf_vport_calc_num_q_groups(rsrc); 3621 3622 return 0; 3623 } 3624 3625 /** 3626 * idpf_is_capability_ena - Default implementation of capability checking 3627 * @adapter: Private data struct 3628 * @all: all or one flag 3629 * @field: caps field to check for flags 3630 * @flag: flag to check 3631 * 3632 * Return true if all capabilities are supported, false otherwise 3633 */ 3634 bool idpf_is_capability_ena(struct idpf_adapter *adapter, bool all, 3635 enum idpf_cap_field field, u64 flag) 3636 { 3637 u8 *caps = (u8 *)&adapter->caps; 3638 u32 *cap_field; 3639 3640 if (!caps) 3641 return false; 3642 3643 if (field == IDPF_BASE_CAPS) 3644 return false; 3645 3646 cap_field = (u32 *)(caps + field); 3647 3648 if (all) 3649 return (*cap_field & flag) == flag; 3650 else 3651 return !!(*cap_field & flag); 3652 } 3653 3654 /** 3655 * idpf_vport_is_cap_ena - Check if vport capability is enabled 3656 * @vport: Private data struct 3657 * @flag: flag(s) to check 3658 * 3659 * Return: true if the capability is supported, false otherwise 3660 */ 3661 bool idpf_vport_is_cap_ena(struct idpf_vport *vport, u16 flag) 3662 { 3663 struct virtchnl2_create_vport *vport_msg; 3664 3665 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3666 3667 return !!(le16_to_cpu(vport_msg->vport_flags) & flag); 3668 } 3669 3670 /** 3671 * idpf_sideband_flow_type_ena - Check if steering is enabled for flow type 3672 * @vport: Private data struct 3673 * @flow_type: flow type to check (from ethtool.h) 3674 * 3675 * Return: true if sideband filters are allowed for @flow_type, false otherwise 3676 */ 3677 bool idpf_sideband_flow_type_ena(struct idpf_vport *vport, u32 flow_type) 3678 { 3679 struct virtchnl2_create_vport *vport_msg; 3680 __le64 caps; 3681 3682 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3683 caps = vport_msg->sideband_flow_caps; 3684 3685 switch (flow_type) { 3686 case TCP_V4_FLOW: 3687 return !!(caps & cpu_to_le64(VIRTCHNL2_FLOW_IPV4_TCP)); 3688 case UDP_V4_FLOW: 3689 return !!(caps & cpu_to_le64(VIRTCHNL2_FLOW_IPV4_UDP)); 3690 default: 3691 return false; 3692 } 3693 } 3694 3695 /** 3696 * idpf_sideband_action_ena - Check if steering is enabled for action 3697 * @vport: Private data struct 3698 * @fsp: flow spec 3699 * 3700 * Return: true if sideband filters are allowed for @fsp, false otherwise 3701 */ 3702 bool idpf_sideband_action_ena(struct idpf_vport *vport, 3703 struct ethtool_rx_flow_spec *fsp) 3704 { 3705 struct virtchnl2_create_vport *vport_msg; 3706 unsigned int supp_actions; 3707 3708 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3709 supp_actions = le32_to_cpu(vport_msg->sideband_flow_actions); 3710 3711 /* Actions Drop/Wake are not supported */ 3712 if (fsp->ring_cookie == RX_CLS_FLOW_DISC || 3713 fsp->ring_cookie == RX_CLS_FLOW_WAKE) 3714 return false; 3715 3716 return !!(supp_actions & VIRTCHNL2_ACTION_QUEUE); 3717 } 3718 3719 unsigned int idpf_fsteer_max_rules(struct idpf_vport *vport) 3720 { 3721 struct virtchnl2_create_vport *vport_msg; 3722 3723 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3724 return le32_to_cpu(vport_msg->flow_steer_max_rules); 3725 } 3726 3727 /** 3728 * idpf_get_vport_id: Get vport id 3729 * @vport: virtual port structure 3730 * 3731 * Return vport id from the adapter persistent data 3732 */ 3733 u32 idpf_get_vport_id(struct idpf_vport *vport) 3734 { 3735 struct virtchnl2_create_vport *vport_msg; 3736 3737 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3738 3739 return le32_to_cpu(vport_msg->vport_id); 3740 } 3741 3742 static void idpf_set_mac_type(const u8 *default_mac_addr, 3743 struct virtchnl2_mac_addr *mac_addr) 3744 { 3745 bool is_primary; 3746 3747 is_primary = ether_addr_equal(default_mac_addr, mac_addr->addr); 3748 mac_addr->type = is_primary ? VIRTCHNL2_MAC_ADDR_PRIMARY : 3749 VIRTCHNL2_MAC_ADDR_EXTRA; 3750 } 3751 3752 /** 3753 * idpf_mac_filter_async_handler - Async callback for mac filters 3754 * @ctx: controlq context structure 3755 * @buff: response buffer pointer and size 3756 * @status: async call return value 3757 * 3758 * In some scenarios driver can't sleep and wait for a reply (e.g.: stack is 3759 * holding rtnl_lock) when adding a new mac filter. It puts us in a difficult 3760 * situation to deal with errors returned on the reply. The best we can 3761 * ultimately do is remove it from our list of mac filters and report the 3762 * error. 3763 */ 3764 static void idpf_mac_filter_async_handler(void *ctx, 3765 struct kvec *buff, 3766 int status) 3767 { 3768 struct virtchnl2_mac_addr_list *ma_list; 3769 struct idpf_vport_config *vport_config; 3770 struct virtchnl2_mac_addr *mac_addr; 3771 struct idpf_adapter *adapter = ctx; 3772 struct idpf_mac_filter *f, *tmp; 3773 struct list_head *ma_list_head; 3774 struct idpf_vport *vport; 3775 u16 num_entries; 3776 int i; 3777 3778 /* if success we're done, we're only here if something bad happened */ 3779 if (!status || status == -ETIMEDOUT) 3780 return; 3781 3782 ma_list = buff->iov_base; 3783 /* make sure at least struct is there */ 3784 if (buff->iov_len < sizeof(*ma_list)) 3785 goto invalid_payload; 3786 3787 mac_addr = ma_list->mac_addr_list; 3788 num_entries = le16_to_cpu(ma_list->num_mac_addr); 3789 /* we should have received a buffer at least this big */ 3790 if (buff->iov_len < struct_size(ma_list, mac_addr_list, num_entries)) 3791 goto invalid_payload; 3792 3793 vport = idpf_vid_to_vport(adapter, le32_to_cpu(ma_list->vport_id)); 3794 if (!vport) 3795 goto invalid_payload; 3796 3797 vport_config = adapter->vport_config[le32_to_cpu(ma_list->vport_id)]; 3798 ma_list_head = &vport_config->user_config.mac_filter_list; 3799 3800 /* We can't do much to reconcile bad filters at this point, however we 3801 * should at least remove them from our list one way or the other so we 3802 * have some idea what good filters we have. 3803 */ 3804 spin_lock_bh(&vport_config->mac_filter_list_lock); 3805 list_for_each_entry_safe(f, tmp, ma_list_head, list) 3806 for (i = 0; i < num_entries; i++) 3807 if (ether_addr_equal(mac_addr[i].addr, f->macaddr)) 3808 list_del(&f->list); 3809 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3810 dev_err_ratelimited(&adapter->pdev->dev, "Received error %d on sending MAC filter request\n", 3811 status); 3812 return; 3813 3814 invalid_payload: 3815 dev_err_ratelimited(&adapter->pdev->dev, "Received invalid MAC filter payload (len %zd)\n", 3816 buff->iov_len); 3817 } 3818 3819 /** 3820 * idpf_add_del_mac_filters - Add/del mac filters 3821 * @adapter: adapter pointer used to send virtchnl message 3822 * @vport_config: persistent vport structure to get the MAC filter list 3823 * @default_mac_addr: default MAC address to compare with 3824 * @vport_id: vport identifier used while preparing the virtchnl message 3825 * @add: Add or delete flag 3826 * @async: Don't wait for return message 3827 * 3828 * Return: 0 on success, error on failure. 3829 **/ 3830 int idpf_add_del_mac_filters(struct idpf_adapter *adapter, 3831 struct idpf_vport_config *vport_config, 3832 const u8 *default_mac_addr, u32 vport_id, 3833 bool add, bool async) 3834 { 3835 struct virtchnl2_mac_addr *mac_addr __free(kfree) = NULL; 3836 struct libie_ctlq_xn_send_params xn_params = { 3837 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 3838 .chnl_opcode = add ? VIRTCHNL2_OP_ADD_MAC_ADDR : 3839 VIRTCHNL2_OP_DEL_MAC_ADDR, 3840 }; 3841 struct virtchnl2_mac_addr_list *ma_list; 3842 u32 num_msgs, total_filters = 0; 3843 struct idpf_mac_filter *f; 3844 int i = 0; 3845 3846 if (async) { 3847 xn_params.resp_cb = idpf_mac_filter_async_handler; 3848 xn_params.send_ctx = adapter; 3849 } 3850 3851 spin_lock_bh(&vport_config->mac_filter_list_lock); 3852 3853 /* Find the number of newly added filters */ 3854 list_for_each_entry(f, &vport_config->user_config.mac_filter_list, 3855 list) { 3856 if (add && f->add) 3857 total_filters++; 3858 else if (!add && f->remove) 3859 total_filters++; 3860 } 3861 3862 if (!total_filters) { 3863 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3864 3865 return 0; 3866 } 3867 3868 /* Fill all the new filters into virtchannel message */ 3869 mac_addr = kzalloc_objs(struct virtchnl2_mac_addr, total_filters, 3870 GFP_ATOMIC); 3871 if (!mac_addr) { 3872 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3873 3874 return -ENOMEM; 3875 } 3876 3877 list_for_each_entry(f, &vport_config->user_config.mac_filter_list, 3878 list) { 3879 if (add && f->add) { 3880 ether_addr_copy(mac_addr[i].addr, f->macaddr); 3881 idpf_set_mac_type(default_mac_addr, &mac_addr[i]); 3882 i++; 3883 f->add = false; 3884 if (i == total_filters) 3885 break; 3886 } 3887 if (!add && f->remove) { 3888 ether_addr_copy(mac_addr[i].addr, f->macaddr); 3889 idpf_set_mac_type(default_mac_addr, &mac_addr[i]); 3890 i++; 3891 f->remove = false; 3892 if (i == total_filters) 3893 break; 3894 } 3895 } 3896 3897 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3898 3899 /* Chunk up the filters into multiple messages to avoid 3900 * sending a control queue message buffer that is too large 3901 */ 3902 num_msgs = DIV_ROUND_UP(total_filters, IDPF_NUM_FILTERS_PER_MSG); 3903 3904 for (u32 i = 0, k = 0; i < num_msgs; i++) { 3905 u32 entries_size, num_entries; 3906 size_t buf_size; 3907 int err; 3908 3909 num_entries = min_t(u32, total_filters, 3910 IDPF_NUM_FILTERS_PER_MSG); 3911 entries_size = sizeof(struct virtchnl2_mac_addr) * num_entries; 3912 buf_size = struct_size(ma_list, mac_addr_list, num_entries); 3913 3914 ma_list = kzalloc(buf_size, GFP_ATOMIC); 3915 if (!ma_list) 3916 return -ENOMEM; 3917 3918 ma_list->vport_id = cpu_to_le32(vport_id); 3919 ma_list->num_mac_addr = cpu_to_le16(num_entries); 3920 memcpy(ma_list->mac_addr_list, &mac_addr[k], entries_size); 3921 3922 err = idpf_send_mb_msg_kfree(adapter, &xn_params, ma_list, 3923 buf_size); 3924 if (err) 3925 return err; 3926 3927 if (!async) 3928 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 3929 3930 k += num_entries; 3931 total_filters -= num_entries; 3932 } 3933 3934 return 0; 3935 } 3936 3937 /** 3938 * idpf_promiscuous_async_handler - async callback for promiscuous mode 3939 * @ctx: controlq context structure 3940 * @buff: response buffer pointer and size 3941 * @status: async call return value 3942 * 3943 * Nobody is waiting for the promiscuous virtchnl message response. Print 3944 * an error message if something went wrong and return. 3945 */ 3946 static void idpf_promiscuous_async_handler(void *ctx, 3947 struct kvec *buff, 3948 int status) 3949 { 3950 struct idpf_adapter *adapter = ctx; 3951 3952 if (status) 3953 dev_err_ratelimited(&adapter->pdev->dev, "Failed to set promiscuous mode: %d\n", 3954 status); 3955 } 3956 3957 /** 3958 * idpf_set_promiscuous - set promiscuous and send message to mailbox 3959 * @adapter: Driver specific private structure 3960 * @config_data: Vport specific config data 3961 * @vport_id: Vport identifier 3962 * 3963 * Request to enable promiscuous mode for the vport. Message is sent 3964 * asynchronously and won't wait for response. Returns 0 on success, negative 3965 * on failure; 3966 */ 3967 int idpf_set_promiscuous(struct idpf_adapter *adapter, 3968 struct idpf_vport_user_config_data *config_data, 3969 u32 vport_id) 3970 { 3971 struct libie_ctlq_xn_send_params xn_params = { 3972 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 3973 .chnl_opcode = VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE, 3974 .resp_cb = idpf_promiscuous_async_handler, 3975 .send_ctx = adapter, 3976 }; 3977 struct virtchnl2_promisc_info vpi; 3978 u16 flags = 0; 3979 3980 if (test_bit(__IDPF_PROMISC_UC, config_data->user_flags)) 3981 flags |= VIRTCHNL2_UNICAST_PROMISC; 3982 if (test_bit(__IDPF_PROMISC_MC, config_data->user_flags)) 3983 flags |= VIRTCHNL2_MULTICAST_PROMISC; 3984 3985 vpi.vport_id = cpu_to_le32(vport_id); 3986 vpi.flags = cpu_to_le16(flags); 3987 3988 return idpf_send_mb_msg_stack(adapter, &xn_params, &vpi); 3989 } 3990 3991 /** 3992 * idpf_idc_rdma_vc_send_sync - virtchnl send callback for IDC registered drivers 3993 * @cdev_info: IDC core device info pointer 3994 * @send_msg: message to send 3995 * @msg_size: size of message to send 3996 * @recv_msg: message to populate on reception of response 3997 * @recv_len: on input, maximum response size; on success, actual response size 3998 * 3999 * Return: 0 on success or error code on failure. 4000 */ 4001 int idpf_idc_rdma_vc_send_sync(struct iidc_rdma_core_dev_info *cdev_info, 4002 u8 *send_msg, u16 msg_size, 4003 u8 *recv_msg, u16 *recv_len) 4004 { 4005 struct idpf_adapter *adapter = pci_get_drvdata(cdev_info->pdev); 4006 struct libie_ctlq_xn_send_params xn_params = { 4007 .chnl_opcode = VIRTCHNL2_OP_RDMA, 4008 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 4009 }; 4010 u8 on_stack_buf[LIBIE_CP_TX_COPYBREAK]; 4011 void *send_buf; 4012 int err; 4013 4014 if (!recv_msg || !recv_len || msg_size > LIBIE_CTLQ_MAX_BUF_LEN) 4015 return -EINVAL; 4016 4017 if (!libie_cp_can_send_onstack(msg_size)) { 4018 send_buf = kzalloc(msg_size, GFP_KERNEL); 4019 if (!send_buf) 4020 return -ENOMEM; 4021 } else { 4022 send_buf = on_stack_buf; 4023 } 4024 4025 memcpy(send_buf, send_msg, msg_size); 4026 err = idpf_send_mb_msg(adapter, &xn_params, send_buf, msg_size); 4027 if (err) 4028 return err; 4029 4030 if (xn_params.recv_mem.iov_len > *recv_len) { 4031 err = -EINVAL; 4032 goto rel_buf; 4033 } 4034 4035 *recv_len = xn_params.recv_mem.iov_len; 4036 memcpy(recv_msg, xn_params.recv_mem.iov_base, *recv_len); 4037 rel_buf: 4038 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 4039 return err; 4040 } 4041 EXPORT_SYMBOL_GPL(idpf_idc_rdma_vc_send_sync); 4042