1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2023 Intel Corporation */ 3 4 #include <linux/export.h> 5 #include <linux/net/intel/libie/pci.h> 6 #include <linux/log2.h> 7 #include <net/libeth/rx.h> 8 9 #include "idpf.h" 10 #include "idpf_virtchnl.h" 11 #include "idpf_ptp.h" 12 13 /** 14 * idpf_vid_to_vport - Translate vport id to vport pointer 15 * @adapter: private data struct 16 * @v_id: vport id to translate 17 * 18 * Returns vport matching v_id, NULL if not found. 19 */ 20 static 21 struct idpf_vport *idpf_vid_to_vport(struct idpf_adapter *adapter, u32 v_id) 22 { 23 u16 num_max_vports = idpf_get_max_vports(adapter); 24 int i; 25 26 for (i = 0; i < num_max_vports; i++) 27 if (adapter->vport_ids[i] == v_id) 28 return adapter->vports[i]; 29 30 return NULL; 31 } 32 33 /** 34 * idpf_handle_event_link - Handle link event message 35 * @adapter: private data struct 36 * @v2e: virtchnl event message 37 */ 38 static void idpf_handle_event_link(struct idpf_adapter *adapter, 39 const struct virtchnl2_event *v2e) 40 { 41 struct idpf_netdev_priv *np; 42 struct idpf_vport *vport; 43 44 vport = idpf_vid_to_vport(adapter, le32_to_cpu(v2e->vport_id)); 45 if (!vport) { 46 dev_err_ratelimited(&adapter->pdev->dev, "Failed to find vport_id %d for link event\n", 47 v2e->vport_id); 48 return; 49 } 50 np = netdev_priv(vport->netdev); 51 52 np->link_speed_mbps = le32_to_cpu(v2e->link_speed); 53 54 if (vport->link_up == v2e->link_status) 55 return; 56 57 vport->link_up = v2e->link_status; 58 59 if (!test_bit(IDPF_VPORT_UP, np->state)) 60 return; 61 62 if (vport->link_up) { 63 netif_tx_start_all_queues(vport->netdev); 64 netif_carrier_on(vport->netdev); 65 } else { 66 netif_tx_stop_all_queues(vport->netdev); 67 netif_carrier_off(vport->netdev); 68 } 69 } 70 71 /** 72 * idpf_recv_event_msg - Receive virtchnl event message 73 * @ctx: control queue context 74 * @ctlq_msg: message to copy from 75 * 76 * Receive virtchnl event message 77 */ 78 void idpf_recv_event_msg(struct libie_ctlq_ctx *ctx, 79 struct libie_ctlq_msg *ctlq_msg) 80 { 81 struct kvec *buff = &ctlq_msg->recv_mem; 82 int payload_size = buff->iov_len; 83 struct idpf_adapter *adapter; 84 struct virtchnl2_event *v2e; 85 u32 event; 86 87 adapter = container_of(ctx, struct idpf_adapter, ctlq_ctx); 88 if (ctlq_msg->chnl_opcode != VIRTCHNL2_OP_EVENT) { 89 dev_dbg(&adapter->pdev->dev, 90 "Unhandled message with opcode %u from CP\n", 91 ctlq_msg->chnl_opcode); 92 goto free_rx_buf; 93 } 94 95 if (payload_size < sizeof(*v2e)) { 96 dev_err_ratelimited(&adapter->pdev->dev, "Failed to receive valid payload for event msg (op %d len %d)\n", 97 ctlq_msg->chnl_opcode, 98 payload_size); 99 goto free_rx_buf; 100 } 101 102 v2e = (struct virtchnl2_event *)buff->iov_base; 103 event = le32_to_cpu(v2e->event); 104 105 switch (event) { 106 case VIRTCHNL2_EVENT_LINK_CHANGE: 107 idpf_handle_event_link(adapter, v2e); 108 break; 109 default: 110 dev_err(&adapter->pdev->dev, 111 "Unknown event %d from PF\n", event); 112 break; 113 } 114 115 free_rx_buf: 116 libie_ctlq_release_rx_buf(buff); 117 } 118 119 /** 120 * idpf_mb_clean - Reclaim the send mailbox queue entries 121 * @asq: send control queue info 122 * @deinit: release all buffers before destroying the queue 123 * 124 * This is a helper function to clean the send mailbox queue entries. 125 */ 126 static void idpf_mb_clean(struct libie_ctlq_info *asq, bool deinit) 127 { 128 libie_ctlq_xn_send_clean(asq, kfree, deinit); 129 } 130 131 #if IS_ENABLED(CONFIG_PTP_1588_CLOCK) 132 /** 133 * idpf_ptp_is_mb_msg - Check if the message is PTP-related 134 * @op: virtchnl opcode 135 * 136 * Return: true if msg is PTP-related, false otherwise. 137 */ 138 static bool idpf_ptp_is_mb_msg(u32 op) 139 { 140 switch (op) { 141 case VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME: 142 case VIRTCHNL2_OP_PTP_GET_CROSS_TIME: 143 case VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME: 144 case VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE: 145 case VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME: 146 case VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS: 147 case VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP: 148 return true; 149 default: 150 return false; 151 } 152 } 153 154 /** 155 * idpf_prepare_ptp_mb_msg - Prepare PTP related message 156 * 157 * @adapter: Driver specific private structure 158 * @op: virtchnl opcode 159 * @ctlq_msg: Corresponding control queue message 160 */ 161 static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, 162 struct libie_ctlq_msg *ctlq_msg) 163 { 164 /* If the message is PTP-related and the secondary mailbox is available, 165 * send the message through the secondary mailbox. 166 */ 167 if (!idpf_ptp_is_mb_msg(op) || !adapter->ptp->secondary_mbx.valid) 168 return; 169 170 ctlq_msg->opcode = LIBIE_CTLQ_SEND_MSG_TO_PEER; 171 ctlq_msg->func_id = adapter->ptp->secondary_mbx.peer_mbx_q_id; 172 ctlq_msg->flags = FIELD_PREP(LIBIE_CTLQ_DESC_FLAG_HOST_ID, 173 adapter->ptp->secondary_mbx.peer_id); 174 } 175 #else /* !CONFIG_PTP_1588_CLOCK */ 176 static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, 177 struct libie_ctlq_msg *ctlq_msg) 178 { } 179 #endif /* CONFIG_PTP_1588_CLOCK */ 180 181 /** 182 * idpf_send_mb_msg - send mailbox message to the device control plane 183 * @adapter: driver specific private structure 184 * @xn_params: Xn send parameters to fill 185 * @send_buf: buffer to send 186 * @send_buf_size: size of the send buffer 187 * 188 * Fill the Xn parameters with the required info to send a virtchnl message. 189 * The send buffer is DMA mapped in the libie to avoid memcpy. 190 * 191 * Cleanup the mailbox queue entries of the previously sent message to 192 * unmap and release the buffer. 193 * 194 * Return: 0 if the request was successful, -%EBUSY if reset is detected 195 * or Tx control queue is full, other negative error code on failure. 196 */ 197 int idpf_send_mb_msg(struct idpf_adapter *adapter, 198 struct libie_ctlq_xn_send_params *xn_params, 199 void *send_buf, size_t send_buf_size) 200 { 201 struct libie_ctlq_msg ctlq_msg = {}; 202 203 if (idpf_is_reset_detected(adapter)) { 204 if (!libie_cp_can_send_onstack(send_buf_size)) 205 kfree(send_buf); 206 207 return -EBUSY; 208 } 209 210 idpf_prepare_ptp_mb_msg(adapter, xn_params->chnl_opcode, &ctlq_msg); 211 xn_params->ctlq_msg = ctlq_msg.opcode ? &ctlq_msg : NULL; 212 213 xn_params->send_buf.iov_base = send_buf; 214 xn_params->send_buf.iov_len = send_buf_size; 215 xn_params->xnm = adapter->xnm; 216 xn_params->ctlq = xn_params->ctlq ? xn_params->ctlq : adapter->asq; 217 xn_params->rel_tx_buf = kfree; 218 219 idpf_mb_clean(xn_params->ctlq, false); 220 221 return libie_ctlq_xn_send(xn_params); 222 } 223 224 /** 225 * idpf_send_mb_msg_kfree - send mailbox message and free the send buffer 226 * @adapter: driver specific private structure 227 * @xn_params: Xn send parameters to fill 228 * @send_buf: buffer to send, can be released with kfree() 229 * @send_buf_size: size of the send buffer 230 * 231 * libie_cp functions consume only buffers above certain size, 232 * smaller buffers are assumed to be on the stack. However, for some 233 * commands with variable message size it makes sense to always use kzalloc(), 234 * which means we have to free smaller buffers ourselves. 235 * 236 * Return: 0 if no unexpected errors were encountered, 237 * negative error code otherwise. 238 */ 239 int idpf_send_mb_msg_kfree(struct idpf_adapter *adapter, 240 struct libie_ctlq_xn_send_params *xn_params, 241 void *send_buf, size_t send_buf_size) 242 { 243 int err = idpf_send_mb_msg(adapter, xn_params, send_buf, send_buf_size); 244 245 if (libie_cp_can_send_onstack(send_buf_size)) 246 kfree(send_buf); 247 248 return err; 249 } 250 251 /** 252 * idpf_send_vf_reset_msg - send one way VF reset message 253 * @adapter: driver specific private structure 254 */ 255 void idpf_send_vf_reset_msg(struct idpf_adapter *adapter) 256 { 257 struct libie_ctlq_info *ctlq = adapter->asq; 258 259 /* Forcefully claim send queue slot */ 260 idpf_mb_clean(ctlq, true); 261 262 scoped_guard(spinlock, &ctlq->lock) { 263 *ctlq->tx_msg[ctlq->next_to_use] = (struct libie_ctlq_msg) { 264 .opcode = LIBIE_CTLQ_SEND_MSG_TO_CP, 265 .chnl_opcode = VIRTCHNL2_OP_RESET_VF, 266 }; 267 268 libie_ctlq_send(adapter->asq, 1); 269 } 270 } 271 272 struct idpf_chunked_msg_params { 273 u32 (*prepare_msg)(u32 vport_id, void *buf, 274 const void *pos, u32 num); 275 276 const void *chunks; 277 u32 num_chunks; 278 279 u32 chunk_sz; 280 u32 config_sz; 281 282 u32 vc_op; 283 u32 vport_id; 284 }; 285 286 struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_adapter *adapter, 287 struct idpf_q_vec_rsrc *qv_rsrc, 288 u32 vport_id, u32 num) 289 { 290 struct idpf_queue_set *qp; 291 292 qp = kzalloc_flex(*qp, qs, num); 293 if (!qp) 294 return NULL; 295 296 qp->adapter = adapter; 297 qp->qv_rsrc = qv_rsrc; 298 qp->vport_id = vport_id; 299 qp->num = num; 300 301 return qp; 302 } 303 304 /** 305 * idpf_send_chunked_msg - send VC message consisting of chunks 306 * @adapter: Driver specific private structure 307 * @params: message params 308 * 309 * Helper function for preparing a message describing queues to be enabled 310 * or disabled. 311 * 312 * Return: the total size of the prepared message. 313 */ 314 static int idpf_send_chunked_msg(struct idpf_adapter *adapter, 315 const struct idpf_chunked_msg_params *params) 316 { 317 struct libie_ctlq_xn_send_params xn_params = { 318 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 319 .chnl_opcode = params->vc_op, 320 }; 321 const void *pos = params->chunks; 322 u32 totqs = params->num_chunks; 323 u32 vid = params->vport_id; 324 u32 num_chunks, num_msgs; 325 326 num_chunks = IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, 327 params->chunk_sz); 328 num_msgs = DIV_ROUND_UP(totqs, num_chunks); 329 330 for (u32 i = 0; i < num_msgs; i++) { 331 u32 buf_sz; 332 void *buf; 333 int err; 334 335 num_chunks = min(num_chunks, totqs); 336 buf_sz = params->config_sz + num_chunks * params->chunk_sz; 337 buf = kzalloc(buf_sz, GFP_KERNEL); 338 if (!buf) 339 return -ENOMEM; 340 341 if (params->prepare_msg(vid, buf, pos, num_chunks) != buf_sz) { 342 kfree(buf); 343 return -EINVAL; 344 } 345 346 err = idpf_send_mb_msg_kfree(adapter, &xn_params, buf, buf_sz); 347 if (err) 348 return err; 349 350 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 351 xn_params.recv_mem = (struct kvec) {}; 352 pos += num_chunks * params->chunk_sz; 353 totqs -= num_chunks; 354 } 355 356 return 0; 357 } 358 359 /** 360 * idpf_wait_for_marker_event_set - wait for software marker response for 361 * selected Tx queues 362 * @qs: set of the Tx queues 363 * 364 * Return: 0 success, -errno on failure. 365 */ 366 static int idpf_wait_for_marker_event_set(const struct idpf_queue_set *qs) 367 { 368 struct net_device *netdev; 369 struct idpf_tx_queue *txq; 370 bool markers_rcvd = true; 371 372 for (u32 i = 0; i < qs->num; i++) { 373 switch (qs->qs[i].type) { 374 case VIRTCHNL2_QUEUE_TYPE_TX: 375 txq = qs->qs[i].txq; 376 377 netdev = txq->netdev; 378 379 idpf_queue_set(SW_MARKER, txq); 380 idpf_wait_for_sw_marker_completion(txq); 381 markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); 382 break; 383 default: 384 break; 385 } 386 } 387 388 if (!markers_rcvd) { 389 netdev_warn(netdev, 390 "Failed to receive marker packets\n"); 391 return -ETIMEDOUT; 392 } 393 394 return 0; 395 } 396 397 /** 398 * idpf_wait_for_marker_event - wait for software marker response 399 * @vport: virtual port data structure 400 * 401 * Return: 0 success, negative on failure. 402 **/ 403 static int idpf_wait_for_marker_event(struct idpf_vport *vport) 404 { 405 struct idpf_queue_set *qs __free(kfree) = NULL; 406 407 qs = idpf_alloc_queue_set(vport->adapter, &vport->dflt_qv_rsrc, 408 vport->vport_id, vport->num_txq); 409 if (!qs) 410 return -ENOMEM; 411 412 for (u32 i = 0; i < qs->num; i++) { 413 qs->qs[i].type = VIRTCHNL2_QUEUE_TYPE_TX; 414 qs->qs[i].txq = vport->txqs[i]; 415 } 416 417 return idpf_wait_for_marker_event_set(qs); 418 } 419 420 /** 421 * idpf_send_ver_msg - send virtchnl version message 422 * @adapter: Driver specific private structure 423 * 424 * Send virtchnl version message. Returns 0 on success, negative on failure. 425 */ 426 static int idpf_send_ver_msg(struct idpf_adapter *adapter) 427 { 428 struct libie_ctlq_xn_send_params xn_params = { 429 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 430 .chnl_opcode = VIRTCHNL2_OP_VERSION, 431 }; 432 struct virtchnl2_version_info *vvi_recv; 433 struct virtchnl2_version_info vvi; 434 u32 major, minor; 435 int err; 436 437 if (adapter->virt_ver_maj) { 438 vvi.major = cpu_to_le32(adapter->virt_ver_maj); 439 vvi.minor = cpu_to_le32(adapter->virt_ver_min); 440 } else { 441 vvi.major = cpu_to_le32(IDPF_VIRTCHNL_VERSION_MAJOR); 442 vvi.minor = cpu_to_le32(IDPF_VIRTCHNL_VERSION_MINOR); 443 } 444 445 err = idpf_send_mb_msg_stack(adapter, &xn_params, &vvi); 446 if (err) 447 return err; 448 449 if (xn_params.recv_mem.iov_len < sizeof(*vvi_recv)) { 450 err = -EIO; 451 goto free_rx_buf; 452 } 453 454 vvi_recv = xn_params.recv_mem.iov_base; 455 major = le32_to_cpu(vvi_recv->major); 456 minor = le32_to_cpu(vvi_recv->minor); 457 458 if (major > IDPF_VIRTCHNL_VERSION_MAJOR) { 459 dev_warn(&adapter->pdev->dev, "Virtchnl major version greater than supported\n"); 460 err = -EINVAL; 461 goto free_rx_buf; 462 } 463 464 if (major == IDPF_VIRTCHNL_VERSION_MAJOR && 465 minor > IDPF_VIRTCHNL_VERSION_MINOR) 466 dev_warn(&adapter->pdev->dev, "Virtchnl minor version didn't match\n"); 467 468 /* If we have a mismatch, resend version to update receiver on what 469 * version we will use. 470 */ 471 if (!adapter->virt_ver_maj && 472 major != IDPF_VIRTCHNL_VERSION_MAJOR && 473 minor != IDPF_VIRTCHNL_VERSION_MINOR) 474 err = -EAGAIN; 475 476 adapter->virt_ver_maj = major; 477 adapter->virt_ver_min = minor; 478 479 free_rx_buf: 480 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 481 482 return err; 483 } 484 485 /** 486 * idpf_send_get_caps_msg - Send virtchnl get capabilities message 487 * @adapter: Driver specific private structure 488 * 489 * Send virtchl get capabilities message. Returns 0 on success, negative on 490 * failure. 491 */ 492 static int idpf_send_get_caps_msg(struct idpf_adapter *adapter) 493 { 494 struct libie_ctlq_xn_send_params xn_params = { 495 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 496 .chnl_opcode = VIRTCHNL2_OP_GET_CAPS, 497 }; 498 struct virtchnl2_get_capabilities caps = {}; 499 int err; 500 501 caps.csum_caps = 502 cpu_to_le32(VIRTCHNL2_CAP_TX_CSUM_L3_IPV4 | 503 VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_TCP | 504 VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_UDP | 505 VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_SCTP | 506 VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_TCP | 507 VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_UDP | 508 VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_SCTP | 509 VIRTCHNL2_CAP_RX_CSUM_L3_IPV4 | 510 VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_TCP | 511 VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_UDP | 512 VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_SCTP | 513 VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_TCP | 514 VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_UDP | 515 VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_SCTP | 516 VIRTCHNL2_CAP_TX_CSUM_L3_SINGLE_TUNNEL | 517 VIRTCHNL2_CAP_RX_CSUM_L3_SINGLE_TUNNEL | 518 VIRTCHNL2_CAP_TX_CSUM_L4_SINGLE_TUNNEL | 519 VIRTCHNL2_CAP_RX_CSUM_L4_SINGLE_TUNNEL | 520 VIRTCHNL2_CAP_RX_CSUM_GENERIC); 521 522 caps.seg_caps = 523 cpu_to_le32(VIRTCHNL2_CAP_SEG_IPV4_TCP | 524 VIRTCHNL2_CAP_SEG_IPV4_UDP | 525 VIRTCHNL2_CAP_SEG_IPV4_SCTP | 526 VIRTCHNL2_CAP_SEG_IPV6_TCP | 527 VIRTCHNL2_CAP_SEG_IPV6_UDP | 528 VIRTCHNL2_CAP_SEG_IPV6_SCTP | 529 VIRTCHNL2_CAP_SEG_TX_SINGLE_TUNNEL); 530 531 caps.rss_caps = 532 cpu_to_le64(VIRTCHNL2_FLOW_IPV4_TCP | 533 VIRTCHNL2_FLOW_IPV4_UDP | 534 VIRTCHNL2_FLOW_IPV4_SCTP | 535 VIRTCHNL2_FLOW_IPV4_OTHER | 536 VIRTCHNL2_FLOW_IPV6_TCP | 537 VIRTCHNL2_FLOW_IPV6_UDP | 538 VIRTCHNL2_FLOW_IPV6_SCTP | 539 VIRTCHNL2_FLOW_IPV6_OTHER); 540 541 caps.hsplit_caps = 542 cpu_to_le32(VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V4 | 543 VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V6); 544 545 caps.rsc_caps = 546 cpu_to_le32(VIRTCHNL2_CAP_RSC_IPV4_TCP | 547 VIRTCHNL2_CAP_RSC_IPV6_TCP); 548 549 caps.other_caps = 550 cpu_to_le64(VIRTCHNL2_CAP_SRIOV | 551 VIRTCHNL2_CAP_RDMA | 552 VIRTCHNL2_CAP_LAN_MEMORY_REGIONS | 553 VIRTCHNL2_CAP_MACFILTER | 554 VIRTCHNL2_CAP_SPLITQ_QSCHED | 555 VIRTCHNL2_CAP_PROMISC | 556 VIRTCHNL2_CAP_LOOPBACK | 557 VIRTCHNL2_CAP_PTP | 558 VIRTCHNL2_CAP_EDT); 559 560 err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); 561 if (err) 562 return err; 563 564 if (xn_params.recv_mem.iov_len < sizeof(adapter->caps)) { 565 err = -EIO; 566 goto free_rx_buf; 567 } 568 569 memcpy(&adapter->caps, xn_params.recv_mem.iov_base, 570 sizeof(adapter->caps)); 571 572 free_rx_buf: 573 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 574 575 return err; 576 } 577 578 /** 579 * idpf_send_get_edt_caps_msg - Send virtchnl get EDT caps msg 580 * @adapter: Driver specific private struct 581 * 582 * Return: 0 on success or error code on failure. 583 */ 584 static int idpf_send_get_edt_caps_msg(struct idpf_adapter *adapter) 585 { 586 struct libie_ctlq_xn_send_params xn_params = { 587 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 588 .chnl_opcode = VIRTCHNL2_OP_GET_EDT_CAPS, 589 }; 590 struct virtchnl2_edt_caps caps = {}; 591 u64 gran_ns, horizon_ns; 592 int err; 593 594 err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); 595 if (err) 596 return err; 597 598 if (xn_params.recv_mem.iov_len < sizeof(caps)) { 599 err = -EIO; 600 goto free_rx_buf; 601 } 602 603 memcpy(&caps, xn_params.recv_mem.iov_base, sizeof(caps)); 604 horizon_ns = le64_to_cpu(caps.time_horizon_ns); 605 gran_ns = le64_to_cpu(caps.tstamp_granularity_ns); 606 if (horizon_ns > U32_MAX) { 607 dev_warn(&adapter->pdev->dev, "EDT horizon exceeds U32\n"); 608 err = -EINVAL; 609 goto free_rx_buf; 610 } 611 if (!gran_ns || !is_power_of_2(gran_ns)) { 612 dev_warn(&adapter->pdev->dev, "Invalid EDT granularity\n"); 613 err = -EINVAL; 614 goto free_rx_buf; 615 } 616 617 adapter->edt_caps.time_horizon_ns = horizon_ns; 618 adapter->edt_caps.tstamp_granularity_pow2 = ilog2(gran_ns); 619 620 free_rx_buf: 621 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 622 623 return err; 624 } 625 626 /** 627 * idpf_mmio_region_non_static - Check if region is not static 628 * @mmio_info: PCI resources info 629 * @reg: region to check 630 * 631 * Return: %true if region can be received though virtchnl command, 632 * %false if region is related to mailbox or resetting 633 */ 634 bool idpf_mmio_region_non_static(struct libie_mmio_info *mmio_info, 635 struct libie_pci_mmio_region *reg) 636 { 637 struct idpf_adapter *adapter = 638 container_of(mmio_info, struct idpf_adapter, 639 ctlq_ctx.mmio_info); 640 641 for (uint i = 0; i < IDPF_MMIO_REG_NUM_STATIC; i++) { 642 if (reg->bar_idx == 0 && 643 reg->offset == adapter->dev_ops.static_reg_info[i].start) 644 return false; 645 } 646 647 return true; 648 } 649 650 /** 651 * idpf_decfg_lan_memory_regions - Unmap non-static memory regions 652 * @adapter: Driver specific private structure 653 */ 654 static void idpf_decfg_lan_memory_regions(struct idpf_adapter *adapter) 655 { 656 libie_pci_unmap_fltr_regs(&adapter->ctlq_ctx.mmio_info, 657 idpf_mmio_region_non_static); 658 } 659 660 /** 661 * idpf_cfg_lan_memory_regions - Get (via virtchnl) and map LAN memory regions 662 * @adapter: Driver specific private struct 663 * 664 * Return: 0 on success or error code on failure. 665 */ 666 static int idpf_cfg_lan_memory_regions(struct idpf_adapter *adapter) 667 { 668 struct virtchnl2_get_lan_memory_regions *send_regions, *rcvd_regions; 669 struct libie_ctlq_xn_send_params xn_params = { 670 .chnl_opcode = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS, 671 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 672 }; 673 size_t send_sz, reply_sz, size; 674 int num_regions; 675 int err = 0; 676 677 send_sz = sizeof(struct virtchnl2_get_lan_memory_regions) + 678 sizeof(struct virtchnl2_mem_region); 679 send_regions = kzalloc(send_sz, GFP_KERNEL); 680 if (!send_regions) 681 return -ENOMEM; 682 683 send_regions->num_memory_regions = cpu_to_le16(1); 684 err = idpf_send_mb_msg_kfree(adapter, &xn_params, send_regions, 685 send_sz); 686 if (err) 687 return err; 688 689 rcvd_regions = xn_params.recv_mem.iov_base; 690 reply_sz = xn_params.recv_mem.iov_len; 691 if (reply_sz < sizeof(*rcvd_regions)) { 692 err = -EIO; 693 goto rel_rx_buf; 694 } 695 num_regions = le16_to_cpu(rcvd_regions->num_memory_regions); 696 size = struct_size(rcvd_regions, mem_reg, num_regions); 697 if (reply_sz < size) { 698 err = -EIO; 699 goto rel_rx_buf; 700 } 701 702 for (int i = 0; i < num_regions; i++) { 703 struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; 704 resource_size_t offset, len; 705 706 offset = le64_to_cpu(rcvd_regions->mem_reg[i].start_offset); 707 len = le64_to_cpu(rcvd_regions->mem_reg[i].size); 708 if (len && !libie_pci_map_mmio_region(mmio, offset, len)) { 709 idpf_decfg_lan_memory_regions(adapter); 710 err = -EIO; 711 goto rel_rx_buf; 712 } 713 } 714 715 rel_rx_buf: 716 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 717 718 return err; 719 } 720 721 /** 722 * idpf_map_remaining_mmio_regs - map MMIO regions outside mbx and rstat 723 * @adapter: Driver specific private structure 724 * 725 * Called when idpf_cfg_lan_memory_regions is not supported. This will 726 * calculate the offsets and sizes for the regions before, in between, and 727 * after the mailbox and rstat MMIO mappings, and map those ranges. 728 * 729 * Return: 0 on success or error code on failure. 730 */ 731 static int idpf_map_remaining_mmio_regs(struct idpf_adapter *adapter) 732 { 733 struct resource *rstat_reg = &adapter->dev_ops.static_reg_info[1]; 734 struct resource *mbx_reg = &adapter->dev_ops.static_reg_info[0]; 735 struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; 736 resource_size_t reg_start, size; 737 bool ok = true; 738 739 /* Region preceding mailbox */ 740 size = mbx_reg->start; 741 ok &= !size || libie_pci_map_mmio_region(mmio, 0, size); 742 743 /* Region between mailbox and rstat */ 744 reg_start = mbx_reg->end + 1; 745 size = rstat_reg->start - reg_start; 746 ok &= !size || libie_pci_map_mmio_region(mmio, reg_start, size); 747 748 /* Region after rstat */ 749 reg_start = rstat_reg->end + 1; 750 size = pci_resource_len(adapter->pdev, 0) - reg_start; 751 ok &= !size || libie_pci_map_mmio_region(mmio, reg_start, size); 752 753 if (!ok) { 754 idpf_decfg_lan_memory_regions(adapter); 755 return -ENOMEM; 756 } 757 758 return 0; 759 } 760 761 /** 762 * idpf_add_del_fsteer_filters - Send virtchnl add/del Flow Steering message 763 * @adapter: adapter info struct 764 * @rule: Flow steering rule to add/delete 765 * @opcode: VIRTCHNL2_OP_ADD_FLOW_RULE to add filter, or 766 * VIRTCHNL2_OP_DEL_FLOW_RULE to delete. All other values are invalid. 767 * 768 * Send ADD/DELETE flow steering virtchnl message and receive the result. 769 * 770 * Return: 0 on success, negative on failure. 771 */ 772 int idpf_add_del_fsteer_filters(struct idpf_adapter *adapter, 773 struct virtchnl2_flow_rule_add_del *rule, 774 enum virtchnl2_op opcode) 775 { 776 struct libie_ctlq_xn_send_params xn_params = { 777 .chnl_opcode = opcode, 778 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 779 }; 780 struct virtchnl2_flow_rule_add_del *rx_rule; 781 int rule_count = le32_to_cpu(rule->count); 782 size_t send_sz; 783 int err; 784 785 if (opcode != VIRTCHNL2_OP_ADD_FLOW_RULE && 786 opcode != VIRTCHNL2_OP_DEL_FLOW_RULE) { 787 kfree(rule); 788 return -EINVAL; 789 } 790 791 send_sz = struct_size(rule, rule_info, rule_count); 792 err = idpf_send_mb_msg_kfree(adapter, &xn_params, rule, send_sz); 793 if (err) 794 return err; 795 796 if (xn_params.recv_mem.iov_len < send_sz) { 797 err = -EIO; 798 goto rel_rx; 799 } 800 801 rx_rule = xn_params.recv_mem.iov_base; 802 for (int i = 0; i < rule_count; i++) { 803 if (rx_rule->rule_info[i].status != 804 cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) { 805 err = -EIO; 806 goto rel_rx; 807 } 808 } 809 810 rel_rx: 811 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 812 return err; 813 } 814 815 /** 816 * idpf_vport_alloc_max_qs - Allocate max queues for a vport 817 * @adapter: Driver specific private structure 818 * @max_q: vport max queue structure 819 */ 820 int idpf_vport_alloc_max_qs(struct idpf_adapter *adapter, 821 struct idpf_vport_max_q *max_q) 822 { 823 struct idpf_avail_queue_info *avail_queues = &adapter->avail_queues; 824 struct virtchnl2_get_capabilities *caps = &adapter->caps; 825 u16 default_vports = idpf_get_default_vports(adapter); 826 u32 max_rx_q, max_tx_q, max_buf_q, max_compl_q; 827 828 mutex_lock(&adapter->queue_lock); 829 830 /* Caps are device-wide. Give each vport an equal piece */ 831 max_rx_q = le16_to_cpu(caps->max_rx_q) / default_vports; 832 max_tx_q = le16_to_cpu(caps->max_tx_q) / default_vports; 833 max_buf_q = le16_to_cpu(caps->max_rx_bufq) / default_vports; 834 max_compl_q = le16_to_cpu(caps->max_tx_complq) / default_vports; 835 836 if (adapter->num_alloc_vports >= default_vports) { 837 max_rx_q = IDPF_MIN_Q; 838 max_tx_q = IDPF_MIN_Q; 839 } 840 841 /* 842 * Harmonize the numbers. The current implementation always creates 843 * `IDPF_MAX_BUFQS_PER_RXQ_GRP` buffer queues for each Rx queue and 844 * one completion queue for each Tx queue for best performance. 845 * If less buffer or completion queues is available, cap the number 846 * of the corresponding Rx/Tx queues. 847 */ 848 max_rx_q = min(max_rx_q, max_buf_q / IDPF_MAX_BUFQS_PER_RXQ_GRP); 849 max_tx_q = min(max_tx_q, max_compl_q); 850 851 max_q->max_rxq = max_rx_q; 852 max_q->max_txq = max_tx_q; 853 max_q->max_bufq = max_rx_q * IDPF_MAX_BUFQS_PER_RXQ_GRP; 854 max_q->max_complq = max_tx_q; 855 856 if (avail_queues->avail_rxq < max_q->max_rxq || 857 avail_queues->avail_txq < max_q->max_txq || 858 avail_queues->avail_bufq < max_q->max_bufq || 859 avail_queues->avail_complq < max_q->max_complq) { 860 mutex_unlock(&adapter->queue_lock); 861 862 return -EINVAL; 863 } 864 865 avail_queues->avail_rxq -= max_q->max_rxq; 866 avail_queues->avail_txq -= max_q->max_txq; 867 avail_queues->avail_bufq -= max_q->max_bufq; 868 avail_queues->avail_complq -= max_q->max_complq; 869 870 mutex_unlock(&adapter->queue_lock); 871 872 return 0; 873 } 874 875 /** 876 * idpf_vport_dealloc_max_qs - Deallocate max queues of a vport 877 * @adapter: Driver specific private structure 878 * @max_q: vport max queue structure 879 */ 880 void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter, 881 struct idpf_vport_max_q *max_q) 882 { 883 struct idpf_avail_queue_info *avail_queues; 884 885 mutex_lock(&adapter->queue_lock); 886 avail_queues = &adapter->avail_queues; 887 888 avail_queues->avail_rxq += max_q->max_rxq; 889 avail_queues->avail_txq += max_q->max_txq; 890 avail_queues->avail_bufq += max_q->max_bufq; 891 avail_queues->avail_complq += max_q->max_complq; 892 893 mutex_unlock(&adapter->queue_lock); 894 } 895 896 /** 897 * idpf_init_avail_queues - Initialize available queues on the device 898 * @adapter: Driver specific private structure 899 */ 900 static void idpf_init_avail_queues(struct idpf_adapter *adapter) 901 { 902 struct idpf_avail_queue_info *avail_queues = &adapter->avail_queues; 903 struct virtchnl2_get_capabilities *caps = &adapter->caps; 904 905 avail_queues->avail_rxq = le16_to_cpu(caps->max_rx_q); 906 avail_queues->avail_txq = le16_to_cpu(caps->max_tx_q); 907 avail_queues->avail_bufq = le16_to_cpu(caps->max_rx_bufq); 908 avail_queues->avail_complq = le16_to_cpu(caps->max_tx_complq); 909 } 910 911 /** 912 * idpf_vport_init_queue_reg_chunks - initialize queue register chunks 913 * @vport_config: persistent vport structure to store the queue register info 914 * @schunks: source chunks to copy data from 915 * 916 * Return: 0 on success, negative on failure. 917 */ 918 static int 919 idpf_vport_init_queue_reg_chunks(struct idpf_vport_config *vport_config, 920 struct virtchnl2_queue_reg_chunks *schunks) 921 { 922 struct idpf_queue_id_reg_info *q_info = &vport_config->qid_reg_info; 923 u16 num_chunks = le16_to_cpu(schunks->num_chunks); 924 925 kfree(q_info->queue_chunks); 926 927 q_info->queue_chunks = kzalloc_objs(*q_info->queue_chunks, num_chunks); 928 if (!q_info->queue_chunks) { 929 q_info->num_chunks = 0; 930 return -ENOMEM; 931 } 932 933 q_info->num_chunks = num_chunks; 934 935 for (u16 i = 0; i < num_chunks; i++) { 936 struct idpf_queue_id_reg_chunk *dchunk = &q_info->queue_chunks[i]; 937 struct virtchnl2_queue_reg_chunk *schunk = &schunks->chunks[i]; 938 939 dchunk->qtail_reg_start = le64_to_cpu(schunk->qtail_reg_start); 940 dchunk->qtail_reg_spacing = le32_to_cpu(schunk->qtail_reg_spacing); 941 dchunk->type = le32_to_cpu(schunk->type); 942 dchunk->start_queue_id = le32_to_cpu(schunk->start_queue_id); 943 dchunk->num_queues = le32_to_cpu(schunk->num_queues); 944 } 945 946 return 0; 947 } 948 949 /** 950 * idpf_get_reg_intr_vecs - Get vector queue register offset 951 * @adapter: adapter structure to get the vector chunks 952 * @reg_vals: Register offsets to store in 953 * @num_vecs: number of entries the @reg_vals array can hold 954 * 955 * Return: number of registers that got populated 956 */ 957 int idpf_get_reg_intr_vecs(struct idpf_adapter *adapter, 958 struct idpf_vec_regs *reg_vals, int num_vecs) 959 { 960 struct virtchnl2_vector_chunks *chunks; 961 struct idpf_vec_regs reg_val; 962 u16 num_vchunks, num_vec; 963 int num_regs = 0, i, j; 964 965 chunks = &adapter->req_vec_chunks->vchunks; 966 num_vchunks = le16_to_cpu(chunks->num_vchunks); 967 968 for (j = 0; j < num_vchunks; j++) { 969 struct virtchnl2_vector_chunk *chunk; 970 u32 dynctl_reg_spacing; 971 u32 itrn_reg_spacing; 972 973 chunk = &chunks->vchunks[j]; 974 num_vec = le16_to_cpu(chunk->num_vectors); 975 reg_val.dyn_ctl_reg = le32_to_cpu(chunk->dynctl_reg_start); 976 reg_val.itrn_reg = le32_to_cpu(chunk->itrn_reg_start); 977 reg_val.itrn_index_spacing = le32_to_cpu(chunk->itrn_index_spacing); 978 979 dynctl_reg_spacing = le32_to_cpu(chunk->dynctl_reg_spacing); 980 itrn_reg_spacing = le32_to_cpu(chunk->itrn_reg_spacing); 981 982 for (i = 0; i < num_vec && num_regs < num_vecs; i++) { 983 reg_vals[num_regs].dyn_ctl_reg = reg_val.dyn_ctl_reg; 984 reg_vals[num_regs].itrn_reg = reg_val.itrn_reg; 985 reg_vals[num_regs].itrn_index_spacing = 986 reg_val.itrn_index_spacing; 987 988 reg_val.dyn_ctl_reg += dynctl_reg_spacing; 989 reg_val.itrn_reg += itrn_reg_spacing; 990 num_regs++; 991 } 992 } 993 994 return num_regs; 995 } 996 997 /** 998 * idpf_vport_get_q_reg - Get the queue registers for the vport 999 * @reg_vals: register values needing to be set 1000 * @num_regs: amount we expect to fill 1001 * @q_type: queue model 1002 * @chunks: queue regs received over mailbox 1003 * 1004 * This function parses the queue register offsets from the queue register 1005 * chunk information, with a specific queue type and stores it into the array 1006 * passed as an argument. It returns the actual number of queue registers that 1007 * are filled. 1008 */ 1009 static int idpf_vport_get_q_reg(u32 *reg_vals, int num_regs, u32 q_type, 1010 struct idpf_queue_id_reg_info *chunks) 1011 { 1012 u16 num_chunks = chunks->num_chunks; 1013 int reg_filled = 0, i; 1014 u32 reg_val; 1015 1016 while (num_chunks--) { 1017 struct idpf_queue_id_reg_chunk *chunk; 1018 u16 num_q; 1019 1020 chunk = &chunks->queue_chunks[num_chunks]; 1021 if (chunk->type != q_type) 1022 continue; 1023 1024 num_q = chunk->num_queues; 1025 reg_val = chunk->qtail_reg_start; 1026 for (i = 0; i < num_q && reg_filled < num_regs ; i++) { 1027 reg_vals[reg_filled++] = reg_val; 1028 reg_val += chunk->qtail_reg_spacing; 1029 } 1030 } 1031 1032 return reg_filled; 1033 } 1034 1035 /** 1036 * __idpf_queue_reg_init - initialize queue registers 1037 * @vport: virtual port structure 1038 * @rsrc: pointer to queue and vector resources 1039 * @reg_vals: registers we are initializing 1040 * @num_regs: how many registers there are in total 1041 * @q_type: queue model 1042 * 1043 * Return number of queues that are initialized 1044 */ 1045 static int __idpf_queue_reg_init(struct idpf_vport *vport, 1046 struct idpf_q_vec_rsrc *rsrc, u32 *reg_vals, 1047 int num_regs, u32 q_type) 1048 { 1049 struct libie_mmio_info *mmio = &vport->adapter->ctlq_ctx.mmio_info; 1050 int i, j, k = 0; 1051 1052 switch (q_type) { 1053 case VIRTCHNL2_QUEUE_TYPE_TX: 1054 for (i = 0; i < rsrc->num_txq_grp; i++) { 1055 struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 1056 1057 for (j = 0; j < tx_qgrp->num_txq && k < num_regs; j++, k++) 1058 tx_qgrp->txqs[j]->tail = 1059 libie_pci_get_mmio_addr(mmio, 1060 reg_vals[k]); 1061 } 1062 break; 1063 case VIRTCHNL2_QUEUE_TYPE_RX: 1064 for (i = 0; i < rsrc->num_rxq_grp; i++) { 1065 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1066 u16 num_rxq = rx_qgrp->singleq.num_rxq; 1067 1068 for (j = 0; j < num_rxq && k < num_regs; j++, k++) { 1069 struct idpf_rx_queue *q; 1070 1071 q = rx_qgrp->singleq.rxqs[j]; 1072 q->tail = libie_pci_get_mmio_addr(mmio, 1073 reg_vals[k]); 1074 } 1075 } 1076 break; 1077 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1078 for (i = 0; i < rsrc->num_rxq_grp; i++) { 1079 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1080 u8 num_bufqs = rsrc->num_bufqs_per_qgrp; 1081 1082 for (j = 0; j < num_bufqs && k < num_regs; j++, k++) { 1083 struct idpf_buf_queue *q; 1084 1085 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 1086 q->tail = libie_pci_get_mmio_addr(mmio, 1087 reg_vals[k]); 1088 } 1089 } 1090 break; 1091 default: 1092 break; 1093 } 1094 1095 return k; 1096 } 1097 1098 /** 1099 * idpf_queue_reg_init - initialize queue registers 1100 * @vport: virtual port structure 1101 * @rsrc: pointer to queue and vector resources 1102 * @chunks: queue registers received over mailbox 1103 * 1104 * Return: 0 on success, negative on failure 1105 */ 1106 int idpf_queue_reg_init(struct idpf_vport *vport, 1107 struct idpf_q_vec_rsrc *rsrc, 1108 struct idpf_queue_id_reg_info *chunks) 1109 { 1110 int num_regs, ret = 0; 1111 u32 *reg_vals; 1112 1113 /* We may never deal with more than 256 same type of queues */ 1114 reg_vals = kzalloc(sizeof(void *) * IDPF_LARGE_MAX_Q, GFP_KERNEL); 1115 if (!reg_vals) 1116 return -ENOMEM; 1117 1118 /* Initialize Tx queue tail register address */ 1119 num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, 1120 VIRTCHNL2_QUEUE_TYPE_TX, 1121 chunks); 1122 if (num_regs < rsrc->num_txq) { 1123 ret = -EINVAL; 1124 goto free_reg_vals; 1125 } 1126 1127 num_regs = __idpf_queue_reg_init(vport, rsrc, reg_vals, num_regs, 1128 VIRTCHNL2_QUEUE_TYPE_TX); 1129 if (num_regs < rsrc->num_txq) { 1130 ret = -EINVAL; 1131 goto free_reg_vals; 1132 } 1133 1134 /* Initialize Rx/buffer queue tail register address based on Rx queue 1135 * model 1136 */ 1137 if (idpf_is_queue_model_split(rsrc->rxq_model)) { 1138 num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, 1139 VIRTCHNL2_QUEUE_TYPE_RX_BUFFER, 1140 chunks); 1141 if (num_regs < rsrc->num_bufq) { 1142 ret = -EINVAL; 1143 goto free_reg_vals; 1144 } 1145 1146 num_regs = __idpf_queue_reg_init(vport, rsrc, reg_vals, num_regs, 1147 VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 1148 if (num_regs < rsrc->num_bufq) { 1149 ret = -EINVAL; 1150 goto free_reg_vals; 1151 } 1152 } else { 1153 num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, 1154 VIRTCHNL2_QUEUE_TYPE_RX, 1155 chunks); 1156 if (num_regs < rsrc->num_rxq) { 1157 ret = -EINVAL; 1158 goto free_reg_vals; 1159 } 1160 1161 num_regs = __idpf_queue_reg_init(vport, rsrc, reg_vals, num_regs, 1162 VIRTCHNL2_QUEUE_TYPE_RX); 1163 if (num_regs < rsrc->num_rxq) { 1164 ret = -EINVAL; 1165 goto free_reg_vals; 1166 } 1167 } 1168 1169 free_reg_vals: 1170 kfree(reg_vals); 1171 1172 return ret; 1173 } 1174 1175 /** 1176 * idpf_send_create_vport_msg - Send virtchnl create vport message 1177 * @adapter: Driver specific private structure 1178 * @max_q: vport max queue info 1179 * 1180 * send virtchnl creae vport message 1181 * 1182 * Returns 0 on success, negative on failure 1183 */ 1184 int idpf_send_create_vport_msg(struct idpf_adapter *adapter, 1185 struct idpf_vport_max_q *max_q) 1186 { 1187 struct libie_ctlq_xn_send_params xn_params = { 1188 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1189 .chnl_opcode = VIRTCHNL2_OP_CREATE_VPORT, 1190 }; 1191 struct virtchnl2_create_vport *vport_msg; 1192 u16 idx = adapter->next_vport; 1193 int err, buf_size; 1194 1195 buf_size = sizeof(struct virtchnl2_create_vport); 1196 vport_msg = kzalloc(buf_size, GFP_KERNEL); 1197 if (!vport_msg) 1198 return -ENOMEM; 1199 1200 vport_msg->vport_type = cpu_to_le16(VIRTCHNL2_VPORT_TYPE_DEFAULT); 1201 vport_msg->vport_index = cpu_to_le16(idx); 1202 1203 if (adapter->req_tx_splitq || !IS_ENABLED(CONFIG_IDPF_SINGLEQ)) 1204 vport_msg->txq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SPLIT); 1205 else 1206 vport_msg->txq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SINGLE); 1207 1208 if (adapter->req_rx_splitq || !IS_ENABLED(CONFIG_IDPF_SINGLEQ)) 1209 vport_msg->rxq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SPLIT); 1210 else 1211 vport_msg->rxq_model = cpu_to_le16(VIRTCHNL2_QUEUE_MODEL_SINGLE); 1212 1213 err = idpf_vport_calc_total_qs(adapter, idx, vport_msg, max_q); 1214 if (err) { 1215 dev_err(&adapter->pdev->dev, "Enough queues are not available"); 1216 goto rel_buf; 1217 } 1218 1219 if (!adapter->vport_params_recvd[idx]) { 1220 adapter->vport_params_recvd[idx] = 1221 kzalloc(LIBIE_CTLQ_MAX_BUF_LEN, GFP_KERNEL); 1222 if (!adapter->vport_params_recvd[idx]) { 1223 err = -ENOMEM; 1224 goto rel_buf; 1225 } 1226 } 1227 1228 err = idpf_send_mb_msg_kfree(adapter, &xn_params, vport_msg, 1229 sizeof(*vport_msg)); 1230 if (err) { 1231 kfree(adapter->vport_params_recvd[idx]); 1232 adapter->vport_params_recvd[idx] = NULL; 1233 return err; 1234 } 1235 1236 memcpy(adapter->vport_params_recvd[idx], xn_params.recv_mem.iov_base, 1237 xn_params.recv_mem.iov_len); 1238 1239 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1240 1241 return 0; 1242 1243 rel_buf: 1244 kfree(vport_msg); 1245 1246 return err; 1247 } 1248 1249 /** 1250 * idpf_check_supported_desc_ids - Verify we have required descriptor support 1251 * @vport: virtual port structure 1252 * 1253 * Return 0 on success, error on failure 1254 */ 1255 int idpf_check_supported_desc_ids(struct idpf_vport *vport) 1256 { 1257 struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; 1258 struct idpf_adapter *adapter = vport->adapter; 1259 struct virtchnl2_create_vport *vport_msg; 1260 u64 rx_desc_ids, tx_desc_ids; 1261 1262 vport_msg = adapter->vport_params_recvd[vport->idx]; 1263 1264 if (!IS_ENABLED(CONFIG_IDPF_SINGLEQ) && 1265 (vport_msg->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE || 1266 vport_msg->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE)) { 1267 pci_err(adapter->pdev, "singleq mode requested, but not compiled-in\n"); 1268 return -EOPNOTSUPP; 1269 } 1270 1271 rx_desc_ids = le64_to_cpu(vport_msg->rx_desc_ids); 1272 tx_desc_ids = le64_to_cpu(vport_msg->tx_desc_ids); 1273 1274 if (idpf_is_queue_model_split(rsrc->rxq_model)) { 1275 if (!(rx_desc_ids & VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M)) { 1276 dev_info(&adapter->pdev->dev, "Minimum RX descriptor support not provided, using the default\n"); 1277 vport_msg->rx_desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1278 } 1279 } else { 1280 if (!(rx_desc_ids & VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M)) 1281 rsrc->base_rxd = true; 1282 } 1283 1284 if (!idpf_is_queue_model_split(rsrc->txq_model)) 1285 return 0; 1286 1287 if ((tx_desc_ids & MIN_SUPPORT_TXDID) != MIN_SUPPORT_TXDID) { 1288 dev_info(&adapter->pdev->dev, "Minimum TX descriptor support not provided, using the default\n"); 1289 vport_msg->tx_desc_ids = cpu_to_le64(MIN_SUPPORT_TXDID); 1290 } 1291 1292 return 0; 1293 } 1294 1295 /** 1296 * idpf_send_destroy_vport_msg - Send virtchnl destroy vport message 1297 * @adapter: adapter pointer used to send virtchnl message 1298 * @vport_id: vport identifier used while preparing the virtchnl message 1299 * 1300 * Return: 0 on success, negative on failure. 1301 */ 1302 int idpf_send_destroy_vport_msg(struct idpf_adapter *adapter, u32 vport_id) 1303 { 1304 struct libie_ctlq_xn_send_params xn_params = { 1305 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1306 .chnl_opcode = VIRTCHNL2_OP_DESTROY_VPORT, 1307 }; 1308 struct virtchnl2_vport v_id; 1309 int err; 1310 1311 v_id.vport_id = cpu_to_le32(vport_id); 1312 1313 err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); 1314 if (err) 1315 return err; 1316 1317 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1318 1319 return 0; 1320 } 1321 1322 /** 1323 * idpf_send_enable_vport_msg - Send virtchnl enable vport message 1324 * @adapter: adapter pointer used to send virtchnl message 1325 * @vport_id: vport identifier used while preparing the virtchnl message 1326 * 1327 * Return: 0 on success, negative on failure. 1328 */ 1329 int idpf_send_enable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) 1330 { 1331 struct libie_ctlq_xn_send_params xn_params = { 1332 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1333 .chnl_opcode = VIRTCHNL2_OP_ENABLE_VPORT, 1334 }; 1335 struct virtchnl2_vport v_id; 1336 int err; 1337 1338 v_id.vport_id = cpu_to_le32(vport_id); 1339 1340 err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); 1341 if (err) 1342 return err; 1343 1344 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1345 1346 return 0; 1347 } 1348 1349 /** 1350 * idpf_send_disable_vport_msg - Send virtchnl disable vport message 1351 * @adapter: adapter pointer used to send virtchnl message 1352 * @vport_id: vport identifier used while preparing the virtchnl message 1353 * 1354 * Return: 0 on success, negative on failure. 1355 */ 1356 int idpf_send_disable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) 1357 { 1358 struct libie_ctlq_xn_send_params xn_params = { 1359 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1360 .chnl_opcode = VIRTCHNL2_OP_DISABLE_VPORT, 1361 }; 1362 struct virtchnl2_vport v_id; 1363 int err; 1364 1365 v_id.vport_id = cpu_to_le32(vport_id); 1366 1367 err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); 1368 if (err) 1369 return err; 1370 1371 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 1372 1373 return 0; 1374 } 1375 1376 /** 1377 * idpf_fill_txq_config_chunk - fill chunk describing the Tx queue 1378 * @rsrc: pointer to queue and vector resources 1379 * @q: Tx queue to be inserted into VC chunk 1380 * @qi: pointer to the buffer containing the VC chunk 1381 */ 1382 static void idpf_fill_txq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1383 const struct idpf_tx_queue *q, 1384 struct virtchnl2_txq_info *qi) 1385 { 1386 u32 val; 1387 1388 qi->queue_id = cpu_to_le32(q->q_id); 1389 qi->model = cpu_to_le16(rsrc->txq_model); 1390 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 1391 qi->ring_len = cpu_to_le16(q->desc_count); 1392 qi->dma_ring_addr = cpu_to_le64(q->dma); 1393 qi->relative_queue_id = cpu_to_le16(q->rel_q_id); 1394 1395 if (!idpf_is_queue_model_split(rsrc->txq_model)) { 1396 qi->sched_mode = cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); 1397 return; 1398 } 1399 1400 if (idpf_queue_has(XDP, q)) 1401 val = q->complq->q_id; 1402 else 1403 val = q->txq_grp->complq->q_id; 1404 1405 qi->tx_compl_queue_id = cpu_to_le16(val); 1406 1407 if (idpf_queue_has(FLOW_SCH_EN, q)) 1408 val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1409 else 1410 val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1411 1412 qi->sched_mode = cpu_to_le16(val); 1413 } 1414 1415 /** 1416 * idpf_fill_complq_config_chunk - fill chunk describing the completion queue 1417 * @rsrc: pointer to queue and vector resources 1418 * @q: completion queue to be inserted into VC chunk 1419 * @qi: pointer to the buffer containing the VC chunk 1420 */ 1421 static void idpf_fill_complq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1422 const struct idpf_compl_queue *q, 1423 struct virtchnl2_txq_info *qi) 1424 { 1425 u32 val; 1426 1427 qi->queue_id = cpu_to_le32(q->q_id); 1428 qi->model = cpu_to_le16(rsrc->txq_model); 1429 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 1430 qi->ring_len = cpu_to_le16(q->desc_count); 1431 qi->dma_ring_addr = cpu_to_le64(q->dma); 1432 1433 if (idpf_queue_has(FLOW_SCH_EN, q)) 1434 val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1435 else 1436 val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1437 1438 qi->sched_mode = cpu_to_le16(val); 1439 } 1440 1441 /** 1442 * idpf_prepare_cfg_txqs_msg - prepare message to configure selected Tx queues 1443 * @vport_id: ID of virtual port queues are associated with 1444 * @buf: buffer containing the message 1445 * @pos: pointer to the first chunk describing the tx queue 1446 * @num_chunks: number of chunks in the message 1447 * 1448 * Helper function for preparing the message describing configuration of 1449 * Tx queues. 1450 * 1451 * Return: the total size of the prepared message. 1452 */ 1453 static u32 idpf_prepare_cfg_txqs_msg(u32 vport_id, void *buf, const void *pos, 1454 u32 num_chunks) 1455 { 1456 struct virtchnl2_config_tx_queues *ctq = buf; 1457 1458 ctq->vport_id = cpu_to_le32(vport_id); 1459 ctq->num_qinfo = cpu_to_le16(num_chunks); 1460 memcpy(ctq->qinfo, pos, num_chunks * sizeof(*ctq->qinfo)); 1461 1462 return struct_size(ctq, qinfo, num_chunks); 1463 } 1464 1465 /** 1466 * idpf_send_config_tx_queue_set_msg - send virtchnl config Tx queues 1467 * message for selected queues 1468 * @qs: set of the Tx queues to configure 1469 * 1470 * Send config queues virtchnl message for queues contained in the @qs array. 1471 * The @qs array can contain Tx queues (or completion queues) only. 1472 * 1473 * Return: 0 on success, -errno on failure. 1474 */ 1475 static int idpf_send_config_tx_queue_set_msg(const struct idpf_queue_set *qs) 1476 { 1477 struct virtchnl2_txq_info *qi __free(kfree) = NULL; 1478 struct idpf_chunked_msg_params params = { 1479 .vport_id = qs->vport_id, 1480 .vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES, 1481 .prepare_msg = idpf_prepare_cfg_txqs_msg, 1482 .config_sz = sizeof(struct virtchnl2_config_tx_queues), 1483 .chunk_sz = sizeof(*qi), 1484 }; 1485 1486 qi = kzalloc_objs(*qi, qs->num); 1487 if (!qi) 1488 return -ENOMEM; 1489 1490 params.chunks = qi; 1491 1492 for (u32 i = 0; i < qs->num; i++) { 1493 if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX) 1494 idpf_fill_txq_config_chunk(qs->qv_rsrc, qs->qs[i].txq, 1495 &qi[params.num_chunks++]); 1496 else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION) 1497 idpf_fill_complq_config_chunk(qs->qv_rsrc, 1498 qs->qs[i].complq, 1499 &qi[params.num_chunks++]); 1500 } 1501 1502 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1503 } 1504 1505 /** 1506 * idpf_send_config_tx_queues_msg - send virtchnl config Tx queues message 1507 * @adapter: adapter pointer used to send virtchnl message 1508 * @rsrc: pointer to queue and vector resources 1509 * @vport_id: vport identifier used while preparing the virtchnl message 1510 * 1511 * Return: 0 on success, -errno on failure. 1512 */ 1513 static int idpf_send_config_tx_queues_msg(struct idpf_adapter *adapter, 1514 struct idpf_q_vec_rsrc *rsrc, 1515 u32 vport_id) 1516 { 1517 struct idpf_queue_set *qs __free(kfree) = NULL; 1518 u32 totqs = rsrc->num_txq + rsrc->num_complq; 1519 u32 k = 0; 1520 1521 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, totqs); 1522 if (!qs) 1523 return -ENOMEM; 1524 1525 /* Populate the queue info buffer with all queue context info */ 1526 for (u32 i = 0; i < rsrc->num_txq_grp; i++) { 1527 const struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 1528 1529 for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 1530 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 1531 qs->qs[k++].txq = tx_qgrp->txqs[j]; 1532 } 1533 1534 if (idpf_is_queue_model_split(rsrc->txq_model)) { 1535 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1536 qs->qs[k++].complq = tx_qgrp->complq; 1537 } 1538 } 1539 1540 /* Make sure accounting agrees */ 1541 if (k != totqs) 1542 return -EINVAL; 1543 1544 return idpf_send_config_tx_queue_set_msg(qs); 1545 } 1546 1547 /** 1548 * idpf_fill_rxq_config_chunk - fill chunk describing the Rx queue 1549 * @rsrc: pointer to queue and vector resources 1550 * @q: Rx queue to be inserted into VC chunk 1551 * @qi: pointer to the buffer containing the VC chunk 1552 */ 1553 static void idpf_fill_rxq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1554 struct idpf_rx_queue *q, 1555 struct virtchnl2_rxq_info *qi) 1556 { 1557 const struct idpf_bufq_set *sets; 1558 1559 qi->queue_id = cpu_to_le32(q->q_id); 1560 qi->model = cpu_to_le16(rsrc->rxq_model); 1561 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 1562 qi->ring_len = cpu_to_le16(q->desc_count); 1563 qi->dma_ring_addr = cpu_to_le64(q->dma); 1564 qi->max_pkt_size = cpu_to_le32(q->rx_max_pkt_size); 1565 qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); 1566 qi->qflags = cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); 1567 if (idpf_queue_has(RSC_EN, q)) 1568 qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1569 1570 if (!idpf_is_queue_model_split(rsrc->rxq_model)) { 1571 qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1572 qi->desc_ids = cpu_to_le64(q->rxdids); 1573 1574 return; 1575 } 1576 1577 sets = q->bufq_sets; 1578 1579 /* 1580 * In splitq mode, RxQ buffer size should be set to that of the first 1581 * buffer queue associated with this RxQ. 1582 */ 1583 q->rx_buf_size = sets[0].bufq.rx_buf_size; 1584 qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1585 1586 qi->rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); 1587 if (rsrc->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { 1588 qi->bufq2_ena = IDPF_BUFQ2_ENA; 1589 qi->rx_bufq2_id = cpu_to_le16(sets[1].bufq.q_id); 1590 } 1591 1592 q->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; 1593 1594 if (idpf_queue_has(HSPLIT_EN, q)) { 1595 qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1596 qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); 1597 } 1598 1599 qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1600 } 1601 1602 /** 1603 * idpf_fill_bufq_config_chunk - fill chunk describing the buffer queue 1604 * @rsrc: pointer to queue and vector resources 1605 * @q: buffer queue to be inserted into VC chunk 1606 * @qi: pointer to the buffer containing the VC chunk 1607 */ 1608 static void idpf_fill_bufq_config_chunk(const struct idpf_q_vec_rsrc *rsrc, 1609 const struct idpf_buf_queue *q, 1610 struct virtchnl2_rxq_info *qi) 1611 { 1612 qi->queue_id = cpu_to_le32(q->q_id); 1613 qi->model = cpu_to_le16(rsrc->rxq_model); 1614 qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 1615 qi->ring_len = cpu_to_le16(q->desc_count); 1616 qi->dma_ring_addr = cpu_to_le64(q->dma); 1617 qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1618 qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); 1619 qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1620 qi->buffer_notif_stride = IDPF_RX_BUF_STRIDE; 1621 if (idpf_queue_has(RSC_EN, q)) 1622 qi->qflags = cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1623 1624 if (idpf_queue_has(HSPLIT_EN, q)) { 1625 qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1626 qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); 1627 } 1628 } 1629 1630 /** 1631 * idpf_prepare_cfg_rxqs_msg - prepare message to configure selected Rx queues 1632 * @vport_id: ID of virtual port queues are associated with 1633 * @buf: buffer containing the message 1634 * @pos: pointer to the first chunk describing the rx queue 1635 * @num_chunks: number of chunks in the message 1636 * 1637 * Helper function for preparing the message describing configuration of 1638 * Rx queues. 1639 * 1640 * Return: the total size of the prepared message. 1641 */ 1642 static u32 idpf_prepare_cfg_rxqs_msg(u32 vport_id, void *buf, const void *pos, 1643 u32 num_chunks) 1644 { 1645 struct virtchnl2_config_rx_queues *crq = buf; 1646 1647 crq->vport_id = cpu_to_le32(vport_id); 1648 crq->num_qinfo = cpu_to_le16(num_chunks); 1649 memcpy(crq->qinfo, pos, num_chunks * sizeof(*crq->qinfo)); 1650 1651 return struct_size(crq, qinfo, num_chunks); 1652 } 1653 1654 /** 1655 * idpf_send_config_rx_queue_set_msg - send virtchnl config Rx queues message 1656 * for selected queues. 1657 * @qs: set of the Rx queues to configure 1658 * 1659 * Send config queues virtchnl message for queues contained in the @qs array. 1660 * The @qs array can contain Rx queues (or buffer queues) only. 1661 * 1662 * Return: 0 on success, -errno on failure. 1663 */ 1664 static int idpf_send_config_rx_queue_set_msg(const struct idpf_queue_set *qs) 1665 { 1666 struct virtchnl2_rxq_info *qi __free(kfree) = NULL; 1667 struct idpf_chunked_msg_params params = { 1668 .vport_id = qs->vport_id, 1669 .vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES, 1670 .prepare_msg = idpf_prepare_cfg_rxqs_msg, 1671 .config_sz = sizeof(struct virtchnl2_config_rx_queues), 1672 .chunk_sz = sizeof(*qi), 1673 }; 1674 1675 qi = kzalloc_objs(*qi, qs->num); 1676 if (!qi) 1677 return -ENOMEM; 1678 1679 params.chunks = qi; 1680 1681 for (u32 i = 0; i < qs->num; i++) { 1682 if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX) 1683 idpf_fill_rxq_config_chunk(qs->qv_rsrc, qs->qs[i].rxq, 1684 &qi[params.num_chunks++]); 1685 else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX_BUFFER) 1686 idpf_fill_bufq_config_chunk(qs->qv_rsrc, qs->qs[i].bufq, 1687 &qi[params.num_chunks++]); 1688 } 1689 1690 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1691 } 1692 1693 /** 1694 * idpf_send_config_rx_queues_msg - send virtchnl config Rx queues message 1695 * @adapter: adapter pointer used to send virtchnl message 1696 * @rsrc: pointer to queue and vector resources 1697 * @vport_id: vport identifier used while preparing the virtchnl message 1698 * 1699 * Return: 0 on success, -errno on failure. 1700 */ 1701 static int idpf_send_config_rx_queues_msg(struct idpf_adapter *adapter, 1702 struct idpf_q_vec_rsrc *rsrc, 1703 u32 vport_id) 1704 { 1705 bool splitq = idpf_is_queue_model_split(rsrc->rxq_model); 1706 struct idpf_queue_set *qs __free(kfree) = NULL; 1707 u32 totqs = rsrc->num_rxq + rsrc->num_bufq; 1708 u32 k = 0; 1709 1710 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, totqs); 1711 if (!qs) 1712 return -ENOMEM; 1713 1714 /* Populate the queue info buffer with all queue context info */ 1715 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 1716 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1717 u32 num_rxq; 1718 1719 if (!splitq) { 1720 num_rxq = rx_qgrp->singleq.num_rxq; 1721 goto rxq; 1722 } 1723 1724 for (u32 j = 0; j < rsrc->num_bufqs_per_qgrp; j++) { 1725 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1726 qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 1727 } 1728 1729 num_rxq = rx_qgrp->splitq.num_rxq_sets; 1730 1731 rxq: 1732 for (u32 j = 0; j < num_rxq; j++) { 1733 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 1734 1735 if (splitq) 1736 qs->qs[k++].rxq = 1737 &rx_qgrp->splitq.rxq_sets[j]->rxq; 1738 else 1739 qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 1740 } 1741 } 1742 1743 /* Make sure accounting agrees */ 1744 if (k != totqs) 1745 return -EINVAL; 1746 1747 return idpf_send_config_rx_queue_set_msg(qs); 1748 } 1749 1750 /** 1751 * idpf_prepare_ena_dis_qs_msg - prepare message to enable/disable selected 1752 * queues 1753 * @vport_id: ID of virtual port queues are associated with 1754 * @buf: buffer containing the message 1755 * @pos: pointer to the first chunk describing the queue 1756 * @num_chunks: number of chunks in the message 1757 * 1758 * Helper function for preparing the message describing queues to be enabled 1759 * or disabled. 1760 * 1761 * Return: the total size of the prepared message. 1762 */ 1763 static u32 idpf_prepare_ena_dis_qs_msg(u32 vport_id, void *buf, const void *pos, 1764 u32 num_chunks) 1765 { 1766 struct virtchnl2_del_ena_dis_queues *eq = buf; 1767 1768 eq->vport_id = cpu_to_le32(vport_id); 1769 eq->chunks.num_chunks = cpu_to_le16(num_chunks); 1770 memcpy(eq->chunks.chunks, pos, 1771 num_chunks * sizeof(*eq->chunks.chunks)); 1772 1773 return struct_size(eq, chunks.chunks, num_chunks); 1774 } 1775 1776 /** 1777 * idpf_send_ena_dis_queue_set_msg - send virtchnl enable or disable queues 1778 * message for selected queues 1779 * @qs: set of the queues to enable or disable 1780 * @en: whether to enable or disable queues 1781 * 1782 * Send enable or disable queues virtchnl message for queues contained 1783 * in the @qs array. 1784 * The @qs array can contain pointers to both Rx and Tx queues. 1785 * 1786 * Return: 0 on success, -errno on failure. 1787 */ 1788 static int idpf_send_ena_dis_queue_set_msg(const struct idpf_queue_set *qs, 1789 bool en) 1790 { 1791 struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; 1792 struct idpf_chunked_msg_params params = { 1793 .vport_id = qs->vport_id, 1794 .vc_op = en ? VIRTCHNL2_OP_ENABLE_QUEUES : 1795 VIRTCHNL2_OP_DISABLE_QUEUES, 1796 .prepare_msg = idpf_prepare_ena_dis_qs_msg, 1797 .config_sz = sizeof(struct virtchnl2_del_ena_dis_queues), 1798 .chunk_sz = sizeof(*qc), 1799 .num_chunks = qs->num, 1800 }; 1801 1802 qc = kzalloc_objs(*qc, qs->num); 1803 if (!qc) 1804 return -ENOMEM; 1805 1806 params.chunks = qc; 1807 1808 for (u32 i = 0; i < qs->num; i++) { 1809 const struct idpf_queue_ptr *q = &qs->qs[i]; 1810 u32 qid; 1811 1812 qc[i].type = cpu_to_le32(q->type); 1813 qc[i].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 1814 1815 switch (q->type) { 1816 case VIRTCHNL2_QUEUE_TYPE_RX: 1817 qid = q->rxq->q_id; 1818 break; 1819 case VIRTCHNL2_QUEUE_TYPE_TX: 1820 qid = q->txq->q_id; 1821 break; 1822 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1823 qid = q->bufq->q_id; 1824 break; 1825 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 1826 qid = q->complq->q_id; 1827 break; 1828 default: 1829 return -EINVAL; 1830 } 1831 1832 qc[i].start_queue_id = cpu_to_le32(qid); 1833 } 1834 1835 return idpf_send_chunked_msg(qs->adapter, ¶ms); 1836 } 1837 1838 /** 1839 * idpf_send_ena_dis_queues_msg - send virtchnl enable or disable queues 1840 * message 1841 * @adapter: adapter pointer used to send virtchnl message 1842 * @rsrc: pointer to queue and vector resources 1843 * @vport_id: vport identifier used while preparing the virtchnl message 1844 * @en: whether to enable or disable queues 1845 * 1846 * Return: 0 on success, -errno on failure. 1847 */ 1848 static int idpf_send_ena_dis_queues_msg(struct idpf_adapter *adapter, 1849 struct idpf_q_vec_rsrc *rsrc, 1850 u32 vport_id, bool en) 1851 { 1852 struct idpf_queue_set *qs __free(kfree) = NULL; 1853 u32 num_txq, num_q, k = 0; 1854 bool split; 1855 1856 num_txq = rsrc->num_txq + rsrc->num_complq; 1857 num_q = num_txq + rsrc->num_rxq + rsrc->num_bufq; 1858 1859 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, num_q); 1860 if (!qs) 1861 return -ENOMEM; 1862 1863 split = idpf_is_queue_model_split(rsrc->txq_model); 1864 1865 for (u32 i = 0; i < rsrc->num_txq_grp; i++) { 1866 const struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 1867 1868 for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 1869 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 1870 qs->qs[k++].txq = tx_qgrp->txqs[j]; 1871 } 1872 1873 if (!split) 1874 continue; 1875 1876 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1877 qs->qs[k++].complq = tx_qgrp->complq; 1878 } 1879 1880 if (k != num_txq) 1881 return -EINVAL; 1882 1883 split = idpf_is_queue_model_split(rsrc->rxq_model); 1884 1885 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 1886 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 1887 u32 num_rxq; 1888 1889 if (split) 1890 num_rxq = rx_qgrp->splitq.num_rxq_sets; 1891 else 1892 num_rxq = rx_qgrp->singleq.num_rxq; 1893 1894 for (u32 j = 0; j < num_rxq; j++) { 1895 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 1896 1897 if (split) 1898 qs->qs[k++].rxq = 1899 &rx_qgrp->splitq.rxq_sets[j]->rxq; 1900 else 1901 qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 1902 } 1903 1904 if (!split) 1905 continue; 1906 1907 for (u32 j = 0; j < rsrc->num_bufqs_per_qgrp; j++) { 1908 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1909 qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 1910 } 1911 } 1912 1913 if (k != num_q) 1914 return -EINVAL; 1915 1916 return idpf_send_ena_dis_queue_set_msg(qs, en); 1917 } 1918 1919 /** 1920 * idpf_prep_map_unmap_queue_set_vector_msg - prepare message to map or unmap 1921 * queue set to the interrupt vector 1922 * @vport_id: ID of virtual port queues are associated with 1923 * @buf: buffer containing the message 1924 * @pos: pointer to the first chunk describing the vector mapping 1925 * @num_chunks: number of chunks in the message 1926 * 1927 * Helper function for preparing the message describing mapping queues to 1928 * q_vectors. 1929 * 1930 * Return: the total size of the prepared message. 1931 */ 1932 static u32 1933 idpf_prep_map_unmap_queue_set_vector_msg(u32 vport_id, void *buf, 1934 const void *pos, u32 num_chunks) 1935 { 1936 struct virtchnl2_queue_vector_maps *vqvm = buf; 1937 1938 vqvm->vport_id = cpu_to_le32(vport_id); 1939 vqvm->num_qv_maps = cpu_to_le16(num_chunks); 1940 memcpy(vqvm->qv_maps, pos, num_chunks * sizeof(*vqvm->qv_maps)); 1941 1942 return struct_size(vqvm, qv_maps, num_chunks); 1943 } 1944 1945 /** 1946 * idpf_send_map_unmap_queue_set_vector_msg - send virtchnl map or unmap 1947 * queue set vector message 1948 * @qs: set of the queues to map or unmap 1949 * @map: true for map and false for unmap 1950 * 1951 * Return: 0 on success, -errno on failure. 1952 */ 1953 static int 1954 idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, 1955 bool map) 1956 { 1957 struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; 1958 struct idpf_chunked_msg_params params = { 1959 .vport_id = qs->vport_id, 1960 .vc_op = map ? VIRTCHNL2_OP_MAP_QUEUE_VECTOR : 1961 VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR, 1962 .prepare_msg = idpf_prep_map_unmap_queue_set_vector_msg, 1963 .config_sz = sizeof(struct virtchnl2_queue_vector_maps), 1964 .chunk_sz = sizeof(*vqv), 1965 .num_chunks = qs->num, 1966 }; 1967 bool split; 1968 1969 vqv = kzalloc_objs(*vqv, qs->num); 1970 if (!vqv) 1971 return -ENOMEM; 1972 1973 params.chunks = vqv; 1974 1975 split = idpf_is_queue_model_split(qs->qv_rsrc->txq_model); 1976 1977 for (u32 i = 0; i < qs->num; i++) { 1978 const struct idpf_queue_ptr *q = &qs->qs[i]; 1979 const struct idpf_q_vector *vec; 1980 u32 qid, v_idx, itr_idx; 1981 1982 vqv[i].queue_type = cpu_to_le32(q->type); 1983 1984 switch (q->type) { 1985 case VIRTCHNL2_QUEUE_TYPE_RX: 1986 qid = q->rxq->q_id; 1987 1988 if (idpf_queue_has(NOIRQ, q->rxq)) 1989 vec = NULL; 1990 else 1991 vec = q->rxq->q_vector; 1992 1993 if (vec) { 1994 v_idx = vec->v_idx; 1995 itr_idx = vec->rx_itr_idx; 1996 } else { 1997 v_idx = qs->qv_rsrc->noirq_v_idx; 1998 itr_idx = VIRTCHNL2_ITR_IDX_0; 1999 } 2000 break; 2001 case VIRTCHNL2_QUEUE_TYPE_TX: 2002 qid = q->txq->q_id; 2003 2004 if (idpf_queue_has(NOIRQ, q->txq)) 2005 vec = NULL; 2006 else if (idpf_queue_has(XDP, q->txq)) 2007 vec = q->txq->complq->q_vector; 2008 else if (split) 2009 vec = q->txq->txq_grp->complq->q_vector; 2010 else 2011 vec = q->txq->q_vector; 2012 2013 if (vec) { 2014 v_idx = vec->v_idx; 2015 itr_idx = vec->tx_itr_idx; 2016 } else { 2017 v_idx = qs->qv_rsrc->noirq_v_idx; 2018 itr_idx = VIRTCHNL2_ITR_IDX_1; 2019 } 2020 break; 2021 default: 2022 return -EINVAL; 2023 } 2024 2025 vqv[i].queue_id = cpu_to_le32(qid); 2026 vqv[i].vector_id = cpu_to_le16(v_idx); 2027 vqv[i].itr_idx = cpu_to_le32(itr_idx); 2028 } 2029 2030 return idpf_send_chunked_msg(qs->adapter, ¶ms); 2031 } 2032 2033 /** 2034 * idpf_send_map_unmap_queue_vector_msg - send virtchnl map or unmap queue 2035 * vector message 2036 * @adapter: adapter pointer used to send virtchnl message 2037 * @rsrc: pointer to queue and vector resources 2038 * @vport_id: vport identifier used while preparing the virtchnl message 2039 * @map: true for map and false for unmap 2040 * 2041 * Return: 0 on success, -errno on failure. 2042 */ 2043 int idpf_send_map_unmap_queue_vector_msg(struct idpf_adapter *adapter, 2044 struct idpf_q_vec_rsrc *rsrc, 2045 u32 vport_id, bool map) 2046 { 2047 struct idpf_queue_set *qs __free(kfree) = NULL; 2048 u32 num_q = rsrc->num_txq + rsrc->num_rxq; 2049 u32 k = 0; 2050 2051 qs = idpf_alloc_queue_set(adapter, rsrc, vport_id, num_q); 2052 if (!qs) 2053 return -ENOMEM; 2054 2055 for (u32 i = 0; i < rsrc->num_txq_grp; i++) { 2056 const struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 2057 2058 for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 2059 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 2060 qs->qs[k++].txq = tx_qgrp->txqs[j]; 2061 } 2062 } 2063 2064 if (k != rsrc->num_txq) 2065 return -EINVAL; 2066 2067 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 2068 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 2069 u32 num_rxq; 2070 2071 if (idpf_is_queue_model_split(rsrc->rxq_model)) 2072 num_rxq = rx_qgrp->splitq.num_rxq_sets; 2073 else 2074 num_rxq = rx_qgrp->singleq.num_rxq; 2075 2076 for (u32 j = 0; j < num_rxq; j++) { 2077 qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 2078 2079 if (idpf_is_queue_model_split(rsrc->rxq_model)) 2080 qs->qs[k++].rxq = 2081 &rx_qgrp->splitq.rxq_sets[j]->rxq; 2082 else 2083 qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 2084 } 2085 } 2086 2087 if (k != num_q) 2088 return -EINVAL; 2089 2090 return idpf_send_map_unmap_queue_set_vector_msg(qs, map); 2091 } 2092 2093 /** 2094 * idpf_send_enable_queue_set_msg - send enable queues virtchnl message for 2095 * selected queues 2096 * @qs: set of the queues 2097 * 2098 * Send enable queues virtchnl message for queues contained in the @qs array. 2099 * 2100 * Return: 0 on success, -errno on failure. 2101 */ 2102 int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs) 2103 { 2104 return idpf_send_ena_dis_queue_set_msg(qs, true); 2105 } 2106 2107 /** 2108 * idpf_send_disable_queue_set_msg - send disable queues virtchnl message for 2109 * selected queues 2110 * @qs: set of the queues 2111 * 2112 * Return: 0 on success, -errno on failure. 2113 */ 2114 int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs) 2115 { 2116 int err; 2117 2118 err = idpf_send_ena_dis_queue_set_msg(qs, false); 2119 if (err) 2120 return err; 2121 2122 return idpf_wait_for_marker_event_set(qs); 2123 } 2124 2125 /** 2126 * idpf_send_config_queue_set_msg - send virtchnl config queues message for 2127 * selected queues 2128 * @qs: set of the queues 2129 * 2130 * Send config queues virtchnl message for queues contained in the @qs array. 2131 * The @qs array can contain both Rx or Tx queues. 2132 * 2133 * Return: 0 on success, -errno on failure. 2134 */ 2135 int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs) 2136 { 2137 int err; 2138 2139 err = idpf_send_config_tx_queue_set_msg(qs); 2140 if (err) 2141 return err; 2142 2143 return idpf_send_config_rx_queue_set_msg(qs); 2144 } 2145 2146 /** 2147 * idpf_send_enable_queues_msg - send enable queues virtchnl message 2148 * @vport: Virtual port private data structure 2149 * 2150 * Will send enable queues virtchnl message. Returns 0 on success, negative on 2151 * failure. 2152 */ 2153 int idpf_send_enable_queues_msg(struct idpf_vport *vport) 2154 { 2155 return idpf_send_ena_dis_queues_msg(vport->adapter, 2156 &vport->dflt_qv_rsrc, 2157 vport->vport_id, true); 2158 } 2159 2160 /** 2161 * idpf_send_disable_queues_msg - send disable queues virtchnl message 2162 * @vport: Virtual port private data structure 2163 * 2164 * Will send disable queues virtchnl message. Returns 0 on success, negative 2165 * on failure. 2166 */ 2167 int idpf_send_disable_queues_msg(struct idpf_vport *vport) 2168 { 2169 int err; 2170 2171 err = idpf_send_ena_dis_queues_msg(vport->adapter, 2172 &vport->dflt_qv_rsrc, 2173 vport->vport_id, false); 2174 if (err) 2175 return err; 2176 2177 return idpf_wait_for_marker_event(vport); 2178 } 2179 2180 /** 2181 * idpf_convert_reg_to_queue_chunks - Copy queue chunk information to the right 2182 * structure 2183 * @dchunks: Destination chunks to store data to 2184 * @schunks: Source chunks to copy data from 2185 * @num_chunks: number of chunks to copy 2186 */ 2187 static void idpf_convert_reg_to_queue_chunks(struct virtchnl2_queue_chunk *dchunks, 2188 struct idpf_queue_id_reg_chunk *schunks, 2189 u16 num_chunks) 2190 { 2191 u16 i; 2192 2193 for (i = 0; i < num_chunks; i++) { 2194 dchunks[i].type = cpu_to_le32(schunks[i].type); 2195 dchunks[i].start_queue_id = cpu_to_le32(schunks[i].start_queue_id); 2196 dchunks[i].num_queues = cpu_to_le32(schunks[i].num_queues); 2197 } 2198 } 2199 2200 /** 2201 * idpf_send_delete_queues_msg - send delete queues virtchnl message 2202 * @adapter: adapter pointer used to send virtchnl message 2203 * @chunks: queue ids received over mailbox 2204 * @vport_id: vport identifier used while preparing the virtchnl message 2205 * 2206 * Return: 0 on success, negative on failure. 2207 */ 2208 int idpf_send_delete_queues_msg(struct idpf_adapter *adapter, 2209 struct idpf_queue_id_reg_info *chunks, 2210 u32 vport_id) 2211 { 2212 struct libie_ctlq_xn_send_params xn_params = { 2213 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2214 .chnl_opcode = VIRTCHNL2_OP_DEL_QUEUES, 2215 }; 2216 struct virtchnl2_del_ena_dis_queues *eq; 2217 ssize_t buf_size; 2218 u16 num_chunks; 2219 int err; 2220 2221 num_chunks = chunks->num_chunks; 2222 buf_size = struct_size(eq, chunks.chunks, num_chunks); 2223 2224 eq = kzalloc(buf_size, GFP_KERNEL); 2225 if (!eq) 2226 return -ENOMEM; 2227 2228 eq->vport_id = cpu_to_le32(vport_id); 2229 eq->chunks.num_chunks = cpu_to_le16(num_chunks); 2230 2231 idpf_convert_reg_to_queue_chunks(eq->chunks.chunks, chunks->queue_chunks, 2232 num_chunks); 2233 2234 err = idpf_send_mb_msg_kfree(adapter, &xn_params, eq, buf_size); 2235 if (err) 2236 return err; 2237 2238 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2239 2240 return 0; 2241 } 2242 2243 /** 2244 * idpf_send_config_queues_msg - Send config queues virtchnl message 2245 * @adapter: adapter pointer used to send virtchnl message 2246 * @rsrc: pointer to queue and vector resources 2247 * @vport_id: vport identifier used while preparing the virtchnl message 2248 * 2249 * Return: 0 on success, negative on failure. 2250 */ 2251 int idpf_send_config_queues_msg(struct idpf_adapter *adapter, 2252 struct idpf_q_vec_rsrc *rsrc, 2253 u32 vport_id) 2254 { 2255 int err; 2256 2257 err = idpf_send_config_tx_queues_msg(adapter, rsrc, vport_id); 2258 if (err) 2259 return err; 2260 2261 return idpf_send_config_rx_queues_msg(adapter, rsrc, vport_id); 2262 } 2263 2264 /** 2265 * idpf_send_add_queues_msg - Send virtchnl add queues message 2266 * @adapter: adapter pointer used to send virtchnl message 2267 * @vport_config: vport persistent structure to store the queue chunk info 2268 * @rsrc: pointer to queue and vector resources 2269 * @vport_id: vport identifier used while preparing the virtchnl message 2270 * 2271 * Return: 0 on success, negative on failure. 2272 */ 2273 int idpf_send_add_queues_msg(struct idpf_adapter *adapter, 2274 struct idpf_vport_config *vport_config, 2275 struct idpf_q_vec_rsrc *rsrc, 2276 u32 vport_id) 2277 { 2278 struct libie_ctlq_xn_send_params xn_params = { 2279 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2280 .chnl_opcode = VIRTCHNL2_OP_ADD_QUEUES, 2281 }; 2282 struct virtchnl2_add_queues *vc_msg; 2283 struct virtchnl2_add_queues aq = {}; 2284 size_t size; 2285 int err; 2286 2287 aq.vport_id = cpu_to_le32(vport_id); 2288 aq.num_tx_q = cpu_to_le16(rsrc->num_txq); 2289 aq.num_tx_complq = cpu_to_le16(rsrc->num_complq); 2290 aq.num_rx_q = cpu_to_le16(rsrc->num_rxq); 2291 aq.num_rx_bufq = cpu_to_le16(rsrc->num_bufq); 2292 2293 err = idpf_send_mb_msg_stack(adapter, &xn_params, &aq); 2294 if (err) 2295 return err; 2296 2297 vc_msg = xn_params.recv_mem.iov_base; 2298 if (xn_params.recv_mem.iov_len < sizeof(*vc_msg)) { 2299 err = -EIO; 2300 goto free_rx_buf; 2301 } 2302 2303 /* compare vc_msg num queues with vport num queues */ 2304 if (le16_to_cpu(vc_msg->num_tx_q) != rsrc->num_txq || 2305 le16_to_cpu(vc_msg->num_rx_q) != rsrc->num_rxq || 2306 le16_to_cpu(vc_msg->num_tx_complq) != rsrc->num_complq || 2307 le16_to_cpu(vc_msg->num_rx_bufq) != rsrc->num_bufq) { 2308 err = -EINVAL; 2309 goto free_rx_buf; 2310 } 2311 2312 size = struct_size(vc_msg, chunks.chunks, 2313 le16_to_cpu(vc_msg->chunks.num_chunks)); 2314 if (xn_params.recv_mem.iov_len < size) { 2315 err = -EIO; 2316 goto free_rx_buf; 2317 } 2318 2319 err = idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks); 2320 2321 free_rx_buf: 2322 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2323 2324 return err; 2325 } 2326 2327 /** 2328 * idpf_send_alloc_vectors_msg - Send virtchnl alloc vectors message 2329 * @adapter: Driver specific private structure 2330 * @num_vectors: number of vectors to be allocated 2331 * 2332 * Returns 0 on success, negative on failure. 2333 */ 2334 int idpf_send_alloc_vectors_msg(struct idpf_adapter *adapter, u16 num_vectors) 2335 { 2336 struct libie_ctlq_xn_send_params xn_params = { 2337 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2338 .chnl_opcode = VIRTCHNL2_OP_ALLOC_VECTORS, 2339 }; 2340 struct virtchnl2_alloc_vectors *rcvd_vec; 2341 struct virtchnl2_alloc_vectors ac = {}; 2342 u16 num_vchunks; 2343 int size, err; 2344 2345 ac.num_vectors = cpu_to_le16(num_vectors); 2346 2347 err = idpf_send_mb_msg_stack(adapter, &xn_params, &ac); 2348 if (err) 2349 return err; 2350 2351 rcvd_vec = xn_params.recv_mem.iov_base; 2352 if (xn_params.recv_mem.iov_len < sizeof(*rcvd_vec)) { 2353 err = -EIO; 2354 goto free_rx_buf; 2355 } 2356 2357 num_vchunks = le16_to_cpu(rcvd_vec->vchunks.num_vchunks); 2358 size = struct_size(rcvd_vec, vchunks.vchunks, num_vchunks); 2359 if (xn_params.recv_mem.iov_len < size) { 2360 err = -EIO; 2361 goto free_rx_buf; 2362 } 2363 2364 kfree(adapter->req_vec_chunks); 2365 adapter->req_vec_chunks = kmemdup(rcvd_vec, size, GFP_KERNEL); 2366 if (!adapter->req_vec_chunks) { 2367 err = -ENOMEM; 2368 goto free_rx_buf; 2369 } 2370 2371 if (le16_to_cpu(adapter->req_vec_chunks->num_vectors) < num_vectors) { 2372 kfree(adapter->req_vec_chunks); 2373 adapter->req_vec_chunks = NULL; 2374 err = -EINVAL; 2375 } 2376 2377 free_rx_buf: 2378 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2379 2380 return err; 2381 } 2382 2383 /** 2384 * idpf_send_dealloc_vectors_msg - Send virtchnl de allocate vectors message 2385 * @adapter: Driver specific private structure 2386 * 2387 * Returns 0 on success, negative on failure. 2388 */ 2389 int idpf_send_dealloc_vectors_msg(struct idpf_adapter *adapter) 2390 { 2391 struct virtchnl2_alloc_vectors *ac = adapter->req_vec_chunks; 2392 struct libie_ctlq_xn_send_params xn_params = { 2393 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2394 .chnl_opcode = VIRTCHNL2_OP_DEALLOC_VECTORS, 2395 }; 2396 struct virtchnl2_vector_chunks *vcs; 2397 int buf_size, err; 2398 2399 buf_size = struct_size(&ac->vchunks, vchunks, 2400 le16_to_cpu(ac->vchunks.num_vchunks)); 2401 vcs = kmemdup(&ac->vchunks, buf_size, GFP_KERNEL); 2402 if (!vcs) 2403 return -ENOMEM; 2404 2405 err = idpf_send_mb_msg_kfree(adapter, &xn_params, vcs, buf_size); 2406 if (err) 2407 return err; 2408 2409 kfree(adapter->req_vec_chunks); 2410 adapter->req_vec_chunks = NULL; 2411 2412 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2413 2414 return 0; 2415 } 2416 2417 /** 2418 * idpf_get_max_vfs - Get max number of vfs supported 2419 * @adapter: Driver specific private structure 2420 * 2421 * Returns max number of VFs 2422 */ 2423 static int idpf_get_max_vfs(struct idpf_adapter *adapter) 2424 { 2425 return le16_to_cpu(adapter->caps.max_sriov_vfs); 2426 } 2427 2428 /** 2429 * idpf_send_set_sriov_vfs_msg - Send virtchnl set sriov vfs message 2430 * @adapter: Driver specific private structure 2431 * @num_vfs: number of virtual functions to be created 2432 * 2433 * Returns 0 on success, negative on failure. 2434 */ 2435 int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs) 2436 { 2437 struct libie_ctlq_xn_send_params xn_params = { 2438 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2439 .chnl_opcode = VIRTCHNL2_OP_SET_SRIOV_VFS, 2440 }; 2441 struct virtchnl2_sriov_vfs_info svi = {}; 2442 int err; 2443 2444 svi.num_vfs = cpu_to_le16(num_vfs); 2445 2446 err = idpf_send_mb_msg_stack(adapter, &xn_params, &svi); 2447 if (err) 2448 return err; 2449 2450 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2451 2452 return 0; 2453 } 2454 2455 /** 2456 * idpf_send_get_stats_msg - Send virtchnl get statistics message 2457 * @np: netdev private structure 2458 * @port_stats: structure to store the vport statistics 2459 * 2460 * Return: 0 on success, negative on failure. 2461 */ 2462 int idpf_send_get_stats_msg(struct idpf_netdev_priv *np, 2463 struct idpf_port_stats *port_stats) 2464 { 2465 struct libie_ctlq_xn_send_params xn_params = { 2466 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2467 .chnl_opcode = VIRTCHNL2_OP_GET_STATS, 2468 }; 2469 struct rtnl_link_stats64 *netstats = &np->netstats; 2470 struct virtchnl2_vport_stats *stats_recv; 2471 struct virtchnl2_vport_stats stats_msg = {}; 2472 int err; 2473 2474 2475 /* Don't send get_stats message if the link is down */ 2476 if (!test_bit(IDPF_VPORT_UP, np->state)) 2477 return 0; 2478 2479 stats_msg.vport_id = cpu_to_le32(np->vport_id); 2480 2481 err = idpf_send_mb_msg_stack(np->adapter, &xn_params, &stats_msg); 2482 if (err) 2483 return err; 2484 2485 if (xn_params.recv_mem.iov_len < sizeof(*stats_recv)) { 2486 err = -EIO; 2487 goto free_rx_buf; 2488 } 2489 2490 stats_recv = xn_params.recv_mem.iov_base; 2491 2492 spin_lock_bh(&np->stats_lock); 2493 2494 netstats->rx_packets = le64_to_cpu(stats_recv->rx_unicast) + 2495 le64_to_cpu(stats_recv->rx_multicast) + 2496 le64_to_cpu(stats_recv->rx_broadcast); 2497 netstats->tx_packets = le64_to_cpu(stats_recv->tx_unicast) + 2498 le64_to_cpu(stats_recv->tx_multicast) + 2499 le64_to_cpu(stats_recv->tx_broadcast); 2500 netstats->rx_bytes = le64_to_cpu(stats_recv->rx_bytes); 2501 netstats->tx_bytes = le64_to_cpu(stats_recv->tx_bytes); 2502 netstats->rx_errors = le64_to_cpu(stats_recv->rx_errors); 2503 netstats->tx_errors = le64_to_cpu(stats_recv->tx_errors); 2504 netstats->rx_dropped = le64_to_cpu(stats_recv->rx_discards); 2505 netstats->tx_dropped = le64_to_cpu(stats_recv->tx_discards); 2506 2507 port_stats->vport_stats = *stats_recv; 2508 2509 spin_unlock_bh(&np->stats_lock); 2510 2511 free_rx_buf: 2512 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2513 2514 return err; 2515 } 2516 2517 /** 2518 * idpf_send_set_rss_lut_msg - Send virtchnl set RSS lut message 2519 * @adapter: adapter pointer used to send virtchnl message 2520 * @rss_data: pointer to RSS key and lut info 2521 * @vport_id: vport identifier used while preparing the virtchnl message 2522 * 2523 * When rxhash is disabled, RSS LUT will be configured with zeros. If rxhash 2524 * is enabled, the LUT values stored in driver's soft copy will be used to setup 2525 * the HW. 2526 * 2527 * Return: 0 on success, negative on failure. 2528 */ 2529 int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter, 2530 struct idpf_rss_data *rss_data, u32 vport_id) 2531 { 2532 struct libie_ctlq_xn_send_params xn_params = { 2533 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2534 .chnl_opcode = VIRTCHNL2_OP_SET_RSS_LUT, 2535 }; 2536 struct virtchnl2_rss_lut *rl; 2537 struct idpf_vport *vport; 2538 int buf_size, i, err; 2539 bool rxhash_ena; 2540 2541 vport = idpf_vid_to_vport(adapter, vport_id); 2542 if (!vport) 2543 return -EINVAL; 2544 2545 rxhash_ena = idpf_is_feature_ena(vport, NETIF_F_RXHASH); 2546 2547 buf_size = struct_size(rl, lut, rss_data->rss_lut_size); 2548 rl = kzalloc(buf_size, GFP_KERNEL); 2549 if (!rl) 2550 return -ENOMEM; 2551 2552 rl->vport_id = cpu_to_le32(vport_id); 2553 rl->lut_entries = cpu_to_le16(rss_data->rss_lut_size); 2554 for (i = 0; i < rss_data->rss_lut_size; i++) 2555 rl->lut[i] = rxhash_ena ? cpu_to_le32(rss_data->rss_lut[i]) : 0; 2556 2557 err = idpf_send_mb_msg_kfree(adapter, &xn_params, rl, buf_size); 2558 if (err) 2559 return err; 2560 2561 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2562 2563 return err; 2564 } 2565 2566 /** 2567 * idpf_send_set_rss_key_msg - Send virtchnl set RSS key message 2568 * @adapter: adapter pointer used to send virtchnl message 2569 * @rss_data: pointer to RSS key and lut info 2570 * @vport_id: vport identifier used while preparing the virtchnl message 2571 * 2572 * Return: 0 on success, negative on failure 2573 */ 2574 int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter, 2575 struct idpf_rss_data *rss_data, u32 vport_id) 2576 { 2577 struct libie_ctlq_xn_send_params xn_params = { 2578 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2579 .chnl_opcode = VIRTCHNL2_OP_SET_RSS_KEY, 2580 }; 2581 struct virtchnl2_rss_key *rk; 2582 int i, buf_size, err; 2583 2584 buf_size = struct_size(rk, key_flex, rss_data->rss_key_size); 2585 rk = kzalloc(buf_size, GFP_KERNEL); 2586 if (!rk) 2587 return -ENOMEM; 2588 2589 rk->vport_id = cpu_to_le32(vport_id); 2590 rk->key_len = cpu_to_le16(rss_data->rss_key_size); 2591 for (i = 0; i < rss_data->rss_key_size; i++) 2592 rk->key_flex[i] = rss_data->rss_key[i]; 2593 2594 err = idpf_send_mb_msg_kfree(adapter, &xn_params, rk, buf_size); 2595 if (err) 2596 return err; 2597 2598 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2599 2600 return err; 2601 } 2602 2603 /** 2604 * idpf_fill_ptype_lookup - Fill L3 specific fields in ptype lookup table 2605 * @ptype: ptype lookup table 2606 * @pstate: state machine for ptype lookup table 2607 * @ipv4: ipv4 or ipv6 2608 * @frag: fragmentation allowed 2609 * 2610 */ 2611 static void idpf_fill_ptype_lookup(struct libeth_rx_pt *ptype, 2612 struct idpf_ptype_state *pstate, 2613 bool ipv4, bool frag) 2614 { 2615 if (!pstate->outer_ip || !pstate->outer_frag) { 2616 pstate->outer_ip = true; 2617 2618 if (ipv4) 2619 ptype->outer_ip = LIBETH_RX_PT_OUTER_IPV4; 2620 else 2621 ptype->outer_ip = LIBETH_RX_PT_OUTER_IPV6; 2622 2623 if (frag) { 2624 ptype->outer_frag = LIBETH_RX_PT_FRAG; 2625 pstate->outer_frag = true; 2626 } 2627 } else { 2628 ptype->tunnel_type = LIBETH_RX_PT_TUNNEL_IP_IP; 2629 pstate->tunnel_state = IDPF_PTYPE_TUNNEL_IP; 2630 2631 if (ipv4) 2632 ptype->tunnel_end_prot = LIBETH_RX_PT_TUNNEL_END_IPV4; 2633 else 2634 ptype->tunnel_end_prot = LIBETH_RX_PT_TUNNEL_END_IPV6; 2635 2636 if (frag) 2637 ptype->tunnel_end_frag = LIBETH_RX_PT_FRAG; 2638 } 2639 } 2640 2641 static void idpf_finalize_ptype_lookup(struct libeth_rx_pt *ptype) 2642 { 2643 if (ptype->payload_layer == LIBETH_RX_PT_PAYLOAD_L2 && 2644 ptype->inner_prot) 2645 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_L4; 2646 else if (ptype->payload_layer == LIBETH_RX_PT_PAYLOAD_L2 && 2647 ptype->outer_ip) 2648 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_L3; 2649 else if (ptype->outer_ip == LIBETH_RX_PT_OUTER_L2) 2650 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_L2; 2651 else 2652 ptype->payload_layer = LIBETH_RX_PT_PAYLOAD_NONE; 2653 2654 libeth_rx_pt_gen_hash_type(ptype); 2655 } 2656 2657 /** 2658 * idpf_parse_protocol_ids - parse protocol IDs for a given packet type 2659 * @ptype: packet type to parse 2660 * @rx_pt: store the parsed packet type info into 2661 */ 2662 static void idpf_parse_protocol_ids(struct virtchnl2_ptype *ptype, 2663 struct libeth_rx_pt *rx_pt) 2664 { 2665 struct idpf_ptype_state pstate = {}; 2666 2667 for (u32 j = 0; j < ptype->proto_id_count; j++) { 2668 u16 id = le16_to_cpu(ptype->proto_id[j]); 2669 2670 switch (id) { 2671 case VIRTCHNL2_PROTO_HDR_GRE: 2672 if (pstate.tunnel_state == IDPF_PTYPE_TUNNEL_IP) { 2673 rx_pt->tunnel_type = 2674 LIBETH_RX_PT_TUNNEL_IP_GRENAT; 2675 pstate.tunnel_state |= 2676 IDPF_PTYPE_TUNNEL_IP_GRENAT; 2677 } 2678 break; 2679 case VIRTCHNL2_PROTO_HDR_MAC: 2680 rx_pt->outer_ip = LIBETH_RX_PT_OUTER_L2; 2681 if (pstate.tunnel_state == IDPF_TUN_IP_GRE) { 2682 rx_pt->tunnel_type = 2683 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC; 2684 pstate.tunnel_state |= 2685 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC; 2686 } 2687 break; 2688 case VIRTCHNL2_PROTO_HDR_IPV4: 2689 idpf_fill_ptype_lookup(rx_pt, &pstate, true, false); 2690 break; 2691 case VIRTCHNL2_PROTO_HDR_IPV6: 2692 idpf_fill_ptype_lookup(rx_pt, &pstate, false, false); 2693 break; 2694 case VIRTCHNL2_PROTO_HDR_IPV4_FRAG: 2695 idpf_fill_ptype_lookup(rx_pt, &pstate, true, true); 2696 break; 2697 case VIRTCHNL2_PROTO_HDR_IPV6_FRAG: 2698 idpf_fill_ptype_lookup(rx_pt, &pstate, false, true); 2699 break; 2700 case VIRTCHNL2_PROTO_HDR_UDP: 2701 rx_pt->inner_prot = LIBETH_RX_PT_INNER_UDP; 2702 break; 2703 case VIRTCHNL2_PROTO_HDR_TCP: 2704 rx_pt->inner_prot = LIBETH_RX_PT_INNER_TCP; 2705 break; 2706 case VIRTCHNL2_PROTO_HDR_SCTP: 2707 rx_pt->inner_prot = LIBETH_RX_PT_INNER_SCTP; 2708 break; 2709 case VIRTCHNL2_PROTO_HDR_ICMP: 2710 rx_pt->inner_prot = LIBETH_RX_PT_INNER_ICMP; 2711 break; 2712 case VIRTCHNL2_PROTO_HDR_PAY: 2713 rx_pt->payload_layer = LIBETH_RX_PT_PAYLOAD_L2; 2714 break; 2715 case VIRTCHNL2_PROTO_HDR_ICMPV6: 2716 case VIRTCHNL2_PROTO_HDR_IPV6_EH: 2717 case VIRTCHNL2_PROTO_HDR_PRE_MAC: 2718 case VIRTCHNL2_PROTO_HDR_POST_MAC: 2719 case VIRTCHNL2_PROTO_HDR_ETHERTYPE: 2720 case VIRTCHNL2_PROTO_HDR_SVLAN: 2721 case VIRTCHNL2_PROTO_HDR_CVLAN: 2722 case VIRTCHNL2_PROTO_HDR_MPLS: 2723 case VIRTCHNL2_PROTO_HDR_MMPLS: 2724 case VIRTCHNL2_PROTO_HDR_PTP: 2725 case VIRTCHNL2_PROTO_HDR_CTRL: 2726 case VIRTCHNL2_PROTO_HDR_LLDP: 2727 case VIRTCHNL2_PROTO_HDR_ARP: 2728 case VIRTCHNL2_PROTO_HDR_ECP: 2729 case VIRTCHNL2_PROTO_HDR_EAPOL: 2730 case VIRTCHNL2_PROTO_HDR_PPPOD: 2731 case VIRTCHNL2_PROTO_HDR_PPPOE: 2732 case VIRTCHNL2_PROTO_HDR_IGMP: 2733 case VIRTCHNL2_PROTO_HDR_AH: 2734 case VIRTCHNL2_PROTO_HDR_ESP: 2735 case VIRTCHNL2_PROTO_HDR_IKE: 2736 case VIRTCHNL2_PROTO_HDR_NATT_KEEP: 2737 case VIRTCHNL2_PROTO_HDR_L2TPV2: 2738 case VIRTCHNL2_PROTO_HDR_L2TPV2_CONTROL: 2739 case VIRTCHNL2_PROTO_HDR_L2TPV3: 2740 case VIRTCHNL2_PROTO_HDR_GTP: 2741 case VIRTCHNL2_PROTO_HDR_GTP_EH: 2742 case VIRTCHNL2_PROTO_HDR_GTPCV2: 2743 case VIRTCHNL2_PROTO_HDR_GTPC_TEID: 2744 case VIRTCHNL2_PROTO_HDR_GTPU: 2745 case VIRTCHNL2_PROTO_HDR_GTPU_UL: 2746 case VIRTCHNL2_PROTO_HDR_GTPU_DL: 2747 case VIRTCHNL2_PROTO_HDR_ECPRI: 2748 case VIRTCHNL2_PROTO_HDR_VRRP: 2749 case VIRTCHNL2_PROTO_HDR_OSPF: 2750 case VIRTCHNL2_PROTO_HDR_TUN: 2751 case VIRTCHNL2_PROTO_HDR_NVGRE: 2752 case VIRTCHNL2_PROTO_HDR_VXLAN: 2753 case VIRTCHNL2_PROTO_HDR_VXLAN_GPE: 2754 case VIRTCHNL2_PROTO_HDR_GENEVE: 2755 case VIRTCHNL2_PROTO_HDR_NSH: 2756 case VIRTCHNL2_PROTO_HDR_QUIC: 2757 case VIRTCHNL2_PROTO_HDR_PFCP: 2758 case VIRTCHNL2_PROTO_HDR_PFCP_NODE: 2759 case VIRTCHNL2_PROTO_HDR_PFCP_SESSION: 2760 case VIRTCHNL2_PROTO_HDR_RTP: 2761 case VIRTCHNL2_PROTO_HDR_NO_PROTO: 2762 break; 2763 default: 2764 break; 2765 } 2766 } 2767 } 2768 2769 /** 2770 * idpf_send_get_rx_ptype_msg - Send virtchnl for ptype info 2771 * @adapter: driver specific private structure 2772 * 2773 * Return: 0 on success, negative on failure. 2774 */ 2775 static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter) 2776 { 2777 struct libie_ctlq_xn_send_params xn_params = { 2778 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2779 .chnl_opcode = VIRTCHNL2_OP_GET_PTYPE_INFO, 2780 }; 2781 struct virtchnl2_get_ptype_info *get_ptype_info; 2782 struct virtchnl2_get_ptype_info *ptype_info; 2783 int err = 0, max_ptype = IDPF_RX_MAX_PTYPE; 2784 int buf_size = sizeof(*get_ptype_info); 2785 struct libeth_rx_pt *singleq_pt_lkup; 2786 struct libeth_rx_pt *splitq_pt_lkup; 2787 int ptypes_recvd = 0, ptype_offset; 2788 u16 next_ptype_id = 0; 2789 2790 singleq_pt_lkup = kzalloc_objs(*singleq_pt_lkup, IDPF_RX_MAX_BASE_PTYPE); 2791 if (!singleq_pt_lkup) 2792 return -ENOMEM; 2793 2794 splitq_pt_lkup = kzalloc_objs(*splitq_pt_lkup, max_ptype); 2795 if (!splitq_pt_lkup) { 2796 err = -ENOMEM; 2797 goto free_singleq; 2798 } 2799 2800 while (next_ptype_id < max_ptype) { 2801 u16 num_ptypes; 2802 2803 get_ptype_info = kzalloc(buf_size, GFP_KERNEL); 2804 if (!get_ptype_info) { 2805 err = -ENOMEM; 2806 goto free_splitq; 2807 } 2808 2809 get_ptype_info->start_ptype_id = cpu_to_le16(next_ptype_id); 2810 2811 if ((next_ptype_id + IDPF_RX_MAX_PTYPES_PER_BUF) > max_ptype) 2812 num_ptypes = max_ptype - next_ptype_id; 2813 else 2814 num_ptypes = IDPF_RX_MAX_PTYPES_PER_BUF; 2815 2816 get_ptype_info->num_ptypes = cpu_to_le16(num_ptypes); 2817 err = idpf_send_mb_msg_kfree(adapter, &xn_params, 2818 get_ptype_info, buf_size); 2819 if (err) 2820 goto free_splitq; 2821 2822 ptype_info = xn_params.recv_mem.iov_base; 2823 if (xn_params.recv_mem.iov_len < sizeof(*ptype_info)) { 2824 err = -EIO; 2825 goto free_rx_buf; 2826 } 2827 ptypes_recvd += le16_to_cpu(ptype_info->num_ptypes); 2828 if (ptypes_recvd > max_ptype) { 2829 err = -EINVAL; 2830 goto free_rx_buf; 2831 } 2832 2833 next_ptype_id = next_ptype_id + num_ptypes; 2834 ptype_offset = IDPF_RX_PTYPE_HDR_SZ; 2835 2836 for (u16 i = 0; i < le16_to_cpu(ptype_info->num_ptypes); i++) { 2837 struct libeth_rx_pt rx_pt = {}; 2838 struct virtchnl2_ptype *ptype; 2839 u16 pt_10, pt_8; 2840 2841 ptype = (struct virtchnl2_ptype *) 2842 ((u8 *)ptype_info + ptype_offset); 2843 if (xn_params.recv_mem.iov_len < 2844 ptype_offset + sizeof(struct virtchnl2_ptype)) { 2845 err = -EINVAL; 2846 goto free_rx_buf; 2847 } 2848 2849 pt_10 = le16_to_cpu(ptype->ptype_id_10); 2850 pt_8 = ptype->ptype_id_8; 2851 2852 ptype_offset += IDPF_GET_PTYPE_SIZE(ptype); 2853 if (xn_params.recv_mem.iov_len < ptype_offset) { 2854 err = -EINVAL; 2855 goto free_rx_buf; 2856 } 2857 2858 /* 0xFFFF indicates end of ptypes */ 2859 if (pt_10 == IDPF_INVALID_PTYPE_ID) 2860 goto out; 2861 if (pt_10 >= max_ptype) { 2862 err = -EINVAL; 2863 goto free_rx_buf; 2864 } 2865 2866 idpf_parse_protocol_ids(ptype, &rx_pt); 2867 idpf_finalize_ptype_lookup(&rx_pt); 2868 2869 /* For a given protocol ID stack, the ptype value might 2870 * vary between ptype_id_10 and ptype_id_8. So store 2871 * them separately for splitq and singleq. Also skip 2872 * the repeated ptypes in case of singleq. 2873 */ 2874 splitq_pt_lkup[pt_10] = rx_pt; 2875 if (!singleq_pt_lkup[pt_8].outer_ip) 2876 singleq_pt_lkup[pt_8] = rx_pt; 2877 } 2878 2879 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2880 xn_params.recv_mem = (struct kvec) {}; 2881 } 2882 2883 out: 2884 adapter->splitq_pt_lkup = splitq_pt_lkup; 2885 adapter->singleq_pt_lkup = singleq_pt_lkup; 2886 splitq_pt_lkup = NULL; 2887 singleq_pt_lkup = NULL; 2888 free_rx_buf: 2889 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2890 free_splitq: 2891 kfree(splitq_pt_lkup); 2892 free_singleq: 2893 kfree(singleq_pt_lkup); 2894 2895 return err; 2896 } 2897 2898 /** 2899 * idpf_rel_rx_pt_lkup - release RX ptype lookup table 2900 * @adapter: adapter pointer to get the lookup table 2901 */ 2902 static void idpf_rel_rx_pt_lkup(struct idpf_adapter *adapter) 2903 { 2904 kfree(adapter->splitq_pt_lkup); 2905 adapter->splitq_pt_lkup = NULL; 2906 2907 kfree(adapter->singleq_pt_lkup); 2908 adapter->singleq_pt_lkup = NULL; 2909 } 2910 2911 /** 2912 * idpf_send_ena_dis_loopback_msg - Send virtchnl enable/disable loopback 2913 * message 2914 * @adapter: adapter pointer used to send virtchnl message 2915 * @vport_id: vport identifier used while preparing the virtchnl message 2916 * @loopback_ena: flag to enable or disable loopback 2917 * 2918 * Return: 0 on success, negative on failure. 2919 */ 2920 int idpf_send_ena_dis_loopback_msg(struct idpf_adapter *adapter, u32 vport_id, 2921 bool loopback_ena) 2922 { 2923 struct libie_ctlq_xn_send_params xn_params = { 2924 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2925 .chnl_opcode = VIRTCHNL2_OP_LOOPBACK, 2926 }; 2927 struct virtchnl2_loopback loopback; 2928 int err; 2929 2930 loopback.vport_id = cpu_to_le32(vport_id); 2931 loopback.enable = loopback_ena; 2932 2933 err = idpf_send_mb_msg_stack(adapter, &xn_params, &loopback); 2934 if (err) 2935 return err; 2936 2937 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 2938 2939 return 0; 2940 } 2941 2942 /** 2943 * idpf_init_dflt_mbx - Setup default mailbox parameters and make request 2944 * @adapter: adapter info struct 2945 * 2946 * Returns 0 on success, negative otherwise 2947 */ 2948 int idpf_init_dflt_mbx(struct idpf_adapter *adapter) 2949 { 2950 struct libie_ctlq_ctx *ctx = &adapter->ctlq_ctx; 2951 struct libie_ctlq_create_info ctlq_info[] = { 2952 { 2953 .type = LIBIE_CTLQ_TYPE_TX, 2954 .id = LIBIE_CTLQ_MBX_ID, 2955 .len = IDPF_DFLT_MBX_Q_LEN, 2956 }, 2957 { 2958 .type = LIBIE_CTLQ_TYPE_RX, 2959 .id = LIBIE_CTLQ_MBX_ID, 2960 .len = IDPF_DFLT_MBX_Q_LEN, 2961 } 2962 }; 2963 struct libie_ctlq_xn_init_params params = { 2964 .num_qs = IDPF_NUM_DFLT_MBX_Q, 2965 .cctlq_info = ctlq_info, 2966 .ctx = ctx, 2967 }; 2968 int err; 2969 2970 adapter->dev_ops.reg_ops.ctlq_reg_init(&ctx->mmio_info, 2971 params.cctlq_info); 2972 2973 err = libie_ctlq_xn_init(¶ms); 2974 if (err) 2975 return err; 2976 2977 adapter->asq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_TX, 2978 LIBIE_CTLQ_MBX_ID); 2979 adapter->arq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_RX, 2980 LIBIE_CTLQ_MBX_ID); 2981 if (!adapter->asq || !adapter->arq) { 2982 adapter->asq = NULL; 2983 adapter->arq = NULL; 2984 libie_ctlq_xn_deinit(params.xnm, ctx); 2985 return -ENOENT; 2986 } 2987 2988 adapter->xnm = params.xnm; 2989 adapter->state = __IDPF_VER_CHECK; 2990 2991 queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); 2992 2993 return 0; 2994 } 2995 2996 /** 2997 * idpf_deinit_dflt_mbx - Free up ctlqs setup 2998 * @adapter: Driver specific private data structure 2999 */ 3000 void idpf_deinit_dflt_mbx(struct idpf_adapter *adapter) 3001 { 3002 idpf_mb_intr_rel_irq(adapter); 3003 cancel_delayed_work_sync(&adapter->mbx_task); 3004 3005 if (adapter->xnm) { 3006 libie_ctlq_xn_shutdown(adapter->xnm); 3007 idpf_mb_clean(adapter->asq, true); 3008 libie_ctlq_xn_deinit(adapter->xnm, &adapter->ctlq_ctx); 3009 } 3010 3011 adapter->arq = NULL; 3012 adapter->asq = NULL; 3013 adapter->xnm = NULL; 3014 } 3015 3016 /** 3017 * idpf_vport_params_buf_rel - Release memory for MailBox resources 3018 * @adapter: Driver specific private data structure 3019 * 3020 * Will release memory to hold the vport parameters received on MailBox 3021 */ 3022 static void idpf_vport_params_buf_rel(struct idpf_adapter *adapter) 3023 { 3024 kfree(adapter->vport_params_recvd); 3025 adapter->vport_params_recvd = NULL; 3026 kfree(adapter->vport_ids); 3027 adapter->vport_ids = NULL; 3028 } 3029 3030 /** 3031 * idpf_vport_params_buf_alloc - Allocate memory for MailBox resources 3032 * @adapter: Driver specific private data structure 3033 * 3034 * Will alloc memory to hold the vport parameters received on MailBox 3035 */ 3036 static int idpf_vport_params_buf_alloc(struct idpf_adapter *adapter) 3037 { 3038 u16 num_max_vports = idpf_get_max_vports(adapter); 3039 3040 adapter->vport_params_recvd = kzalloc_objs(*adapter->vport_params_recvd, 3041 num_max_vports); 3042 if (!adapter->vport_params_recvd) 3043 return -ENOMEM; 3044 3045 adapter->vport_ids = kcalloc(num_max_vports, sizeof(u32), GFP_KERNEL); 3046 if (!adapter->vport_ids) 3047 goto err_mem; 3048 3049 if (adapter->vport_config) 3050 return 0; 3051 3052 adapter->vport_config = kzalloc_objs(*adapter->vport_config, 3053 num_max_vports); 3054 if (!adapter->vport_config) 3055 goto err_mem; 3056 3057 return 0; 3058 3059 err_mem: 3060 idpf_vport_params_buf_rel(adapter); 3061 3062 return -ENOMEM; 3063 } 3064 3065 /** 3066 * idpf_vc_core_init - Initialize state machine and get driver specific 3067 * resources 3068 * @adapter: Driver specific private structure 3069 * 3070 * This function will initialize the state machine and request all necessary 3071 * resources required by the device driver. Once the state machine is 3072 * initialized, allocate memory to store vport specific information and also 3073 * requests required interrupts. 3074 * 3075 * Returns 0 on success, -EAGAIN function will get called again, 3076 * otherwise negative on failure. 3077 */ 3078 int idpf_vc_core_init(struct idpf_adapter *adapter) 3079 { 3080 int task_delay = 30; 3081 u16 num_max_vports; 3082 int err = 0; 3083 3084 while (adapter->state != __IDPF_INIT_SW) { 3085 switch (adapter->state) { 3086 case __IDPF_VER_CHECK: 3087 err = idpf_send_ver_msg(adapter); 3088 switch (err) { 3089 case 0: 3090 /* success, move state machine forward */ 3091 adapter->state = __IDPF_GET_CAPS; 3092 fallthrough; 3093 case -EAGAIN: 3094 goto restart; 3095 default: 3096 /* Something bad happened, try again but only a 3097 * few times. 3098 */ 3099 goto init_failed; 3100 } 3101 case __IDPF_GET_CAPS: 3102 err = idpf_send_get_caps_msg(adapter); 3103 if (err) 3104 goto init_failed; 3105 adapter->state = __IDPF_INIT_SW; 3106 break; 3107 default: 3108 dev_err(&adapter->pdev->dev, "Device is in bad state: %d\n", 3109 adapter->state); 3110 err = -EINVAL; 3111 goto init_failed; 3112 } 3113 break; 3114 restart: 3115 /* Give enough time before proceeding further with 3116 * state machine 3117 */ 3118 msleep(task_delay); 3119 } 3120 3121 if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_LAN_MEMORY_REGIONS)) { 3122 err = idpf_cfg_lan_memory_regions(adapter); 3123 if (err) { 3124 dev_err(&adapter->pdev->dev, "Failed to configure LAN memory regions: %d\n", 3125 err); 3126 return -EINVAL; 3127 } 3128 } else { 3129 /* Fallback to mapping the remaining regions of the entire BAR */ 3130 err = idpf_map_remaining_mmio_regs(adapter); 3131 if (err) { 3132 dev_err(&adapter->pdev->dev, "Failed to configure BAR0 region(s): %d\n", 3133 err); 3134 return err; 3135 } 3136 } 3137 3138 memset(&adapter->edt_caps, 0, sizeof(adapter->edt_caps)); 3139 if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_EDT)) { 3140 err = idpf_send_get_edt_caps_msg(adapter); 3141 if (err) 3142 dev_err(&adapter->pdev->dev, 3143 "EDT init failed, err=%d\n", err); 3144 } 3145 3146 pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter)); 3147 num_max_vports = idpf_get_max_vports(adapter); 3148 adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports); 3149 if (!adapter->vports) { 3150 err = -ENOMEM; 3151 goto decfg_regions; 3152 } 3153 3154 if (!adapter->netdevs) { 3155 adapter->netdevs = kzalloc_objs(struct net_device *, 3156 num_max_vports); 3157 if (!adapter->netdevs) { 3158 err = -ENOMEM; 3159 goto err_netdev_alloc; 3160 } 3161 } 3162 3163 err = idpf_vport_params_buf_alloc(adapter); 3164 if (err) { 3165 dev_err(&adapter->pdev->dev, "Failed to alloc vport params buffer: %d\n", 3166 err); 3167 goto err_netdev_alloc; 3168 } 3169 3170 /* Set max_vports only after vports, netdevs and vport_config buffers 3171 * are allocated to make sure max_vport bound loops don't end up 3172 * crashing, following allocation errors on init. 3173 */ 3174 adapter->max_vports = num_max_vports; 3175 3176 /* Start the mailbox task before requesting vectors. This will ensure 3177 * vector information response from mailbox is handled 3178 */ 3179 queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); 3180 3181 queue_delayed_work(adapter->serv_wq, &adapter->serv_task, 3182 msecs_to_jiffies(5 * (adapter->pdev->devfn & 0x07))); 3183 3184 err = idpf_intr_req(adapter); 3185 if (err) { 3186 dev_err(&adapter->pdev->dev, "failed to enable interrupt vectors: %d\n", 3187 err); 3188 goto err_intr_req; 3189 } 3190 3191 err = idpf_send_get_rx_ptype_msg(adapter); 3192 if (err) { 3193 dev_err(&adapter->pdev->dev, "failed to get RX ptypes: %d\n", 3194 err); 3195 goto intr_rel; 3196 } 3197 3198 err = idpf_ptp_init(adapter); 3199 if (err) 3200 pci_err(adapter->pdev, "PTP init failed, err=%pe\n", 3201 ERR_PTR(err)); 3202 3203 idpf_init_avail_queues(adapter); 3204 3205 /* Skew the delay for init tasks for each function based on fn number 3206 * to prevent every function from making the same call simultaneously. 3207 */ 3208 queue_delayed_work(adapter->init_wq, &adapter->init_task, 3209 msecs_to_jiffies(5 * (adapter->pdev->devfn & 0x07))); 3210 3211 set_bit(IDPF_VC_CORE_INIT, adapter->flags); 3212 3213 return 0; 3214 3215 intr_rel: 3216 idpf_intr_rel(adapter); 3217 err_intr_req: 3218 cancel_delayed_work_sync(&adapter->serv_task); 3219 cancel_delayed_work_sync(&adapter->mbx_task); 3220 idpf_vport_params_buf_rel(adapter); 3221 err_netdev_alloc: 3222 kfree(adapter->vports); 3223 adapter->vports = NULL; 3224 decfg_regions: 3225 idpf_decfg_lan_memory_regions(adapter); 3226 return err; 3227 3228 init_failed: 3229 /* Don't retry if we're trying to go down, just bail. */ 3230 if (test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) 3231 return err; 3232 3233 if (++adapter->mb_wait_count > IDPF_MB_MAX_ERR) { 3234 dev_err(&adapter->pdev->dev, "Failed to establish mailbox communications with hardware\n"); 3235 3236 return -EFAULT; 3237 } 3238 /* If it reached here, it is possible that mailbox queue initialization 3239 * register writes might not have taken effect. Retry to initialize 3240 * the mailbox again 3241 */ 3242 adapter->state = __IDPF_VER_CHECK; 3243 libie_ctlq_xn_shutdown(adapter->xnm); 3244 set_bit(IDPF_HR_DRV_LOAD, adapter->flags); 3245 queue_delayed_work(adapter->vc_event_wq, &adapter->vc_event_task, 3246 msecs_to_jiffies(task_delay)); 3247 3248 return -EAGAIN; 3249 } 3250 3251 /** 3252 * idpf_vc_core_deinit - Device deinit routine 3253 * @adapter: Driver specific private structure 3254 * 3255 */ 3256 void idpf_vc_core_deinit(struct idpf_adapter *adapter) 3257 { 3258 bool remove_in_prog; 3259 3260 if (!test_bit(IDPF_VC_CORE_INIT, adapter->flags)) 3261 return; 3262 3263 /* Avoid transaction timeouts when called during reset */ 3264 remove_in_prog = test_bit(IDPF_REMOVE_IN_PROG, adapter->flags); 3265 if (!remove_in_prog) 3266 libie_ctlq_xn_shutdown(adapter->xnm); 3267 3268 idpf_ptp_release(adapter); 3269 idpf_deinit_task(adapter); 3270 idpf_idc_deinit_core_aux_device(adapter); 3271 idpf_rel_rx_pt_lkup(adapter); 3272 idpf_intr_rel(adapter); 3273 3274 if (remove_in_prog) 3275 libie_ctlq_xn_shutdown(adapter->xnm); 3276 3277 cancel_delayed_work_sync(&adapter->serv_task); 3278 cancel_delayed_work_sync(&adapter->mbx_task); 3279 3280 idpf_vport_params_buf_rel(adapter); 3281 3282 kfree(adapter->vports); 3283 adapter->vports = NULL; 3284 3285 idpf_decfg_lan_memory_regions(adapter); 3286 clear_bit(IDPF_VC_CORE_INIT, adapter->flags); 3287 } 3288 3289 /** 3290 * idpf_vport_alloc_vec_indexes - Get relative vector indexes 3291 * @vport: virtual port data struct 3292 * @rsrc: pointer to queue and vector resources 3293 * 3294 * This function requests the vector information required for the vport and 3295 * stores the vector indexes received from the 'global vector distribution' 3296 * in the vport's queue vectors array. 3297 * 3298 * Return: 0 on success, error on failure 3299 */ 3300 int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport, 3301 struct idpf_q_vec_rsrc *rsrc) 3302 { 3303 struct idpf_vector_info vec_info; 3304 int num_alloc_vecs; 3305 u32 req; 3306 3307 vec_info.num_curr_vecs = rsrc->num_q_vectors; 3308 if (vec_info.num_curr_vecs) 3309 vec_info.num_curr_vecs += IDPF_RESERVED_VECS; 3310 3311 /* XDPSQs are all bound to the NOIRQ vector from IDPF_RESERVED_VECS */ 3312 req = max(rsrc->num_txq - vport->num_xdp_txq, rsrc->num_rxq) + 3313 IDPF_RESERVED_VECS; 3314 vec_info.num_req_vecs = req; 3315 3316 vec_info.default_vport = vport->default_vport; 3317 vec_info.index = vport->idx; 3318 3319 num_alloc_vecs = idpf_req_rel_vector_indexes(vport->adapter, 3320 rsrc->q_vector_idxs, 3321 &vec_info); 3322 if (num_alloc_vecs <= 0) { 3323 dev_err(&vport->adapter->pdev->dev, "Vector distribution failed: %d\n", 3324 num_alloc_vecs); 3325 return -EINVAL; 3326 } 3327 3328 rsrc->num_q_vectors = num_alloc_vecs - IDPF_RESERVED_VECS; 3329 3330 return 0; 3331 } 3332 3333 /** 3334 * idpf_vport_init - Initialize virtual port 3335 * @vport: virtual port to be initialized 3336 * @max_q: vport max queue info 3337 * 3338 * Will initialize vport with the info received through MB earlier 3339 * 3340 * Return: 0 on success, negative on failure. 3341 */ 3342 int idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q) 3343 { 3344 struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; 3345 struct idpf_adapter *adapter = vport->adapter; 3346 struct virtchnl2_create_vport *vport_msg; 3347 struct idpf_vport_config *vport_config; 3348 u16 tx_itr[] = {2, 8, 64, 128, 256}; 3349 u16 rx_itr[] = {2, 8, 32, 96, 128}; 3350 struct idpf_rss_data *rss_data; 3351 u16 idx = vport->idx; 3352 int err; 3353 3354 vport_config = adapter->vport_config[idx]; 3355 rss_data = &vport_config->user_config.rss_data; 3356 vport_msg = adapter->vport_params_recvd[idx]; 3357 3358 err = idpf_vport_init_queue_reg_chunks(vport_config, 3359 &vport_msg->chunks); 3360 if (err) 3361 return err; 3362 3363 vport_config->max_q.max_txq = max_q->max_txq; 3364 vport_config->max_q.max_rxq = max_q->max_rxq; 3365 vport_config->max_q.max_complq = max_q->max_complq; 3366 vport_config->max_q.max_bufq = max_q->max_bufq; 3367 3368 rsrc->txq_model = le16_to_cpu(vport_msg->txq_model); 3369 rsrc->rxq_model = le16_to_cpu(vport_msg->rxq_model); 3370 vport->vport_type = le16_to_cpu(vport_msg->vport_type); 3371 vport->vport_id = le32_to_cpu(vport_msg->vport_id); 3372 3373 rss_data->rss_key_size = min_t(u16, NETDEV_RSS_KEY_LEN, 3374 le16_to_cpu(vport_msg->rss_key_size)); 3375 rss_data->rss_lut_size = le16_to_cpu(vport_msg->rss_lut_size); 3376 3377 ether_addr_copy(vport->default_mac_addr, vport_msg->default_mac_addr); 3378 vport->max_mtu = le16_to_cpu(vport_msg->max_mtu) - LIBETH_RX_LL_LEN; 3379 3380 /* Initialize Tx and Rx profiles for Dynamic Interrupt Moderation */ 3381 memcpy(vport->rx_itr_profile, rx_itr, IDPF_DIM_PROFILE_SLOTS); 3382 memcpy(vport->tx_itr_profile, tx_itr, IDPF_DIM_PROFILE_SLOTS); 3383 3384 idpf_vport_set_hsplit(vport, ETHTOOL_TCP_DATA_SPLIT_ENABLED); 3385 3386 idpf_vport_init_num_qs(vport, vport_msg, rsrc); 3387 idpf_vport_calc_num_q_desc(vport, rsrc); 3388 idpf_vport_calc_num_q_groups(rsrc); 3389 idpf_vport_alloc_vec_indexes(vport, rsrc); 3390 3391 vport->crc_enable = adapter->crc_enable; 3392 3393 if (!(vport_msg->vport_flags & 3394 cpu_to_le16(VIRTCHNL2_VPORT_UPLINK_PORT))) 3395 return 0; 3396 3397 err = idpf_ptp_get_vport_tstamps_caps(vport); 3398 if (err) { 3399 /* Do not error on timestamp failure */ 3400 pci_dbg(vport->adapter->pdev, "Tx timestamping not supported\n"); 3401 return 0; 3402 } 3403 3404 INIT_WORK(&vport->tstamp_task, idpf_tstamp_task); 3405 3406 return 0; 3407 } 3408 3409 /** 3410 * idpf_get_vec_ids - Initialize vector id from Mailbox parameters 3411 * @adapter: adapter structure to get the mailbox vector id 3412 * @vecids: Array of vector ids 3413 * @num_vecids: number of vector ids 3414 * @chunks: vector ids received over mailbox 3415 * 3416 * Will initialize the mailbox vector id which is received from the 3417 * get capabilities and data queue vector ids with ids received as 3418 * mailbox parameters. 3419 * Returns number of ids filled 3420 */ 3421 int idpf_get_vec_ids(struct idpf_adapter *adapter, 3422 u16 *vecids, int num_vecids, 3423 struct virtchnl2_vector_chunks *chunks) 3424 { 3425 u16 num_chunks = le16_to_cpu(chunks->num_vchunks); 3426 int num_vecid_filled = 0; 3427 int i, j; 3428 3429 vecids[num_vecid_filled] = adapter->mb_vector.v_idx; 3430 num_vecid_filled++; 3431 3432 for (j = 0; j < num_chunks; j++) { 3433 struct virtchnl2_vector_chunk *chunk; 3434 u16 start_vecid, num_vec; 3435 3436 chunk = &chunks->vchunks[j]; 3437 num_vec = le16_to_cpu(chunk->num_vectors); 3438 start_vecid = le16_to_cpu(chunk->start_vector_id); 3439 3440 for (i = 0; i < num_vec; i++) { 3441 if ((num_vecid_filled + i) < num_vecids) { 3442 vecids[num_vecid_filled + i] = start_vecid; 3443 start_vecid++; 3444 } else { 3445 break; 3446 } 3447 } 3448 num_vecid_filled = num_vecid_filled + i; 3449 } 3450 3451 return num_vecid_filled; 3452 } 3453 3454 /** 3455 * idpf_vport_get_queue_ids - Initialize queue id from Mailbox parameters 3456 * @qids: Array of queue ids 3457 * @num_qids: number of queue ids 3458 * @q_type: queue model 3459 * @chunks: queue ids received over mailbox 3460 * 3461 * Will initialize all queue ids with ids received as mailbox parameters 3462 * Returns number of ids filled 3463 */ 3464 static int idpf_vport_get_queue_ids(u32 *qids, int num_qids, u16 q_type, 3465 struct idpf_queue_id_reg_info *chunks) 3466 { 3467 u16 num_chunks = chunks->num_chunks; 3468 u32 num_q_id_filled = 0, i; 3469 u32 start_q_id, num_q; 3470 3471 while (num_chunks--) { 3472 struct idpf_queue_id_reg_chunk *chunk; 3473 3474 chunk = &chunks->queue_chunks[num_chunks]; 3475 if (chunk->type != q_type) 3476 continue; 3477 3478 num_q = chunk->num_queues; 3479 start_q_id = chunk->start_queue_id; 3480 3481 for (i = 0; i < num_q; i++) { 3482 if ((num_q_id_filled + i) < num_qids) { 3483 qids[num_q_id_filled + i] = start_q_id; 3484 start_q_id++; 3485 } else { 3486 break; 3487 } 3488 } 3489 num_q_id_filled = num_q_id_filled + i; 3490 } 3491 3492 return num_q_id_filled; 3493 } 3494 3495 /** 3496 * __idpf_vport_queue_ids_init - Initialize queue ids from Mailbox parameters 3497 * @vport: virtual port for which the queues ids are initialized 3498 * @rsrc: pointer to queue and vector resources 3499 * @qids: queue ids 3500 * @num_qids: number of queue ids 3501 * @q_type: type of queue 3502 * 3503 * Will initialize all queue ids with ids received as mailbox 3504 * parameters. Returns number of queue ids initialized. 3505 */ 3506 static int __idpf_vport_queue_ids_init(struct idpf_vport *vport, 3507 struct idpf_q_vec_rsrc *rsrc, 3508 const u32 *qids, 3509 int num_qids, 3510 u32 q_type) 3511 { 3512 int i, j, k = 0; 3513 3514 switch (q_type) { 3515 case VIRTCHNL2_QUEUE_TYPE_TX: 3516 for (i = 0; i < rsrc->num_txq_grp; i++) { 3517 struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 3518 3519 for (j = 0; j < tx_qgrp->num_txq && k < num_qids; j++, k++) 3520 tx_qgrp->txqs[j]->q_id = qids[k]; 3521 } 3522 break; 3523 case VIRTCHNL2_QUEUE_TYPE_RX: 3524 for (i = 0; i < rsrc->num_rxq_grp; i++) { 3525 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 3526 u16 num_rxq; 3527 3528 if (idpf_is_queue_model_split(rsrc->rxq_model)) 3529 num_rxq = rx_qgrp->splitq.num_rxq_sets; 3530 else 3531 num_rxq = rx_qgrp->singleq.num_rxq; 3532 3533 for (j = 0; j < num_rxq && k < num_qids; j++, k++) { 3534 struct idpf_rx_queue *q; 3535 3536 if (idpf_is_queue_model_split(rsrc->rxq_model)) 3537 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 3538 else 3539 q = rx_qgrp->singleq.rxqs[j]; 3540 q->q_id = qids[k]; 3541 } 3542 } 3543 break; 3544 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 3545 for (i = 0; i < rsrc->num_txq_grp && k < num_qids; i++, k++) { 3546 struct idpf_txq_group *tx_qgrp = &rsrc->txq_grps[i]; 3547 3548 tx_qgrp->complq->q_id = qids[k]; 3549 } 3550 break; 3551 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 3552 for (i = 0; i < rsrc->num_rxq_grp; i++) { 3553 struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 3554 u8 num_bufqs = rsrc->num_bufqs_per_qgrp; 3555 3556 for (j = 0; j < num_bufqs && k < num_qids; j++, k++) { 3557 struct idpf_buf_queue *q; 3558 3559 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 3560 q->q_id = qids[k]; 3561 } 3562 } 3563 break; 3564 default: 3565 break; 3566 } 3567 3568 return k; 3569 } 3570 3571 /** 3572 * idpf_vport_queue_ids_init - Initialize queue ids from Mailbox parameters 3573 * @vport: virtual port for which the queues ids are initialized 3574 * @rsrc: pointer to queue and vector resources 3575 * @chunks: queue ids received over mailbox 3576 * 3577 * Will initialize all queue ids with ids received as mailbox parameters. 3578 * 3579 * Return: 0 on success, negative if all the queues are not initialized. 3580 */ 3581 int idpf_vport_queue_ids_init(struct idpf_vport *vport, 3582 struct idpf_q_vec_rsrc *rsrc, 3583 struct idpf_queue_id_reg_info *chunks) 3584 { 3585 int num_ids, err = 0; 3586 u16 q_type; 3587 u32 *qids; 3588 3589 qids = kcalloc(IDPF_MAX_QIDS, sizeof(u32), GFP_KERNEL); 3590 if (!qids) 3591 return -ENOMEM; 3592 3593 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, 3594 VIRTCHNL2_QUEUE_TYPE_TX, 3595 chunks); 3596 if (num_ids < rsrc->num_txq) { 3597 err = -EINVAL; 3598 goto mem_rel; 3599 } 3600 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, num_ids, 3601 VIRTCHNL2_QUEUE_TYPE_TX); 3602 if (num_ids < rsrc->num_txq) { 3603 err = -EINVAL; 3604 goto mem_rel; 3605 } 3606 3607 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, 3608 VIRTCHNL2_QUEUE_TYPE_RX, 3609 chunks); 3610 if (num_ids < rsrc->num_rxq) { 3611 err = -EINVAL; 3612 goto mem_rel; 3613 } 3614 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, num_ids, 3615 VIRTCHNL2_QUEUE_TYPE_RX); 3616 if (num_ids < rsrc->num_rxq) { 3617 err = -EINVAL; 3618 goto mem_rel; 3619 } 3620 3621 if (!idpf_is_queue_model_split(rsrc->txq_model)) 3622 goto check_rxq; 3623 3624 q_type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 3625 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, q_type, chunks); 3626 if (num_ids < rsrc->num_complq) { 3627 err = -EINVAL; 3628 goto mem_rel; 3629 } 3630 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, 3631 num_ids, q_type); 3632 if (num_ids < rsrc->num_complq) { 3633 err = -EINVAL; 3634 goto mem_rel; 3635 } 3636 3637 check_rxq: 3638 if (!idpf_is_queue_model_split(rsrc->rxq_model)) 3639 goto mem_rel; 3640 3641 q_type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 3642 num_ids = idpf_vport_get_queue_ids(qids, IDPF_MAX_QIDS, q_type, chunks); 3643 if (num_ids < rsrc->num_bufq) { 3644 err = -EINVAL; 3645 goto mem_rel; 3646 } 3647 num_ids = __idpf_vport_queue_ids_init(vport, rsrc, qids, 3648 num_ids, q_type); 3649 if (num_ids < rsrc->num_bufq) 3650 err = -EINVAL; 3651 3652 mem_rel: 3653 kfree(qids); 3654 3655 return err; 3656 } 3657 3658 /** 3659 * idpf_vport_adjust_qs - Adjust to new requested queues 3660 * @vport: virtual port data struct 3661 * @rsrc: pointer to queue and vector resources 3662 * 3663 * Renegotiate queues. Returns 0 on success, negative on failure. 3664 */ 3665 int idpf_vport_adjust_qs(struct idpf_vport *vport, struct idpf_q_vec_rsrc *rsrc) 3666 { 3667 struct virtchnl2_create_vport vport_msg; 3668 int err; 3669 3670 vport_msg.txq_model = cpu_to_le16(rsrc->txq_model); 3671 vport_msg.rxq_model = cpu_to_le16(rsrc->rxq_model); 3672 err = idpf_vport_calc_total_qs(vport->adapter, vport->idx, &vport_msg, 3673 NULL); 3674 if (err) 3675 return err; 3676 3677 idpf_vport_init_num_qs(vport, &vport_msg, rsrc); 3678 idpf_vport_calc_num_q_groups(rsrc); 3679 3680 return 0; 3681 } 3682 3683 /** 3684 * idpf_is_capability_ena - Default implementation of capability checking 3685 * @adapter: Private data struct 3686 * @all: all or one flag 3687 * @field: caps field to check for flags 3688 * @flag: flag to check 3689 * 3690 * Return true if all capabilities are supported, false otherwise 3691 */ 3692 bool idpf_is_capability_ena(struct idpf_adapter *adapter, bool all, 3693 enum idpf_cap_field field, u64 flag) 3694 { 3695 u8 *caps = (u8 *)&adapter->caps; 3696 u32 *cap_field; 3697 3698 if (!caps) 3699 return false; 3700 3701 if (field == IDPF_BASE_CAPS) 3702 return false; 3703 3704 cap_field = (u32 *)(caps + field); 3705 3706 if (all) 3707 return (*cap_field & flag) == flag; 3708 else 3709 return !!(*cap_field & flag); 3710 } 3711 3712 /** 3713 * idpf_vport_is_cap_ena - Check if vport capability is enabled 3714 * @vport: Private data struct 3715 * @flag: flag(s) to check 3716 * 3717 * Return: true if the capability is supported, false otherwise 3718 */ 3719 bool idpf_vport_is_cap_ena(struct idpf_vport *vport, u16 flag) 3720 { 3721 struct virtchnl2_create_vport *vport_msg; 3722 3723 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3724 3725 return !!(le16_to_cpu(vport_msg->vport_flags) & flag); 3726 } 3727 3728 /** 3729 * idpf_sideband_flow_type_ena - Check if steering is enabled for flow type 3730 * @vport: Private data struct 3731 * @flow_type: flow type to check (from ethtool.h) 3732 * 3733 * Return: true if sideband filters are allowed for @flow_type, false otherwise 3734 */ 3735 bool idpf_sideband_flow_type_ena(struct idpf_vport *vport, u32 flow_type) 3736 { 3737 struct virtchnl2_create_vport *vport_msg; 3738 __le64 caps; 3739 3740 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3741 caps = vport_msg->sideband_flow_caps; 3742 3743 switch (flow_type) { 3744 case TCP_V4_FLOW: 3745 return !!(caps & cpu_to_le64(VIRTCHNL2_FLOW_IPV4_TCP)); 3746 case UDP_V4_FLOW: 3747 return !!(caps & cpu_to_le64(VIRTCHNL2_FLOW_IPV4_UDP)); 3748 default: 3749 return false; 3750 } 3751 } 3752 3753 /** 3754 * idpf_sideband_action_ena - Check if steering is enabled for action 3755 * @vport: Private data struct 3756 * @fsp: flow spec 3757 * 3758 * Return: true if sideband filters are allowed for @fsp, false otherwise 3759 */ 3760 bool idpf_sideband_action_ena(struct idpf_vport *vport, 3761 struct ethtool_rx_flow_spec *fsp) 3762 { 3763 struct virtchnl2_create_vport *vport_msg; 3764 unsigned int supp_actions; 3765 3766 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3767 supp_actions = le32_to_cpu(vport_msg->sideband_flow_actions); 3768 3769 /* Actions Drop/Wake are not supported */ 3770 if (fsp->ring_cookie == RX_CLS_FLOW_DISC || 3771 fsp->ring_cookie == RX_CLS_FLOW_WAKE) 3772 return false; 3773 3774 return !!(supp_actions & VIRTCHNL2_ACTION_QUEUE); 3775 } 3776 3777 unsigned int idpf_fsteer_max_rules(struct idpf_vport *vport) 3778 { 3779 struct virtchnl2_create_vport *vport_msg; 3780 3781 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3782 return le32_to_cpu(vport_msg->flow_steer_max_rules); 3783 } 3784 3785 /** 3786 * idpf_get_vport_id: Get vport id 3787 * @vport: virtual port structure 3788 * 3789 * Return vport id from the adapter persistent data 3790 */ 3791 u32 idpf_get_vport_id(struct idpf_vport *vport) 3792 { 3793 struct virtchnl2_create_vport *vport_msg; 3794 3795 vport_msg = vport->adapter->vport_params_recvd[vport->idx]; 3796 3797 return le32_to_cpu(vport_msg->vport_id); 3798 } 3799 3800 static void idpf_set_mac_type(const u8 *default_mac_addr, 3801 struct virtchnl2_mac_addr *mac_addr) 3802 { 3803 bool is_primary; 3804 3805 is_primary = ether_addr_equal(default_mac_addr, mac_addr->addr); 3806 mac_addr->type = is_primary ? VIRTCHNL2_MAC_ADDR_PRIMARY : 3807 VIRTCHNL2_MAC_ADDR_EXTRA; 3808 } 3809 3810 /** 3811 * idpf_mac_filter_async_handler - Async callback for mac filters 3812 * @ctx: controlq context structure 3813 * @buff: response buffer pointer and size 3814 * @status: async call return value 3815 * 3816 * In some scenarios driver can't sleep and wait for a reply (e.g.: stack is 3817 * holding rtnl_lock) when adding a new mac filter. It puts us in a difficult 3818 * situation to deal with errors returned on the reply. The best we can 3819 * ultimately do is remove it from our list of mac filters and report the 3820 * error. 3821 */ 3822 static void idpf_mac_filter_async_handler(void *ctx, 3823 struct kvec *buff, 3824 int status) 3825 { 3826 struct virtchnl2_mac_addr_list *ma_list; 3827 struct idpf_vport_config *vport_config; 3828 struct virtchnl2_mac_addr *mac_addr; 3829 struct idpf_adapter *adapter = ctx; 3830 struct idpf_mac_filter *f, *tmp; 3831 struct list_head *ma_list_head; 3832 struct idpf_vport *vport; 3833 u16 num_entries; 3834 int i; 3835 3836 /* if success we're done, we're only here if something bad happened */ 3837 if (!status || status == -ETIMEDOUT) 3838 return; 3839 3840 ma_list = buff->iov_base; 3841 /* make sure at least struct is there */ 3842 if (buff->iov_len < sizeof(*ma_list)) 3843 goto invalid_payload; 3844 3845 mac_addr = ma_list->mac_addr_list; 3846 num_entries = le16_to_cpu(ma_list->num_mac_addr); 3847 /* we should have received a buffer at least this big */ 3848 if (buff->iov_len < struct_size(ma_list, mac_addr_list, num_entries)) 3849 goto invalid_payload; 3850 3851 vport = idpf_vid_to_vport(adapter, le32_to_cpu(ma_list->vport_id)); 3852 if (!vport) 3853 goto invalid_payload; 3854 3855 vport_config = adapter->vport_config[le32_to_cpu(ma_list->vport_id)]; 3856 ma_list_head = &vport_config->user_config.mac_filter_list; 3857 3858 /* We can't do much to reconcile bad filters at this point, however we 3859 * should at least remove them from our list one way or the other so we 3860 * have some idea what good filters we have. 3861 */ 3862 spin_lock_bh(&vport_config->mac_filter_list_lock); 3863 list_for_each_entry_safe(f, tmp, ma_list_head, list) 3864 for (i = 0; i < num_entries; i++) 3865 if (ether_addr_equal(mac_addr[i].addr, f->macaddr)) 3866 list_del(&f->list); 3867 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3868 dev_err_ratelimited(&adapter->pdev->dev, "Received error %d on sending MAC filter request\n", 3869 status); 3870 return; 3871 3872 invalid_payload: 3873 dev_err_ratelimited(&adapter->pdev->dev, "Received invalid MAC filter payload (len %zd)\n", 3874 buff->iov_len); 3875 } 3876 3877 /** 3878 * idpf_add_del_mac_filters - Add/del mac filters 3879 * @adapter: adapter pointer used to send virtchnl message 3880 * @vport_config: persistent vport structure to get the MAC filter list 3881 * @default_mac_addr: default MAC address to compare with 3882 * @vport_id: vport identifier used while preparing the virtchnl message 3883 * @add: Add or delete flag 3884 * @async: Don't wait for return message 3885 * 3886 * Return: 0 on success, error on failure. 3887 **/ 3888 int idpf_add_del_mac_filters(struct idpf_adapter *adapter, 3889 struct idpf_vport_config *vport_config, 3890 const u8 *default_mac_addr, u32 vport_id, 3891 bool add, bool async) 3892 { 3893 struct virtchnl2_mac_addr *mac_addr __free(kfree) = NULL; 3894 struct libie_ctlq_xn_send_params xn_params = { 3895 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 3896 .chnl_opcode = add ? VIRTCHNL2_OP_ADD_MAC_ADDR : 3897 VIRTCHNL2_OP_DEL_MAC_ADDR, 3898 }; 3899 struct virtchnl2_mac_addr_list *ma_list; 3900 u32 num_msgs, total_filters = 0; 3901 struct idpf_mac_filter *f; 3902 int i = 0; 3903 3904 if (async) { 3905 xn_params.resp_cb = idpf_mac_filter_async_handler; 3906 xn_params.send_ctx = adapter; 3907 } 3908 3909 spin_lock_bh(&vport_config->mac_filter_list_lock); 3910 3911 /* Find the number of newly added filters */ 3912 list_for_each_entry(f, &vport_config->user_config.mac_filter_list, 3913 list) { 3914 if (add && f->add) 3915 total_filters++; 3916 else if (!add && f->remove) 3917 total_filters++; 3918 } 3919 3920 if (!total_filters) { 3921 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3922 3923 return 0; 3924 } 3925 3926 /* Fill all the new filters into virtchannel message */ 3927 mac_addr = kzalloc_objs(struct virtchnl2_mac_addr, total_filters, 3928 GFP_ATOMIC); 3929 if (!mac_addr) { 3930 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3931 3932 return -ENOMEM; 3933 } 3934 3935 list_for_each_entry(f, &vport_config->user_config.mac_filter_list, 3936 list) { 3937 if (add && f->add) { 3938 ether_addr_copy(mac_addr[i].addr, f->macaddr); 3939 idpf_set_mac_type(default_mac_addr, &mac_addr[i]); 3940 i++; 3941 f->add = false; 3942 if (i == total_filters) 3943 break; 3944 } 3945 if (!add && f->remove) { 3946 ether_addr_copy(mac_addr[i].addr, f->macaddr); 3947 idpf_set_mac_type(default_mac_addr, &mac_addr[i]); 3948 i++; 3949 f->remove = false; 3950 if (i == total_filters) 3951 break; 3952 } 3953 } 3954 3955 spin_unlock_bh(&vport_config->mac_filter_list_lock); 3956 3957 /* Chunk up the filters into multiple messages to avoid 3958 * sending a control queue message buffer that is too large 3959 */ 3960 num_msgs = DIV_ROUND_UP(total_filters, IDPF_NUM_FILTERS_PER_MSG); 3961 3962 for (u32 i = 0, k = 0; i < num_msgs; i++) { 3963 u32 entries_size, num_entries; 3964 size_t buf_size; 3965 int err; 3966 3967 num_entries = min_t(u32, total_filters, 3968 IDPF_NUM_FILTERS_PER_MSG); 3969 entries_size = sizeof(struct virtchnl2_mac_addr) * num_entries; 3970 buf_size = struct_size(ma_list, mac_addr_list, num_entries); 3971 3972 ma_list = kzalloc(buf_size, GFP_ATOMIC); 3973 if (!ma_list) 3974 return -ENOMEM; 3975 3976 ma_list->vport_id = cpu_to_le32(vport_id); 3977 ma_list->num_mac_addr = cpu_to_le16(num_entries); 3978 memcpy(ma_list->mac_addr_list, &mac_addr[k], entries_size); 3979 3980 err = idpf_send_mb_msg_kfree(adapter, &xn_params, ma_list, 3981 buf_size); 3982 if (err) 3983 return err; 3984 3985 if (!async) 3986 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 3987 3988 k += num_entries; 3989 total_filters -= num_entries; 3990 } 3991 3992 return 0; 3993 } 3994 3995 /** 3996 * idpf_promiscuous_async_handler - async callback for promiscuous mode 3997 * @ctx: controlq context structure 3998 * @buff: response buffer pointer and size 3999 * @status: async call return value 4000 * 4001 * Nobody is waiting for the promiscuous virtchnl message response. Print 4002 * an error message if something went wrong and return. 4003 */ 4004 static void idpf_promiscuous_async_handler(void *ctx, 4005 struct kvec *buff, 4006 int status) 4007 { 4008 struct idpf_adapter *adapter = ctx; 4009 4010 if (status) 4011 dev_err_ratelimited(&adapter->pdev->dev, "Failed to set promiscuous mode: %d\n", 4012 status); 4013 } 4014 4015 /** 4016 * idpf_set_promiscuous - set promiscuous and send message to mailbox 4017 * @adapter: Driver specific private structure 4018 * @config_data: Vport specific config data 4019 * @vport_id: Vport identifier 4020 * 4021 * Request to enable promiscuous mode for the vport. Message is sent 4022 * asynchronously and won't wait for response. Returns 0 on success, negative 4023 * on failure; 4024 */ 4025 int idpf_set_promiscuous(struct idpf_adapter *adapter, 4026 struct idpf_vport_user_config_data *config_data, 4027 u32 vport_id) 4028 { 4029 struct libie_ctlq_xn_send_params xn_params = { 4030 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 4031 .chnl_opcode = VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE, 4032 .resp_cb = idpf_promiscuous_async_handler, 4033 .send_ctx = adapter, 4034 }; 4035 struct virtchnl2_promisc_info vpi; 4036 u16 flags = 0; 4037 4038 if (test_bit(__IDPF_PROMISC_UC, config_data->user_flags)) 4039 flags |= VIRTCHNL2_UNICAST_PROMISC; 4040 if (test_bit(__IDPF_PROMISC_MC, config_data->user_flags)) 4041 flags |= VIRTCHNL2_MULTICAST_PROMISC; 4042 4043 vpi.vport_id = cpu_to_le32(vport_id); 4044 vpi.flags = cpu_to_le16(flags); 4045 4046 return idpf_send_mb_msg_stack(adapter, &xn_params, &vpi); 4047 } 4048 4049 /** 4050 * idpf_idc_rdma_vc_send_sync - virtchnl send callback for IDC registered drivers 4051 * @cdev_info: IDC core device info pointer 4052 * @send_msg: message to send 4053 * @msg_size: size of message to send 4054 * @recv_msg: message to populate on reception of response 4055 * @recv_len: on input, maximum response size; on success, actual response size 4056 * 4057 * Return: 0 on success or error code on failure. 4058 */ 4059 int idpf_idc_rdma_vc_send_sync(struct iidc_rdma_core_dev_info *cdev_info, 4060 u8 *send_msg, u16 msg_size, 4061 u8 *recv_msg, u16 *recv_len) 4062 { 4063 struct idpf_adapter *adapter = pci_get_drvdata(cdev_info->pdev); 4064 struct libie_ctlq_xn_send_params xn_params = { 4065 .chnl_opcode = VIRTCHNL2_OP_RDMA, 4066 .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 4067 }; 4068 u8 on_stack_buf[LIBIE_CP_TX_COPYBREAK]; 4069 void *send_buf; 4070 int err; 4071 4072 if (!recv_msg || !recv_len || msg_size > LIBIE_CTLQ_MAX_BUF_LEN) 4073 return -EINVAL; 4074 4075 if (!libie_cp_can_send_onstack(msg_size)) { 4076 send_buf = kzalloc(msg_size, GFP_KERNEL); 4077 if (!send_buf) 4078 return -ENOMEM; 4079 } else { 4080 send_buf = on_stack_buf; 4081 } 4082 4083 memcpy(send_buf, send_msg, msg_size); 4084 err = idpf_send_mb_msg(adapter, &xn_params, send_buf, msg_size); 4085 if (err) 4086 return err; 4087 4088 if (xn_params.recv_mem.iov_len > *recv_len) { 4089 err = -EINVAL; 4090 goto rel_buf; 4091 } 4092 4093 *recv_len = xn_params.recv_mem.iov_len; 4094 memcpy(recv_msg, xn_params.recv_mem.iov_base, *recv_len); 4095 rel_buf: 4096 libie_ctlq_release_rx_buf(&xn_params.recv_mem); 4097 return err; 4098 } 4099 EXPORT_SYMBOL_GPL(idpf_idc_rdma_vc_send_sync); 4100