1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2020 RackTop Systems, Inc. 16 */ 17 18 /* 19 * Mellanox Connect-X 4/5/6 driver. 20 * 21 * More details in mlxcx.c 22 */ 23 24 #ifndef _MLXCX_H 25 #define _MLXCX_H 26 27 /* 28 * mlxcx(7D) defintions 29 */ 30 31 #include <sys/ddi.h> 32 #include <sys/sunddi.h> 33 #include <sys/ddifm.h> 34 #include <sys/id_space.h> 35 #include <sys/list.h> 36 #include <sys/taskq_impl.h> 37 #include <sys/stddef.h> 38 #include <sys/stream.h> 39 #include <sys/strsun.h> 40 #include <sys/mac_provider.h> 41 #include <sys/mac_ether.h> 42 #include <sys/cpuvar.h> 43 #include <sys/ethernet.h> 44 45 #include <inet/ip.h> 46 #include <inet/ip6.h> 47 48 #include <sys/ddifm.h> 49 #include <sys/fm/protocol.h> 50 #include <sys/fm/util.h> 51 #include <sys/fm/io/ddi.h> 52 53 #include <mlxcx_reg.h> 54 55 #ifdef __cplusplus 56 extern "C" { 57 #endif 58 59 /* 60 * Get access to the first PCI BAR. 61 */ 62 #define MLXCX_REG_NUMBER 1 63 64 /* 65 * The command queue is supposed to be a page, which is 4k. 66 */ 67 #define MLXCX_CMD_DMA_PAGE_SIZE 4096 68 69 /* 70 * Queues can allocate in units of this much memory. 71 */ 72 #define MLXCX_QUEUE_DMA_PAGE_SIZE 4096 73 74 /* 75 * We advertise two sizes of groups to MAC -- a certain number of "large" 76 * groups (including the default group, which is sized to at least ncpus) 77 * followed by a certain number of "small" groups. 78 * 79 * This allows us to have a larger amount of classification resources available 80 * for zones/VMs without resorting to software classification. 81 */ 82 #define MLXCX_RX_NGROUPS_LARGE_DFLT 2 83 #define MLXCX_RX_NRINGS_PER_LARGE_GROUP_DFLT 16 84 #define MLXCX_RX_NGROUPS_SMALL_DFLT 256 85 #define MLXCX_RX_NRINGS_PER_SMALL_GROUP_DFLT 4 86 87 #define MLXCX_TX_NGROUPS_DFLT 1 88 #define MLXCX_TX_NRINGS_PER_GROUP_DFLT 64 89 90 /* 91 * Queues will be sized to (1 << *Q_SIZE_SHIFT) entries long. 92 */ 93 #define MLXCX_EQ_SIZE_SHIFT_DFLT 9 94 95 /* 96 * The CQ, SQ and RQ sizes can effect throughput on higher speed interfaces. 97 * EQ less so, as it only takes a single EQ entry to indicate there are 98 * multiple completions on the CQ. 99 * 100 * Particularly on the Rx side, the RQ (and corresponding CQ) would run 101 * low on available entries. A symptom of this is the refill taskq running 102 * frequently. A larger RQ (and CQ) alleviates this, and as there is a 103 * close relationship between SQ and CQ size, the SQ is increased too. 104 */ 105 #define MLXCX_CQ_SIZE_SHIFT_DFLT 10 106 #define MLXCX_CQ_SIZE_SHIFT_25G 12 107 108 /* 109 * Default to making SQs bigger than RQs for 9k MTU, since most packets will 110 * spill over into more than one slot. RQ WQEs are always 1 slot. 111 */ 112 #define MLXCX_SQ_SIZE_SHIFT_DFLT 11 113 #define MLXCX_SQ_SIZE_SHIFT_25G 13 114 115 #define MLXCX_RQ_SIZE_SHIFT_DFLT 10 116 #define MLXCX_RQ_SIZE_SHIFT_25G 12 117 118 #define MLXCX_CQ_HWM_GAP 16 119 #define MLXCX_CQ_LWM_GAP 24 120 121 #define MLXCX_WQ_HWM_GAP MLXCX_CQ_HWM_GAP 122 #define MLXCX_WQ_LWM_GAP MLXCX_CQ_LWM_GAP 123 124 #define MLXCX_RQ_REFILL_STEP 64 125 126 /* 127 * CQ event moderation 128 */ 129 #define MLXCX_CQEMOD_PERIOD_USEC_DFLT 50 130 #define MLXCX_CQEMOD_COUNT_DFLT \ 131 (8 * ((1 << MLXCX_CQ_SIZE_SHIFT_DFLT) / 10)) 132 133 /* 134 * EQ interrupt moderation 135 */ 136 #define MLXCX_INTRMOD_PERIOD_USEC_DFLT 10 137 138 /* Size of root flow tables */ 139 #define MLXCX_FTBL_ROOT_SIZE_SHIFT_DFLT 12 140 141 /* Size of 2nd level flow tables for VLAN filtering */ 142 #define MLXCX_FTBL_VLAN_SIZE_SHIFT_DFLT 4 143 144 /* 145 * How big does an mblk have to be before we dma_bind() it instead of 146 * bcopying? 147 */ 148 #define MLXCX_TX_BIND_THRESHOLD_DFLT 2048 149 150 /* 151 * How often to check the status of completion queues for overflow and 152 * other problems. 153 */ 154 #define MLXCX_WQ_CHECK_INTERVAL_SEC_DFLT 300 155 #define MLXCX_CQ_CHECK_INTERVAL_SEC_DFLT 300 156 #define MLXCX_EQ_CHECK_INTERVAL_SEC_DFLT 30 157 158 /* 159 * After this many packets, the packets received so far are passed to 160 * the mac layer. 161 */ 162 #define MLXCX_RX_PER_CQ_DEFAULT 256 163 #define MLXCX_RX_PER_CQ_MIN 16 164 #define MLXCX_RX_PER_CQ_MAX 4096 165 166 #define MLXCX_DOORBELL_TRIES_DFLT 3 167 extern uint_t mlxcx_doorbell_tries; 168 169 #define MLXCX_STUCK_INTR_COUNT_DFLT 128 170 extern uint_t mlxcx_stuck_intr_count; 171 172 #define MLXCX_BUF_BIND_MAX_ATTEMTPS 50 173 174 #define MLXCX_MTU_OFFSET \ 175 (sizeof (struct ether_vlan_header) + ETHERFCSL) 176 177 /* 178 * This is the current version of the command structure that the driver expects 179 * to be found in the ISS. 180 */ 181 #define MLXCX_CMD_REVISION 5 182 183 #ifdef DEBUG 184 #define MLXCX_DMA_SYNC(dma, flag) VERIFY0(ddi_dma_sync( \ 185 (dma).mxdb_dma_handle, 0, 0, \ 186 (flag))) 187 #else 188 #define MLXCX_DMA_SYNC(dma, flag) (void) ddi_dma_sync( \ 189 (dma).mxdb_dma_handle, 0, 0, \ 190 (flag)) 191 #endif 192 193 #define MLXCX_FM_SERVICE_MLXCX "mlxcx" 194 195 /* 196 * This macro defines the expected value of the 'Interface Step Sequence ID' 197 * (issi) which represents the version of the start up and tear down sequence. 198 * We must check that hardware supports this and tell it which version we're 199 * using as well. 200 */ 201 #define MLXCX_CURRENT_ISSI 1 202 203 /* 204 * This is the size of a page that the hardware expects from us when 205 * manipulating pages. 206 */ 207 #define MLXCX_HW_PAGE_SIZE 4096 208 209 /* 210 * This is a special lkey value used to terminate a list of scatter pointers. 211 */ 212 #define MLXCX_NULL_LKEY 0x100 213 214 /* 215 * The max function id we support in manage pages requests. 216 * At the moment we only support/expect func 0 from manage pages, but 217 * structures and code are in place to support any number. 218 */ 219 #define MLXCX_FUNC_ID_MAX 0 220 221 /* 222 * Forwards 223 */ 224 struct mlxcx; 225 typedef struct mlxcx mlxcx_t; 226 typedef struct mlxcx_cmd mlxcx_cmd_t; 227 typedef struct mlxcx_port mlxcx_port_t; 228 229 typedef struct { 230 mlxcx_t *mlp_mlx; 231 int32_t mlp_npages; 232 uint16_t mlp_func; 233 } mlxcx_pages_request_t; 234 235 typedef struct mlxcx_async_param { 236 mlxcx_t *mla_mlx; 237 taskq_ent_t mla_tqe; 238 boolean_t mla_pending; 239 kmutex_t mla_mtx; 240 241 /* 242 * Parameters specific to the function dispatched. 243 */ 244 union { 245 void *mla_arg; 246 mlxcx_pages_request_t mla_pages; 247 mlxcx_port_t *mla_port; 248 }; 249 } mlxcx_async_param_t; 250 251 typedef enum { 252 MLXCX_DMABUF_HDL_ALLOC = 1 << 0, 253 MLXCX_DMABUF_MEM_ALLOC = 1 << 1, 254 MLXCX_DMABUF_BOUND = 1 << 2, 255 MLXCX_DMABUF_FOREIGN = 1 << 3, 256 } mlxcx_dma_buffer_flags_t; 257 258 typedef struct mlxcx_dma_buffer { 259 mlxcx_dma_buffer_flags_t mxdb_flags; 260 caddr_t mxdb_va; /* Buffer VA */ 261 size_t mxdb_len; /* Buffer logical len */ 262 ddi_acc_handle_t mxdb_acc_handle; 263 ddi_dma_handle_t mxdb_dma_handle; 264 uint_t mxdb_ncookies; 265 } mlxcx_dma_buffer_t; 266 267 typedef struct mlxcx_dev_page { 268 list_node_t mxdp_list; 269 avl_node_t mxdp_tree; 270 uintptr_t mxdp_pa; 271 mlxcx_dma_buffer_t mxdp_dma; 272 } mlxcx_dev_page_t; 273 274 /* 275 * Data structure to keep track of all information related to the command queue. 276 */ 277 typedef enum { 278 MLXCX_CMD_QUEUE_S_IDLE = 1, 279 MLXCX_CMD_QUEUE_S_BUSY, 280 MLXCX_CMD_QUEUE_S_BROKEN 281 } mlxcx_cmd_queue_status_t; 282 283 typedef struct mlxcx_cmd_queue { 284 kmutex_t mcmd_lock; 285 kcondvar_t mcmd_cv; 286 mlxcx_dma_buffer_t mcmd_dma; 287 288 boolean_t mcmd_polled; 289 290 uint8_t mcmd_size_l2; 291 uint8_t mcmd_stride_l2; 292 uint_t mcmd_size; 293 /* 294 * The mask has a bit for each command slot, there are a maximum 295 * of 32 slots. When the bit is set in the mask, it indicates 296 * the slot is available. 297 */ 298 uint32_t mcmd_mask; 299 300 mlxcx_cmd_t *mcmd_active[MLXCX_CMD_MAX]; 301 302 ddi_taskq_t *mcmd_taskq; 303 id_space_t *mcmd_tokens; 304 } mlxcx_cmd_queue_t; 305 306 typedef struct mlxcd_cmd_mbox { 307 list_node_t mlbox_node; 308 mlxcx_dma_buffer_t mlbox_dma; 309 mlxcx_cmd_mailbox_t *mlbox_data; 310 } mlxcx_cmd_mbox_t; 311 312 typedef enum { 313 MLXCX_EQ_ALLOC = 1 << 0, /* dma mem alloc'd, size set */ 314 MLXCX_EQ_CREATED = 1 << 1, /* CREATE_EQ sent to hw */ 315 MLXCX_EQ_DESTROYED = 1 << 2, /* DESTROY_EQ sent to hw */ 316 MLXCX_EQ_ARMED = 1 << 3, /* Armed through the UAR */ 317 MLXCX_EQ_POLLING = 1 << 4, /* Currently being polled */ 318 } mlxcx_eventq_state_t; 319 320 typedef struct mlxcx_bf { 321 kmutex_t mbf_mtx; 322 uint_t mbf_cnt; 323 uint_t mbf_even; 324 uint_t mbf_odd; 325 } mlxcx_bf_t; 326 327 typedef struct mlxcx_uar { 328 boolean_t mlu_allocated; 329 uint_t mlu_num; 330 uint_t mlu_base; 331 332 volatile uint_t mlu_bfcnt; 333 mlxcx_bf_t mlu_bf[MLXCX_BF_PER_UAR]; 334 } mlxcx_uar_t; 335 336 typedef struct mlxcx_pd { 337 boolean_t mlpd_allocated; 338 uint32_t mlpd_num; 339 } mlxcx_pd_t; 340 341 typedef struct mlxcx_tdom { 342 boolean_t mltd_allocated; 343 uint32_t mltd_num; 344 } mlxcx_tdom_t; 345 346 typedef enum { 347 MLXCX_PORT_VPORT_PROMISC = 1 << 0, 348 } mlxcx_port_flags_t; 349 350 typedef struct mlxcx_flow_table mlxcx_flow_table_t; 351 typedef struct mlxcx_flow_group mlxcx_flow_group_t; 352 353 typedef struct { 354 uint64_t mlps_rx_drops; 355 } mlxcx_port_stats_t; 356 357 typedef enum { 358 MLXCX_PORT_INIT = 1 << 0 359 } mlxcx_port_init_t; 360 361 struct mlxcx_port { 362 kmutex_t mlp_mtx; 363 mlxcx_port_init_t mlp_init; 364 mlxcx_t *mlp_mlx; 365 /* 366 * The mlp_num we have here starts at zero (it's an index), but the 367 * numbering we have to use for register access starts at 1. We 368 * currently write mlp_num into the other_vport fields in mlxcx_cmd.c 369 * (where 0 is a magic number meaning "my vport") so if we ever add 370 * support for virtualisation features and deal with more than one 371 * vport, we will probably have to change this. 372 */ 373 uint_t mlp_num; 374 mlxcx_port_flags_t mlp_flags; 375 uint64_t mlp_guid; 376 uint8_t mlp_mac_address[ETHERADDRL]; 377 378 uint_t mlp_mtu; 379 uint_t mlp_max_mtu; 380 381 mlxcx_port_status_t mlp_admin_status; 382 mlxcx_port_status_t mlp_oper_status; 383 384 boolean_t mlp_autoneg; 385 mlxcx_eth_proto_t mlp_max_proto; 386 mlxcx_eth_proto_t mlp_admin_proto; 387 mlxcx_eth_proto_t mlp_oper_proto; 388 mlxcx_pplm_fec_active_t mlp_fec_active; 389 link_fec_t mlp_fec_requested; 390 391 mlxcx_eth_inline_mode_t mlp_wqe_min_inline; 392 393 /* Root flow tables */ 394 mlxcx_flow_table_t *mlp_rx_flow; 395 mlxcx_flow_table_t *mlp_tx_flow; 396 397 mlxcx_flow_group_t *mlp_promisc; 398 mlxcx_flow_group_t *mlp_bcast; 399 mlxcx_flow_group_t *mlp_umcast; 400 401 avl_tree_t mlp_dmac_fe; 402 403 mlxcx_port_stats_t mlp_stats; 404 405 mlxcx_module_status_t mlp_last_modstate; 406 mlxcx_module_error_type_t mlp_last_moderr; 407 408 mlxcx_async_param_t mlx_port_event; 409 }; 410 411 typedef enum { 412 MLXCX_EQ_TYPE_ANY, 413 MLXCX_EQ_TYPE_RX, 414 MLXCX_EQ_TYPE_TX 415 } mlxcx_eventq_type_t; 416 417 typedef struct mlxcx_event_queue { 418 kmutex_t mleq_mtx; 419 mlxcx_t *mleq_mlx; 420 mlxcx_eventq_state_t mleq_state; 421 mlxcx_eventq_type_t mleq_type; 422 423 mlxcx_dma_buffer_t mleq_dma; 424 425 size_t mleq_entshift; 426 size_t mleq_nents; 427 mlxcx_eventq_ent_t *mleq_ent; 428 uint32_t mleq_cc; /* consumer counter */ 429 uint32_t mleq_cc_armed; 430 431 uint32_t mleq_events; 432 433 uint32_t mleq_badintrs; 434 435 /* Hardware eq number */ 436 uint_t mleq_num; 437 /* Index into the mlxcx_t's interrupts array */ 438 uint_t mleq_intr_index; 439 440 /* UAR region that has this EQ's doorbell in it */ 441 mlxcx_uar_t *mleq_uar; 442 443 /* Tree of CQn => mlxcx_completion_queue_t */ 444 avl_tree_t mleq_cqs; 445 446 uint32_t mleq_check_disarm_cc; 447 uint_t mleq_check_disarm_cnt; 448 } mlxcx_event_queue_t; 449 450 typedef enum { 451 MLXCX_TIS_CREATED = 1 << 0, 452 MLXCX_TIS_DESTROYED = 1 << 1, 453 } mlxcx_tis_state_t; 454 455 typedef struct mlxcx_tis { 456 mlxcx_tis_state_t mltis_state; 457 list_node_t mltis_entry; 458 uint_t mltis_num; 459 mlxcx_tdom_t *mltis_tdom; 460 } mlxcx_tis_t; 461 462 typedef enum { 463 MLXCX_BUFFER_INIT, 464 MLXCX_BUFFER_FREE, 465 MLXCX_BUFFER_ON_WQ, 466 MLXCX_BUFFER_ON_LOAN, 467 MLXCX_BUFFER_ON_CHAIN, 468 } mlxcx_buffer_state_t; 469 470 typedef enum { 471 MLXCX_SHARD_READY, 472 MLXCX_SHARD_DRAINING, 473 } mlxcx_shard_state_t; 474 475 typedef struct mlxcx_buf_shard { 476 mlxcx_shard_state_t mlbs_state; 477 list_node_t mlbs_entry; 478 kmutex_t mlbs_mtx; 479 list_t mlbs_busy; 480 list_t mlbs_free; 481 list_t mlbs_loaned; 482 kcondvar_t mlbs_free_nonempty; 483 } mlxcx_buf_shard_t; 484 485 typedef struct mlxcx_buffer { 486 mlxcx_buf_shard_t *mlb_shard; 487 list_node_t mlb_entry; 488 list_node_t mlb_cq_entry; 489 490 struct mlxcx_buffer *mlb_tx_head; /* head of tx chain */ 491 list_t mlb_tx_chain; 492 list_node_t mlb_tx_chain_entry; 493 494 boolean_t mlb_foreign; 495 size_t mlb_used; 496 mblk_t *mlb_tx_mp; 497 498 /* 499 * The number of work queue basic blocks this buf uses. 500 */ 501 uint_t mlb_wqebbs; 502 503 mlxcx_t *mlb_mlx; 504 mlxcx_buffer_state_t mlb_state; 505 uint_t mlb_wqe_index; 506 mlxcx_dma_buffer_t mlb_dma; 507 mblk_t *mlb_mp; 508 frtn_t mlb_frtn; 509 } mlxcx_buffer_t; 510 511 typedef enum { 512 MLXCX_CQ_ALLOC = 1 << 0, 513 MLXCX_CQ_CREATED = 1 << 1, 514 MLXCX_CQ_DESTROYED = 1 << 2, 515 MLXCX_CQ_EQAVL = 1 << 3, 516 MLXCX_CQ_BLOCKED_MAC = 1 << 4, 517 MLXCX_CQ_TEARDOWN = 1 << 5, 518 MLXCX_CQ_POLLING = 1 << 6, 519 MLXCX_CQ_ARMED = 1 << 7, 520 } mlxcx_completionq_state_t; 521 522 typedef struct mlxcx_work_queue mlxcx_work_queue_t; 523 524 typedef struct mlxcx_completion_queue { 525 kmutex_t mlcq_mtx; 526 mlxcx_t *mlcq_mlx; 527 mlxcx_completionq_state_t mlcq_state; 528 529 mlxcx_port_stats_t *mlcq_stats; 530 531 list_node_t mlcq_entry; 532 avl_node_t mlcq_eq_entry; 533 534 uint_t mlcq_num; 535 536 mlxcx_work_queue_t *mlcq_wq; 537 mlxcx_event_queue_t *mlcq_eq; 538 539 /* UAR region that has this CQ's UAR doorbell in it */ 540 mlxcx_uar_t *mlcq_uar; 541 542 mlxcx_dma_buffer_t mlcq_dma; 543 544 size_t mlcq_entshift; 545 size_t mlcq_nents; 546 mlxcx_completionq_ent_t *mlcq_ent; 547 uint32_t mlcq_cc; /* consumer counter */ 548 uint32_t mlcq_cc_armed; /* cc at last arm */ 549 uint32_t mlcq_ec; /* event counter */ 550 uint32_t mlcq_ec_armed; /* ec at last arm */ 551 552 mlxcx_dma_buffer_t mlcq_doorbell_dma; 553 mlxcx_completionq_doorbell_t *mlcq_doorbell; 554 555 uint64_t mlcq_bufcnt; 556 size_t mlcq_bufhwm; 557 size_t mlcq_buflwm; 558 list_t mlcq_buffers; 559 kmutex_t mlcq_bufbmtx; 560 list_t mlcq_buffers_b; 561 562 uint_t mlcq_check_disarm_cnt; 563 uint64_t mlcq_check_disarm_cc; 564 565 uint_t mlcq_cqemod_period_usec; 566 uint_t mlcq_cqemod_count; 567 568 mac_ring_handle_t mlcq_mac_hdl; 569 uint64_t mlcq_mac_gen; 570 571 boolean_t mlcq_fm_repd_qstate; 572 } mlxcx_completion_queue_t; 573 574 typedef enum { 575 MLXCX_WQ_ALLOC = 1 << 0, 576 MLXCX_WQ_CREATED = 1 << 1, 577 MLXCX_WQ_STARTED = 1 << 2, 578 MLXCX_WQ_DESTROYED = 1 << 3, 579 MLXCX_WQ_TEARDOWN = 1 << 4, 580 MLXCX_WQ_BUFFERS = 1 << 5, 581 MLXCX_WQ_REFILLING = 1 << 6, 582 MLXCX_WQ_BLOCKED_MAC = 1 << 7 583 } mlxcx_workq_state_t; 584 585 typedef enum { 586 MLXCX_WQ_TYPE_SENDQ = 1, 587 MLXCX_WQ_TYPE_RECVQ 588 } mlxcx_workq_type_t; 589 590 typedef struct mlxcx_ring_group mlxcx_ring_group_t; 591 592 struct mlxcx_work_queue { 593 kmutex_t mlwq_mtx; 594 mlxcx_t *mlwq_mlx; 595 mlxcx_workq_type_t mlwq_type; 596 mlxcx_workq_state_t mlwq_state; 597 598 list_node_t mlwq_entry; 599 list_node_t mlwq_group_entry; 600 601 mlxcx_ring_group_t *mlwq_group; 602 603 uint_t mlwq_num; 604 605 mlxcx_completion_queue_t *mlwq_cq; 606 mlxcx_pd_t *mlwq_pd; 607 608 /* Required for send queues */ 609 mlxcx_tis_t *mlwq_tis; 610 611 /* UAR region that has this WQ's blueflame buffers in it */ 612 mlxcx_uar_t *mlwq_uar; 613 614 mlxcx_dma_buffer_t mlwq_dma; 615 616 mlxcx_eth_inline_mode_t mlwq_inline_mode; 617 size_t mlwq_entshift; 618 size_t mlwq_nents; 619 /* Discriminate based on mwq_type */ 620 union { 621 mlxcx_sendq_ent_t *mlwq_send_ent; 622 mlxcx_sendq_extra_ent_t *mlwq_send_extra_ent; 623 mlxcx_recvq_ent_t *mlwq_recv_ent; 624 mlxcx_sendq_bf_t *mlwq_bf_ent; 625 }; 626 uint64_t mlwq_pc; /* producer counter */ 627 628 uint64_t mlwq_wqebb_used; 629 size_t mlwq_bufhwm; 630 size_t mlwq_buflwm; 631 632 mlxcx_dma_buffer_t mlwq_doorbell_dma; 633 mlxcx_workq_doorbell_t *mlwq_doorbell; 634 635 mlxcx_buf_shard_t *mlwq_bufs; 636 mlxcx_buf_shard_t *mlwq_foreign_bufs; 637 638 taskq_ent_t mlwq_tqe; 639 640 boolean_t mlwq_fm_repd_qstate; 641 }; 642 643 #define MLXCX_RQT_MAX_SIZE 64 644 645 typedef enum { 646 MLXCX_RQT_CREATED = 1 << 0, 647 MLXCX_RQT_DESTROYED = 1 << 1, 648 MLXCX_RQT_DIRTY = 1 << 2, 649 } mlxcx_rqtable_state_t; 650 651 typedef struct mlxcx_rqtable { 652 mlxcx_rqtable_state_t mlrqt_state; 653 list_node_t mlrqt_entry; 654 uint_t mlrqt_num; 655 656 size_t mlrqt_max; 657 size_t mlrqt_used; 658 659 size_t mlrqt_rq_size; 660 mlxcx_work_queue_t **mlrqt_rq; 661 } mlxcx_rqtable_t; 662 663 typedef enum { 664 MLXCX_TIR_CREATED = 1 << 0, 665 MLXCX_TIR_DESTROYED = 1 << 1, 666 } mlxcx_tir_state_t; 667 668 typedef struct mlxcx_tir { 669 mlxcx_tir_state_t mltir_state; 670 list_node_t mltir_entry; 671 uint_t mltir_num; 672 mlxcx_tdom_t *mltir_tdom; 673 mlxcx_tir_type_t mltir_type; 674 union { 675 mlxcx_rqtable_t *mltir_rqtable; 676 mlxcx_work_queue_t *mltir_rq; 677 }; 678 mlxcx_tir_hash_fn_t mltir_hash_fn; 679 uint8_t mltir_toeplitz_key[40]; 680 mlxcx_tir_rx_hash_l3_type_t mltir_l3_type; 681 mlxcx_tir_rx_hash_l4_type_t mltir_l4_type; 682 mlxcx_tir_rx_hash_fields_t mltir_hash_fields; 683 } mlxcx_tir_t; 684 685 typedef enum { 686 MLXCX_FLOW_GROUP_CREATED = 1 << 0, 687 MLXCX_FLOW_GROUP_BUSY = 1 << 1, 688 MLXCX_FLOW_GROUP_DESTROYED = 1 << 2, 689 } mlxcx_flow_group_state_t; 690 691 typedef enum { 692 MLXCX_FLOW_MATCH_SMAC = 1 << 0, 693 MLXCX_FLOW_MATCH_DMAC = 1 << 1, 694 MLXCX_FLOW_MATCH_VLAN = 1 << 2, 695 MLXCX_FLOW_MATCH_VID = 1 << 3, 696 MLXCX_FLOW_MATCH_IP_VER = 1 << 4, 697 MLXCX_FLOW_MATCH_SRCIP = 1 << 5, 698 MLXCX_FLOW_MATCH_DSTIP = 1 << 6, 699 MLXCX_FLOW_MATCH_IP_PROTO = 1 << 7, 700 MLXCX_FLOW_MATCH_SQN = 1 << 8, 701 MLXCX_FLOW_MATCH_VXLAN = 1 << 9, 702 } mlxcx_flow_mask_t; 703 704 struct mlxcx_flow_group { 705 list_node_t mlfg_entry; 706 list_node_t mlfg_role_entry; 707 mlxcx_flow_group_state_t mlfg_state; 708 mlxcx_flow_table_t *mlfg_table; 709 uint_t mlfg_num; 710 size_t mlfg_start_idx; 711 size_t mlfg_size; 712 size_t mlfg_avail; 713 list_t mlfg_entries; 714 mlxcx_flow_mask_t mlfg_mask; 715 }; 716 717 typedef enum { 718 MLXCX_FLOW_ENTRY_RESERVED = 1 << 0, 719 MLXCX_FLOW_ENTRY_CREATED = 1 << 1, 720 MLXCX_FLOW_ENTRY_DELETED = 1 << 2, 721 MLXCX_FLOW_ENTRY_DIRTY = 1 << 3, 722 } mlxcx_flow_entry_state_t; 723 724 typedef struct { 725 mlxcx_tir_t *mlfed_tir; 726 mlxcx_flow_table_t *mlfed_flow; 727 } mlxcx_flow_entry_dest_t; 728 729 typedef struct mlxcx_flow_entry { 730 list_node_t mlfe_group_entry; 731 avl_node_t mlfe_dmac_entry; 732 mlxcx_flow_entry_state_t mlfe_state; 733 mlxcx_flow_table_t *mlfe_table; 734 mlxcx_flow_group_t *mlfe_group; 735 uint_t mlfe_index; 736 737 mlxcx_flow_action_t mlfe_action; 738 739 /* Criteria for match */ 740 uint8_t mlfe_smac[ETHERADDRL]; 741 uint8_t mlfe_dmac[ETHERADDRL]; 742 743 mlxcx_vlan_type_t mlfe_vlan_type; 744 uint16_t mlfe_vid; 745 746 uint_t mlfe_ip_version; 747 uint8_t mlfe_srcip[IPV6_ADDR_LEN]; 748 uint8_t mlfe_dstip[IPV6_ADDR_LEN]; 749 750 uint_t mlfe_ip_proto; 751 uint16_t mlfe_sport; 752 uint16_t mlfe_dport; 753 754 uint32_t mlfe_sqn; 755 uint32_t mlfe_vxlan_vni; 756 757 /* Destinations */ 758 size_t mlfe_ndest; 759 mlxcx_flow_entry_dest_t mlfe_dest[MLXCX_FLOW_MAX_DESTINATIONS]; 760 761 /* 762 * mlxcx_group_mac_ts joining this entry to N ring groups 763 * only used by FEs on the root rx flow table 764 */ 765 list_t mlfe_ring_groups; 766 } mlxcx_flow_entry_t; 767 768 typedef enum { 769 MLXCX_FLOW_TABLE_CREATED = 1 << 0, 770 MLXCX_FLOW_TABLE_DESTROYED = 1 << 1, 771 MLXCX_FLOW_TABLE_ROOT = 1 << 2 772 } mlxcx_flow_table_state_t; 773 774 struct mlxcx_flow_table { 775 kmutex_t mlft_mtx; 776 mlxcx_flow_table_state_t mlft_state; 777 uint_t mlft_level; 778 uint_t mlft_num; 779 mlxcx_flow_table_type_t mlft_type; 780 781 mlxcx_port_t *mlft_port; 782 783 size_t mlft_entshift; 784 size_t mlft_nents; 785 786 size_t mlft_entsize; 787 mlxcx_flow_entry_t *mlft_ent; 788 789 /* First entry not yet claimed by a group */ 790 size_t mlft_next_ent; 791 792 list_t mlft_groups; 793 }; 794 795 typedef enum { 796 MLXCX_GROUP_RX, 797 MLXCX_GROUP_TX 798 } mlxcx_group_type_t; 799 800 typedef enum { 801 MLXCX_GROUP_INIT = 1 << 0, 802 MLXCX_GROUP_WQS = 1 << 1, 803 MLXCX_GROUP_TIRTIS = 1 << 2, 804 MLXCX_GROUP_FLOWS = 1 << 3, 805 MLXCX_GROUP_RUNNING = 1 << 4, 806 MLXCX_GROUP_RQT = 1 << 5, 807 } mlxcx_group_state_t; 808 809 #define MLXCX_RX_HASH_FT_SIZE_SHIFT 4 810 811 typedef enum { 812 MLXCX_TIR_ROLE_IPv4 = 0, 813 MLXCX_TIR_ROLE_IPv6, 814 MLXCX_TIR_ROLE_TCPv4, 815 MLXCX_TIR_ROLE_TCPv6, 816 MLXCX_TIR_ROLE_UDPv4, 817 MLXCX_TIR_ROLE_UDPv6, 818 MLXCX_TIR_ROLE_OTHER, 819 820 MLXCX_TIRS_PER_GROUP 821 } mlxcx_tir_role_t; 822 823 typedef struct { 824 avl_node_t mlgm_group_entry; 825 list_node_t mlgm_fe_entry; 826 mlxcx_ring_group_t *mlgm_group; 827 uint8_t mlgm_mac[6]; 828 mlxcx_flow_entry_t *mlgm_fe; 829 } mlxcx_group_mac_t; 830 831 typedef struct { 832 list_node_t mlgv_entry; 833 boolean_t mlgv_tagged; 834 uint16_t mlgv_vid; 835 mlxcx_flow_entry_t *mlgv_fe; 836 } mlxcx_group_vlan_t; 837 838 struct mlxcx_ring_group { 839 kmutex_t mlg_mtx; 840 mlxcx_t *mlg_mlx; 841 mlxcx_group_state_t mlg_state; 842 mlxcx_group_type_t mlg_type; 843 844 mac_group_handle_t mlg_mac_hdl; 845 846 union { 847 mlxcx_tis_t mlg_tis; 848 mlxcx_tir_t mlg_tir[MLXCX_TIRS_PER_GROUP]; 849 }; 850 mlxcx_port_t *mlg_port; 851 852 size_t mlg_nwqs; 853 size_t mlg_wqs_size; 854 mlxcx_work_queue_t *mlg_wqs; 855 856 mlxcx_rqtable_t *mlg_rqt; 857 858 /* 859 * Flow table for matching VLAN IDs 860 */ 861 mlxcx_flow_table_t *mlg_rx_vlan_ft; 862 mlxcx_flow_group_t *mlg_rx_vlan_fg; 863 mlxcx_flow_group_t *mlg_rx_vlan_def_fg; 864 mlxcx_flow_group_t *mlg_rx_vlan_promisc_fg; 865 list_t mlg_rx_vlans; 866 867 taskq_t *mlg_refill_tq; 868 869 /* 870 * Flow table for separating out by protocol before hashing 871 */ 872 mlxcx_flow_table_t *mlg_rx_hash_ft; 873 874 /* 875 * Links to flow entries on the root flow table which are pointing to 876 * our rx_vlan_ft. 877 */ 878 avl_tree_t mlg_rx_macs; 879 }; 880 881 typedef enum mlxcx_cmd_state { 882 MLXCX_CMD_S_DONE = 1 << 0, 883 MLXCX_CMD_S_ERROR = 1 << 1 884 } mlxcx_cmd_state_t; 885 886 struct mlxcx_cmd { 887 struct mlxcx *mlcmd_mlxp; 888 kmutex_t mlcmd_lock; 889 kcondvar_t mlcmd_cv; 890 891 boolean_t mlcmd_poll; 892 uint8_t mlcmd_token; 893 mlxcx_cmd_op_t mlcmd_op; 894 895 /* 896 * Command data and extended mailboxes for responses. 897 */ 898 const void *mlcmd_in; 899 uint32_t mlcmd_inlen; 900 void *mlcmd_out; 901 uint32_t mlcmd_outlen; 902 list_t mlcmd_mbox_in; 903 uint8_t mlcmd_nboxes_in; 904 list_t mlcmd_mbox_out; 905 uint8_t mlcmd_nboxes_out; 906 /* 907 * Status information. 908 */ 909 mlxcx_cmd_state_t mlcmd_state; 910 uint8_t mlcmd_status; 911 }; 912 913 /* 914 * Our view of capabilities. 915 */ 916 typedef struct mlxcx_hca_cap { 917 mlxcx_hca_cap_mode_t mhc_mode; 918 mlxcx_hca_cap_type_t mhc_type; 919 union { 920 uint8_t mhc_bulk[MLXCX_HCA_CAP_SIZE]; 921 mlxcx_hca_cap_general_caps_t mhc_general; 922 mlxcx_hca_cap_eth_caps_t mhc_eth; 923 mlxcx_hca_cap_flow_caps_t mhc_flow; 924 }; 925 } mlxcx_hca_cap_t; 926 927 typedef struct { 928 /* Cooked values */ 929 boolean_t mlc_checksum; 930 boolean_t mlc_lso; 931 boolean_t mlc_vxlan; 932 size_t mlc_max_lso_size; 933 size_t mlc_max_rqt_size; 934 935 size_t mlc_max_rx_ft_shift; 936 size_t mlc_max_rx_fe_dest; 937 size_t mlc_max_rx_flows; 938 size_t mlc_max_rx_ft; 939 940 size_t mlc_max_tir; 941 942 /* Raw caps data */ 943 mlxcx_hca_cap_t mlc_hca_cur; 944 mlxcx_hca_cap_t mlc_hca_max; 945 mlxcx_hca_cap_t mlc_ether_cur; 946 mlxcx_hca_cap_t mlc_ether_max; 947 mlxcx_hca_cap_t mlc_nic_flow_cur; 948 mlxcx_hca_cap_t mlc_nic_flow_max; 949 } mlxcx_caps_t; 950 951 typedef struct { 952 uint_t mldp_eq_size_shift; 953 uint_t mldp_cq_size_shift; 954 uint_t mldp_cq_size_shift_default; 955 uint_t mldp_rq_size_shift; 956 uint_t mldp_rq_size_shift_default; 957 uint_t mldp_sq_size_shift; 958 uint_t mldp_sq_size_shift_default; 959 uint_t mldp_cqemod_period_usec; 960 uint_t mldp_cqemod_count; 961 uint_t mldp_intrmod_period_usec; 962 uint_t mldp_rx_ngroups_large; 963 uint_t mldp_rx_ngroups_small; 964 uint_t mldp_rx_nrings_per_large_group; 965 uint_t mldp_rx_nrings_per_small_group; 966 uint_t mldp_rx_per_cq; 967 uint_t mldp_tx_ngroups; 968 uint_t mldp_tx_nrings_per_group; 969 uint_t mldp_ftbl_root_size_shift; 970 size_t mldp_tx_bind_threshold; 971 uint_t mldp_ftbl_vlan_size_shift; 972 uint64_t mldp_eq_check_interval_sec; 973 uint64_t mldp_cq_check_interval_sec; 974 uint64_t mldp_wq_check_interval_sec; 975 } mlxcx_drv_props_t; 976 977 typedef enum { 978 MLXCX_ATTACH_FM = 1 << 0, 979 MLXCX_ATTACH_PCI_CONFIG = 1 << 1, 980 MLXCX_ATTACH_REGS = 1 << 2, 981 MLXCX_ATTACH_CMD = 1 << 3, 982 MLXCX_ATTACH_ENABLE_HCA = 1 << 4, 983 MLXCX_ATTACH_PAGE_LIST = 1 << 5, 984 MLXCX_ATTACH_INIT_HCA = 1 << 6, 985 MLXCX_ATTACH_UAR_PD_TD = 1 << 7, 986 MLXCX_ATTACH_INTRS = 1 << 8, 987 MLXCX_ATTACH_PORTS = 1 << 9, 988 MLXCX_ATTACH_MAC_HDL = 1 << 10, 989 MLXCX_ATTACH_CQS = 1 << 11, 990 MLXCX_ATTACH_WQS = 1 << 12, 991 MLXCX_ATTACH_GROUPS = 1 << 13, 992 MLXCX_ATTACH_BUFS = 1 << 14, 993 MLXCX_ATTACH_CAPS = 1 << 15, 994 MLXCX_ATTACH_CHKTIMERS = 1 << 16, 995 MLXCX_ATTACH_ASYNC_TQ = 1 << 17, 996 } mlxcx_attach_progress_t; 997 998 struct mlxcx { 999 /* entry on the mlxcx_glist */ 1000 list_node_t mlx_gentry; 1001 1002 dev_info_t *mlx_dip; 1003 int mlx_inst; 1004 mlxcx_attach_progress_t mlx_attach; 1005 1006 mlxcx_drv_props_t mlx_props; 1007 1008 /* 1009 * Misc. data 1010 */ 1011 uint16_t mlx_fw_maj; 1012 uint16_t mlx_fw_min; 1013 uint16_t mlx_fw_rev; 1014 uint16_t mlx_cmd_rev; 1015 1016 /* 1017 * Various capabilities of hardware. 1018 */ 1019 mlxcx_caps_t *mlx_caps; 1020 1021 uint_t mlx_max_sdu; 1022 uint_t mlx_sdu; 1023 1024 /* 1025 * FM State 1026 */ 1027 int mlx_fm_caps; 1028 1029 /* 1030 * PCI Data 1031 */ 1032 ddi_acc_handle_t mlx_cfg_handle; 1033 ddi_acc_handle_t mlx_regs_handle; 1034 caddr_t mlx_regs_base; 1035 1036 /* 1037 * MAC handle 1038 */ 1039 mac_handle_t mlx_mac_hdl; 1040 1041 /* 1042 * Main command queue for issuing general FW control commands. 1043 */ 1044 mlxcx_cmd_queue_t mlx_cmd; 1045 1046 /* 1047 * Interrupts 1048 */ 1049 uint_t mlx_intr_pri; 1050 uint_t mlx_intr_type; /* always MSI-X */ 1051 int mlx_intr_count; 1052 size_t mlx_intr_size; /* allocation size */ 1053 int mlx_intr_cq0; 1054 ddi_intr_handle_t *mlx_intr_handles; 1055 1056 /* 1057 * Basic firmware resources which we use for a variety of things. 1058 * The UAR is a reference to a page where CQ and EQ doorbells are 1059 * located. It also holds all the BlueFlame stuff (which we don't 1060 * use). 1061 */ 1062 mlxcx_uar_t mlx_uar; 1063 /* 1064 * The PD (Protection Domain) and TDOM (Transport Domain) are opaque 1065 * entities to us (they're Infiniband constructs we don't actually care 1066 * about) -- we just allocate them and shove their ID numbers in 1067 * whenever we're asked for one. 1068 * 1069 * The "reserved" LKEY is what we should put in queue entries that 1070 * have references to memory to indicate that they're using linear 1071 * addresses (comes from the QUERY_SPECIAL_CONTEXTS cmd). 1072 */ 1073 mlxcx_pd_t mlx_pd; 1074 mlxcx_tdom_t mlx_tdom; 1075 uint_t mlx_rsvd_lkey; 1076 1077 /* 1078 * Our event queues. These are 1:1 with interrupts. 1079 */ 1080 size_t mlx_eqs_size; /* allocation size */ 1081 mlxcx_event_queue_t *mlx_eqs; 1082 1083 /* 1084 * Page list. These represent the set of 4k pages we've given to 1085 * hardware. 1086 * 1087 * We can add to this list at the request of hardware from interrupt 1088 * context (the PAGE_REQUEST event), so it's protected by pagemtx. 1089 */ 1090 kmutex_t mlx_pagemtx; 1091 uint_t mlx_npages; 1092 avl_tree_t mlx_pages; 1093 1094 mlxcx_async_param_t mlx_npages_req[MLXCX_FUNC_ID_MAX + 1]; 1095 1096 /* 1097 * Taskq for processing asynchronous events which may issue 1098 * commands to the HCA. 1099 */ 1100 taskq_t *mlx_async_tq; 1101 1102 /* 1103 * Port state 1104 */ 1105 uint_t mlx_nports; 1106 size_t mlx_ports_size; 1107 mlxcx_port_t *mlx_ports; 1108 1109 /* 1110 * Completion queues (CQs). These are also indexed off the 1111 * event_queue_ts that they each report to. 1112 */ 1113 list_t mlx_cqs; 1114 1115 uint_t mlx_next_eq; 1116 1117 /* 1118 * Work queues (WQs). 1119 */ 1120 list_t mlx_wqs; 1121 1122 /* 1123 * Ring groups 1124 */ 1125 size_t mlx_rx_ngroups; 1126 size_t mlx_rx_groups_size; 1127 mlxcx_ring_group_t *mlx_rx_groups; 1128 1129 size_t mlx_tx_ngroups; 1130 size_t mlx_tx_groups_size; 1131 mlxcx_ring_group_t *mlx_tx_groups; 1132 1133 kmem_cache_t *mlx_bufs_cache; 1134 list_t mlx_buf_shards; 1135 1136 ddi_periodic_t mlx_eq_checktimer; 1137 ddi_periodic_t mlx_cq_checktimer; 1138 ddi_periodic_t mlx_wq_checktimer; 1139 }; 1140 1141 /* 1142 * Register access 1143 */ 1144 extern uint16_t mlxcx_get16(mlxcx_t *, uintptr_t); 1145 extern uint32_t mlxcx_get32(mlxcx_t *, uintptr_t); 1146 extern uint64_t mlxcx_get64(mlxcx_t *, uintptr_t); 1147 1148 extern void mlxcx_put32(mlxcx_t *, uintptr_t, uint32_t); 1149 extern void mlxcx_put64(mlxcx_t *, uintptr_t, uint64_t); 1150 1151 extern void mlxcx_uar_put32(mlxcx_t *, mlxcx_uar_t *, uintptr_t, uint32_t); 1152 extern void mlxcx_uar_put64(mlxcx_t *, mlxcx_uar_t *, uintptr_t, uint64_t); 1153 1154 /* 1155 * Logging functions. 1156 */ 1157 extern void mlxcx_warn(mlxcx_t *, const char *, ...); 1158 extern void mlxcx_note(mlxcx_t *, const char *, ...); 1159 extern void mlxcx_panic(mlxcx_t *, const char *, ...); 1160 1161 extern void mlxcx_fm_ereport(mlxcx_t *, const char *); 1162 1163 extern void mlxcx_check_sq(mlxcx_t *, mlxcx_work_queue_t *); 1164 extern void mlxcx_check_rq(mlxcx_t *, mlxcx_work_queue_t *); 1165 1166 /* 1167 * DMA Functions 1168 */ 1169 extern void mlxcx_dma_free(mlxcx_dma_buffer_t *); 1170 extern boolean_t mlxcx_dma_alloc(mlxcx_t *, mlxcx_dma_buffer_t *, 1171 ddi_dma_attr_t *, ddi_device_acc_attr_t *, boolean_t, size_t, boolean_t); 1172 extern boolean_t mlxcx_dma_init(mlxcx_t *, mlxcx_dma_buffer_t *, 1173 ddi_dma_attr_t *, boolean_t); 1174 extern boolean_t mlxcx_dma_bind_mblk(mlxcx_t *, mlxcx_dma_buffer_t *, 1175 const mblk_t *, size_t, boolean_t); 1176 extern boolean_t mlxcx_dma_alloc_offset(mlxcx_t *, mlxcx_dma_buffer_t *, 1177 ddi_dma_attr_t *, ddi_device_acc_attr_t *, boolean_t, 1178 size_t, size_t, boolean_t); 1179 extern void mlxcx_dma_unbind(mlxcx_t *, mlxcx_dma_buffer_t *); 1180 extern void mlxcx_dma_acc_attr(mlxcx_t *, ddi_device_acc_attr_t *); 1181 extern void mlxcx_dma_page_attr(mlxcx_t *, ddi_dma_attr_t *); 1182 extern void mlxcx_dma_queue_attr(mlxcx_t *, ddi_dma_attr_t *); 1183 extern void mlxcx_dma_qdbell_attr(mlxcx_t *, ddi_dma_attr_t *); 1184 extern void mlxcx_dma_buf_attr(mlxcx_t *, ddi_dma_attr_t *); 1185 1186 extern boolean_t mlxcx_give_pages(mlxcx_t *, int32_t, int32_t *); 1187 1188 static inline const ddi_dma_cookie_t * 1189 mlxcx_dma_cookie_iter(const mlxcx_dma_buffer_t *db, 1190 const ddi_dma_cookie_t *prev) 1191 { 1192 ASSERT(db->mxdb_flags & MLXCX_DMABUF_BOUND); 1193 return (ddi_dma_cookie_iter(db->mxdb_dma_handle, prev)); 1194 } 1195 1196 static inline const ddi_dma_cookie_t * 1197 mlxcx_dma_cookie_one(const mlxcx_dma_buffer_t *db) 1198 { 1199 ASSERT(db->mxdb_flags & MLXCX_DMABUF_BOUND); 1200 return (ddi_dma_cookie_one(db->mxdb_dma_handle)); 1201 } 1202 1203 /* 1204 * From mlxcx_intr.c 1205 */ 1206 extern boolean_t mlxcx_intr_setup(mlxcx_t *); 1207 extern void mlxcx_intr_teardown(mlxcx_t *); 1208 extern void mlxcx_arm_eq(mlxcx_t *, mlxcx_event_queue_t *); 1209 extern void mlxcx_arm_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1210 extern void mlxcx_update_cqci(mlxcx_t *, mlxcx_completion_queue_t *); 1211 1212 extern mblk_t *mlxcx_rx_poll(mlxcx_t *, mlxcx_completion_queue_t *, size_t); 1213 1214 /* 1215 * From mlxcx_gld.c 1216 */ 1217 extern boolean_t mlxcx_register_mac(mlxcx_t *); 1218 1219 /* 1220 * From mlxcx_ring.c 1221 */ 1222 extern boolean_t mlxcx_wq_alloc_dma(mlxcx_t *, mlxcx_work_queue_t *); 1223 extern void mlxcx_wq_rele_dma(mlxcx_t *, mlxcx_work_queue_t *); 1224 1225 extern boolean_t mlxcx_buf_create(mlxcx_t *, mlxcx_buf_shard_t *, 1226 mlxcx_buffer_t **); 1227 extern boolean_t mlxcx_buf_create_foreign(mlxcx_t *, mlxcx_buf_shard_t *, 1228 mlxcx_buffer_t **); 1229 extern mlxcx_buffer_t *mlxcx_buf_take(mlxcx_t *, mlxcx_work_queue_t *); 1230 extern size_t mlxcx_buf_take_n(mlxcx_t *, mlxcx_work_queue_t *, 1231 mlxcx_buffer_t **, size_t); 1232 extern boolean_t mlxcx_buf_loan(mlxcx_t *, mlxcx_buffer_t *); 1233 extern void mlxcx_buf_return(mlxcx_t *, mlxcx_buffer_t *); 1234 extern void mlxcx_buf_return_chain(mlxcx_t *, mlxcx_buffer_t *, boolean_t); 1235 extern void mlxcx_buf_destroy(mlxcx_t *, mlxcx_buffer_t *); 1236 extern void mlxcx_shard_ready(mlxcx_buf_shard_t *); 1237 extern void mlxcx_shard_draining(mlxcx_buf_shard_t *); 1238 1239 extern uint_t mlxcx_buf_bind_or_copy(mlxcx_t *, mlxcx_work_queue_t *, 1240 mblk_t *, size_t, mlxcx_buffer_t **); 1241 1242 extern boolean_t mlxcx_rx_group_setup(mlxcx_t *, mlxcx_ring_group_t *); 1243 extern boolean_t mlxcx_tx_group_setup(mlxcx_t *, mlxcx_ring_group_t *); 1244 1245 extern boolean_t mlxcx_rx_group_start(mlxcx_t *, mlxcx_ring_group_t *); 1246 extern boolean_t mlxcx_tx_ring_start(mlxcx_t *, mlxcx_ring_group_t *, 1247 mlxcx_work_queue_t *); 1248 extern boolean_t mlxcx_rx_ring_start(mlxcx_t *, mlxcx_ring_group_t *, 1249 mlxcx_work_queue_t *); 1250 1251 extern boolean_t mlxcx_rq_add_buffer(mlxcx_t *, mlxcx_work_queue_t *, 1252 mlxcx_buffer_t *); 1253 extern boolean_t mlxcx_rq_add_buffers(mlxcx_t *, mlxcx_work_queue_t *, 1254 mlxcx_buffer_t **, size_t); 1255 extern boolean_t mlxcx_sq_add_buffer(mlxcx_t *, mlxcx_work_queue_t *, 1256 uint8_t *, size_t, uint32_t, mlxcx_buffer_t *); 1257 extern boolean_t mlxcx_sq_add_nop(mlxcx_t *, mlxcx_work_queue_t *); 1258 extern void mlxcx_rq_refill(mlxcx_t *, mlxcx_work_queue_t *); 1259 1260 extern void mlxcx_teardown_groups(mlxcx_t *); 1261 extern void mlxcx_wq_teardown(mlxcx_t *, mlxcx_work_queue_t *); 1262 extern void mlxcx_cq_teardown(mlxcx_t *, mlxcx_completion_queue_t *); 1263 extern void mlxcx_teardown_rx_group(mlxcx_t *, mlxcx_ring_group_t *); 1264 extern void mlxcx_teardown_tx_group(mlxcx_t *, mlxcx_ring_group_t *); 1265 1266 extern void mlxcx_tx_completion(mlxcx_t *, mlxcx_completion_queue_t *, 1267 mlxcx_completionq_ent_t *, mlxcx_buffer_t *); 1268 extern mblk_t *mlxcx_rx_completion(mlxcx_t *, mlxcx_completion_queue_t *, 1269 mlxcx_completionq_ent_t *, mlxcx_buffer_t *); 1270 1271 extern mlxcx_buf_shard_t *mlxcx_mlbs_create(mlxcx_t *); 1272 1273 /* 1274 * Flow mgmt 1275 */ 1276 extern boolean_t mlxcx_add_umcast_entry(mlxcx_t *, mlxcx_port_t *, 1277 mlxcx_ring_group_t *, const uint8_t *); 1278 extern boolean_t mlxcx_remove_umcast_entry(mlxcx_t *, mlxcx_port_t *, 1279 mlxcx_ring_group_t *, const uint8_t *); 1280 extern void mlxcx_remove_all_umcast_entries(mlxcx_t *, mlxcx_port_t *, 1281 mlxcx_ring_group_t *); 1282 extern boolean_t mlxcx_setup_flow_group(mlxcx_t *, mlxcx_flow_table_t *, 1283 mlxcx_flow_group_t *); 1284 extern void mlxcx_teardown_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1285 1286 extern void mlxcx_remove_all_vlan_entries(mlxcx_t *, mlxcx_ring_group_t *); 1287 extern boolean_t mlxcx_remove_vlan_entry(mlxcx_t *, mlxcx_ring_group_t *, 1288 boolean_t, uint16_t); 1289 extern boolean_t mlxcx_add_vlan_entry(mlxcx_t *, mlxcx_ring_group_t *, 1290 boolean_t, uint16_t); 1291 1292 /* 1293 * Command functions 1294 */ 1295 extern boolean_t mlxcx_cmd_queue_init(mlxcx_t *); 1296 extern void mlxcx_cmd_queue_fini(mlxcx_t *); 1297 1298 extern void mlxcx_cmd_completion(mlxcx_t *, mlxcx_eventq_ent_t *); 1299 extern void mlxcx_cmd_eq_enable(mlxcx_t *); 1300 extern void mlxcx_cmd_eq_disable(mlxcx_t *); 1301 1302 extern boolean_t mlxcx_cmd_enable_hca(mlxcx_t *); 1303 extern boolean_t mlxcx_cmd_disable_hca(mlxcx_t *); 1304 1305 extern boolean_t mlxcx_cmd_query_issi(mlxcx_t *, uint_t *); 1306 extern boolean_t mlxcx_cmd_set_issi(mlxcx_t *, uint16_t); 1307 1308 extern boolean_t mlxcx_cmd_query_pages(mlxcx_t *, uint_t, int32_t *); 1309 extern boolean_t mlxcx_cmd_give_pages(mlxcx_t *, uint_t, int32_t, 1310 mlxcx_dev_page_t **); 1311 extern boolean_t mlxcx_cmd_return_pages(mlxcx_t *, int32_t, uint64_t *, 1312 int32_t *); 1313 1314 extern boolean_t mlxcx_cmd_query_hca_cap(mlxcx_t *, mlxcx_hca_cap_type_t, 1315 mlxcx_hca_cap_mode_t, mlxcx_hca_cap_t *); 1316 1317 extern boolean_t mlxcx_cmd_set_driver_version(mlxcx_t *, const char *); 1318 1319 extern boolean_t mlxcx_cmd_init_hca(mlxcx_t *); 1320 extern boolean_t mlxcx_cmd_teardown_hca(mlxcx_t *); 1321 1322 extern boolean_t mlxcx_cmd_alloc_uar(mlxcx_t *, mlxcx_uar_t *); 1323 extern boolean_t mlxcx_cmd_dealloc_uar(mlxcx_t *, mlxcx_uar_t *); 1324 1325 extern boolean_t mlxcx_cmd_alloc_pd(mlxcx_t *, mlxcx_pd_t *); 1326 extern boolean_t mlxcx_cmd_dealloc_pd(mlxcx_t *, mlxcx_pd_t *); 1327 1328 extern boolean_t mlxcx_cmd_alloc_tdom(mlxcx_t *, mlxcx_tdom_t *); 1329 extern boolean_t mlxcx_cmd_dealloc_tdom(mlxcx_t *, mlxcx_tdom_t *); 1330 1331 extern boolean_t mlxcx_cmd_create_eq(mlxcx_t *, mlxcx_event_queue_t *); 1332 extern boolean_t mlxcx_cmd_destroy_eq(mlxcx_t *, mlxcx_event_queue_t *); 1333 extern boolean_t mlxcx_cmd_query_eq(mlxcx_t *, mlxcx_event_queue_t *, 1334 mlxcx_eventq_ctx_t *); 1335 1336 extern boolean_t mlxcx_cmd_create_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1337 extern boolean_t mlxcx_cmd_destroy_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1338 extern boolean_t mlxcx_cmd_query_cq(mlxcx_t *, mlxcx_completion_queue_t *, 1339 mlxcx_completionq_ctx_t *); 1340 1341 extern boolean_t mlxcx_cmd_create_rq(mlxcx_t *, mlxcx_work_queue_t *); 1342 extern boolean_t mlxcx_cmd_start_rq(mlxcx_t *, mlxcx_work_queue_t *); 1343 extern boolean_t mlxcx_cmd_stop_rq(mlxcx_t *, mlxcx_work_queue_t *); 1344 extern boolean_t mlxcx_cmd_destroy_rq(mlxcx_t *, mlxcx_work_queue_t *); 1345 extern boolean_t mlxcx_cmd_query_rq(mlxcx_t *, mlxcx_work_queue_t *, 1346 mlxcx_rq_ctx_t *); 1347 1348 extern boolean_t mlxcx_cmd_create_tir(mlxcx_t *, mlxcx_tir_t *); 1349 extern boolean_t mlxcx_cmd_destroy_tir(mlxcx_t *, mlxcx_tir_t *); 1350 1351 extern boolean_t mlxcx_cmd_create_sq(mlxcx_t *, mlxcx_work_queue_t *); 1352 extern boolean_t mlxcx_cmd_start_sq(mlxcx_t *, mlxcx_work_queue_t *); 1353 extern boolean_t mlxcx_cmd_stop_sq(mlxcx_t *, mlxcx_work_queue_t *); 1354 extern boolean_t mlxcx_cmd_destroy_sq(mlxcx_t *, mlxcx_work_queue_t *); 1355 extern boolean_t mlxcx_cmd_query_sq(mlxcx_t *, mlxcx_work_queue_t *, 1356 mlxcx_sq_ctx_t *); 1357 1358 extern boolean_t mlxcx_cmd_create_tis(mlxcx_t *, mlxcx_tis_t *); 1359 extern boolean_t mlxcx_cmd_destroy_tis(mlxcx_t *, mlxcx_tis_t *); 1360 1361 extern boolean_t mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *, mlxcx_port_t *); 1362 extern boolean_t mlxcx_cmd_query_special_ctxs(mlxcx_t *); 1363 1364 extern boolean_t mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *, mlxcx_port_t *, 1365 mlxcx_modify_nic_vport_ctx_fields_t); 1366 1367 extern boolean_t mlxcx_cmd_create_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1368 extern boolean_t mlxcx_cmd_destroy_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1369 extern boolean_t mlxcx_cmd_set_flow_table_root(mlxcx_t *, mlxcx_flow_table_t *); 1370 1371 extern boolean_t mlxcx_cmd_create_flow_group(mlxcx_t *, mlxcx_flow_group_t *); 1372 extern boolean_t mlxcx_cmd_set_flow_table_entry(mlxcx_t *, 1373 mlxcx_flow_entry_t *); 1374 extern boolean_t mlxcx_cmd_delete_flow_table_entry(mlxcx_t *, 1375 mlxcx_flow_entry_t *); 1376 extern boolean_t mlxcx_cmd_destroy_flow_group(mlxcx_t *, mlxcx_flow_group_t *); 1377 1378 extern boolean_t mlxcx_cmd_access_register(mlxcx_t *, mlxcx_cmd_reg_opmod_t, 1379 mlxcx_register_id_t, mlxcx_register_data_t *); 1380 extern boolean_t mlxcx_cmd_query_port_mtu(mlxcx_t *, mlxcx_port_t *); 1381 extern boolean_t mlxcx_cmd_query_port_status(mlxcx_t *, mlxcx_port_t *); 1382 extern boolean_t mlxcx_cmd_modify_port_status(mlxcx_t *, mlxcx_port_t *, 1383 mlxcx_port_status_t); 1384 extern boolean_t mlxcx_cmd_query_port_speed(mlxcx_t *, mlxcx_port_t *); 1385 extern boolean_t mlxcx_cmd_query_port_fec(mlxcx_t *, mlxcx_port_t *); 1386 extern boolean_t mlxcx_cmd_modify_port_fec(mlxcx_t *, mlxcx_port_t *, 1387 mlxcx_pplm_fec_caps_t); 1388 1389 extern boolean_t mlxcx_cmd_set_port_mtu(mlxcx_t *, mlxcx_port_t *); 1390 1391 extern boolean_t mlxcx_cmd_create_rqt(mlxcx_t *, mlxcx_rqtable_t *); 1392 extern boolean_t mlxcx_cmd_destroy_rqt(mlxcx_t *, mlxcx_rqtable_t *); 1393 1394 extern boolean_t mlxcx_cmd_set_int_mod(mlxcx_t *, uint_t, uint_t); 1395 1396 extern boolean_t mlxcx_cmd_query_module_status(mlxcx_t *, uint_t, 1397 mlxcx_module_status_t *, mlxcx_module_error_type_t *); 1398 extern boolean_t mlxcx_cmd_set_port_led(mlxcx_t *, mlxcx_port_t *, uint16_t); 1399 1400 /* Comparator for avl_ts */ 1401 extern int mlxcx_cq_compare(const void *, const void *); 1402 extern int mlxcx_dmac_fe_compare(const void *, const void *); 1403 extern int mlxcx_grmac_compare(const void *, const void *); 1404 extern int mlxcx_page_compare(const void *, const void *); 1405 1406 extern void mlxcx_update_link_state(mlxcx_t *, mlxcx_port_t *); 1407 1408 extern void mlxcx_eth_proto_to_string(mlxcx_eth_proto_t, char *, size_t); 1409 extern const char *mlxcx_port_status_string(mlxcx_port_status_t); 1410 1411 extern const char *mlxcx_event_name(mlxcx_event_t); 1412 1413 #ifdef __cplusplus 1414 } 1415 #endif 1416 1417 #endif /* _MLXCX_H */ 1418