1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * Copyright 2017 Joyent, Inc. 26 * Copyright 2025 Oxide Computer Company 27 */ 28 29 #ifndef _SYS_MAC_SOFT_RING_H 30 #define _SYS_MAC_SOFT_RING_H 31 32 #ifdef __cplusplus 33 extern "C" { 34 #endif 35 36 #include <sys/types.h> 37 #include <sys/cpuvar.h> 38 #include <sys/cpupart.h> 39 #include <sys/processor.h> 40 #include <sys/stream.h> 41 #include <sys/squeue.h> 42 #include <sys/dlpi.h> 43 #include <sys/mac_impl.h> 44 #include <sys/mac_stat.h> 45 46 #define S_RING_NAMELEN 64 47 48 #define MAX_SR_FANOUT 24 49 50 extern boolean_t mac_soft_ring_enable; 51 extern boolean_t mac_latency_optimize; 52 53 typedef struct mac_soft_ring_s mac_soft_ring_t; 54 typedef struct mac_soft_ring_set_s mac_soft_ring_set_t; 55 56 typedef void (*mac_soft_ring_drain_func_t)(mac_soft_ring_t *); 57 typedef mac_tx_cookie_t (*mac_tx_func_t)(mac_soft_ring_set_t *, mblk_t *, 58 uintptr_t, uint16_t, mblk_t **); 59 60 61 /* Tx notify callback */ 62 typedef struct mac_tx_notify_cb_s { 63 mac_cb_t mtnf_link; /* Linked list of callbacks */ 64 mac_tx_notify_t mtnf_fn; /* The callback function */ 65 void *mtnf_arg; /* Callback function argument */ 66 } mac_tx_notify_cb_t; 67 68 struct mac_soft_ring_s { 69 /* Keep the most used members 64bytes cache aligned */ 70 kmutex_t s_ring_lock; /* lock before using any member */ 71 uint16_t s_ring_type; /* processing model of the sq */ 72 uint16_t s_ring_state; /* state flags and message count */ 73 int s_ring_count; /* # of mblocks in mac_soft_ring */ 74 size_t s_ring_size; /* Size of data queued */ 75 mblk_t *s_ring_first; /* first mblk chain or NULL */ 76 mblk_t *s_ring_last; /* last mblk chain or NULL */ 77 78 mac_direct_rx_t s_ring_rx_func; 79 void *s_ring_rx_arg1; 80 mac_resource_handle_t s_ring_rx_arg2; 81 82 /* 83 * Threshold after which packets get dropped. 84 * Is always greater than s_ring_tx_hiwat 85 */ 86 int s_ring_tx_max_q_cnt; 87 /* # of mblocks after which to apply flow control */ 88 int s_ring_tx_hiwat; 89 /* # of mblocks after which to relieve flow control */ 90 int s_ring_tx_lowat; 91 boolean_t s_ring_tx_woken_up; 92 uint32_t s_ring_hiwat_cnt; /* times blocked for Tx descs */ 93 94 void *s_ring_tx_arg1; 95 void *s_ring_tx_arg2; 96 97 /* Tx notify callback */ 98 mac_cb_info_t s_ring_notify_cb_info; /* cb list info */ 99 mac_cb_t *s_ring_notify_cb_list; /* The cb list */ 100 101 clock_t s_ring_awaken; /* time async thread was awakened */ 102 103 kthread_t *s_ring_run; /* Current thread processing sq */ 104 processorid_t s_ring_cpuid; /* processor to bind to */ 105 processorid_t s_ring_cpuid_save; /* saved cpuid during offline */ 106 kcondvar_t s_ring_async; /* async thread blocks on */ 107 clock_t s_ring_wait; /* lbolts to wait after a fill() */ 108 timeout_id_t s_ring_tid; /* timer id of pending timeout() */ 109 kthread_t *s_ring_worker; /* kernel thread id */ 110 char s_ring_name[S_RING_NAMELEN + 1]; 111 uint32_t s_ring_total_inpkt; 112 uint32_t s_ring_total_rbytes; 113 uint32_t s_ring_drops; 114 struct mac_client_impl_s *s_ring_mcip; 115 kstat_t *s_ring_ksp; 116 117 /* Teardown, poll disable control ops */ 118 kcondvar_t s_ring_client_cv; /* Client wait for control op */ 119 120 mac_soft_ring_set_t *s_ring_set; /* The SRS this ring belongs to */ 121 mac_soft_ring_t *s_ring_next; 122 mac_soft_ring_t *s_ring_prev; 123 mac_soft_ring_drain_func_t s_ring_drain_func; 124 125 mac_tx_stats_t s_st_stat; 126 }; 127 128 typedef void (*mac_srs_drain_proc_t)(mac_soft_ring_set_t *, uint_t); 129 130 /* Transmit side Soft Ring Set */ 131 typedef struct mac_srs_tx_s { 132 /* Members for Tx size processing */ 133 uint32_t st_mode; 134 mac_tx_func_t st_func; 135 void *st_arg1; 136 void *st_arg2; 137 mac_group_t *st_group; /* TX group for share */ 138 boolean_t st_woken_up; 139 140 /* 141 * st_max_q_cnt is the queue depth threshold to limit 142 * outstanding packets on the Tx SRS. Once the limit 143 * is reached, Tx SRS will drop packets until the 144 * limit goes below the threshold. 145 */ 146 uint32_t st_max_q_cnt; /* max. outstanding packets */ 147 /* 148 * st_hiwat is used Tx serializer and bandwidth mode. 149 * This is the queue depth threshold upto which 150 * packets will get buffered with no flow-control 151 * back pressure applied to the caller. Once this 152 * threshold is reached, back pressure will be 153 * applied to the caller of mac_tx() (mac_tx() starts 154 * returning a cookie to indicate a blocked SRS). 155 * st_hiwat should always be lesser than or equal to 156 * st_max_q_cnt. 157 */ 158 uint32_t st_hiwat; /* mblk cnt to apply flow control */ 159 uint32_t st_lowat; /* mblk cnt to relieve flow control */ 160 uint32_t st_hiwat_cnt; /* times blocked for Tx descs */ 161 mac_tx_stats_t st_stat; 162 mac_capab_aggr_t st_capab_aggr; 163 /* 164 * st_soft_rings is used as an array to store aggr Tx soft 165 * rings. When aggr_find_tx_ring() returns a pseudo ring, 166 * the associated soft ring has to be found. st_soft_rings 167 * array stores the soft ring associated with a pseudo Tx 168 * ring and it can be accessed using the pseudo ring 169 * index (mr_index). Note that the ring index is unique 170 * for each ring in a group. 171 */ 172 mac_soft_ring_t **st_soft_rings; 173 } mac_srs_tx_t; 174 175 /* Receive side Soft Ring Set */ 176 typedef struct mac_srs_rx_s { 177 /* 178 * Upcall Function for fanout, Rx processing etc. Perhaps 179 * the same 3 members below can be used for Tx 180 * processing, but looking around, mac_rx_func_t has 181 * proliferated too much into various files at different 182 * places. I am leaving the consolidation battle for 183 * another day. 184 */ 185 mac_direct_rx_t sr_func; /* srs_lock */ 186 void *sr_arg1; /* srs_lock */ 187 mac_resource_handle_t sr_arg2; /* srs_lock */ 188 mac_rx_func_t sr_lower_proc; /* Atomically changed */ 189 uint32_t sr_poll_pkt_cnt; 190 uint32_t sr_poll_thres; 191 192 /* mblk cnt to apply flow control */ 193 uint32_t sr_hiwat; 194 /* mblk cnt to relieve flow control */ 195 uint32_t sr_lowat; 196 mac_rx_stats_t sr_stat; 197 198 /* Times polling was enabled */ 199 uint32_t sr_poll_on; 200 /* Times polling was enabled by worker thread */ 201 uint32_t sr_worker_poll_on; 202 /* Times polling was disabled */ 203 uint32_t sr_poll_off; 204 /* Poll thread signalled count */ 205 uint32_t sr_poll_thr_sig; 206 /* Poll thread busy */ 207 uint32_t sr_poll_thr_busy; 208 /* SRS drains, stays in poll mode but doesn't poll */ 209 uint32_t sr_poll_drain_no_poll; 210 /* 211 * SRS has nothing to do and no packets in H/W but 212 * there is a backlog in softrings. SRS stays in 213 * poll mode but doesn't do polling. 214 */ 215 uint32_t sr_poll_no_poll; 216 /* Active polling restarted */ 217 uint32_t sr_below_hiwat; 218 /* Found packets in last poll so try and poll again */ 219 uint32_t sr_poll_again; 220 /* 221 * Packets in queue but poll thread not allowed to process so 222 * signal the worker thread. 223 */ 224 uint32_t sr_poll_sig_worker; 225 /* 226 * Poll thread has nothing to do and H/W has nothing so 227 * reenable the interrupts. 228 */ 229 uint32_t sr_poll_intr_enable; 230 /* 231 * Poll thread has nothing to do and worker thread was already 232 * running so it can decide to reenable interrupt or poll again. 233 */ 234 uint32_t sr_poll_goto_sleep; 235 /* Worker thread goes back to draining the queue */ 236 uint32_t sr_drain_again; 237 /* More Packets in queue so signal the poll thread to drain */ 238 uint32_t sr_drain_poll_sig; 239 /* More Packets in queue so signal the worker thread to drain */ 240 uint32_t sr_drain_worker_sig; 241 /* Poll thread is already running so worker has nothing to do */ 242 uint32_t sr_drain_poll_running; 243 /* We have packets already queued so keep polling */ 244 uint32_t sr_drain_keep_polling; 245 /* Drain is done and interrupts are reenabled */ 246 uint32_t sr_drain_finish_intr; 247 /* Polling thread needs to schedule worker wakeup */ 248 uint32_t sr_poll_worker_wakeup; 249 } mac_srs_rx_t; 250 251 /* 252 * mac_soft_ring_set_s: 253 * This is used both for Tx and Rx side. The srs_type identifies Rx or 254 * Tx type. 255 * 256 * Note that the structure is carefully crafted, with Rx elements coming 257 * first followed by Tx specific members. Future additions to this 258 * structure should follow the same guidelines. 259 * 260 * Rx-side notes: 261 * mac_rx_classify_flow_add() always creates a mac_soft_ring_set_t and fn_flow 262 * points to info from it (func = srs_lower_proc, arg = soft_ring_set). On 263 * interrupt path, srs_lower_proc does B/W adjustment and switch to polling mode 264 * (if poll capable) and feeds the packets to soft_ring_list via choosen 265 * fanout type (specified by srs_type). In poll mode, the poll thread which is 266 * also a pointer can pick up the packets and feed them to various 267 * soft_ring_list. 268 * 269 * The srs_type can either be protocol based or fanout based where fanout itelf 270 * can be various types 271 * 272 * The polling works by turning off interrupts as soon as a packets 273 * are queued on the soft ring set. Once the backlog is clear and poll 274 * thread return empty handed i.e. Rx ring doesn't have anything, the 275 * interrupt is turned back on. For this purpose we keep a separate 276 * srs_poll_pkt_cnt counter which tracks the packets queued between SRS 277 * and the soft rings as well. The counter is incremented when packets 278 * are queued and decremented when SRS processes them (in case it has 279 * no soft rings) or the soft ring process them. Its important that 280 * in case SRS has softrings, the decrement doesn't happen till the 281 * packet is processed by the soft rings since it takes very little time 282 * for SRS to queue packet from SRS to soft rings and it will keep 283 * bringing more packets in the system faster than soft rings can 284 * process them. 285 * 286 * Tx side notes: 287 * The srs structure acts as a serializer with a worker thread. The 288 * default behavior of srs though is to act as a pass-thru. The queues 289 * (srs_first, srs_last, srs_count) get used when Tx ring runs out of Tx 290 * descriptors or to enforce bandwidth limits. 291 * 292 * When multiple Tx rings are present, the SRS state will be set to 293 * SRS_FANOUT_OTH. Outgoing packets coming into mac_tx_srs_process() 294 * function will be fanned out to one of the Tx side soft rings based on 295 * a hint passed in mac_tx_srs_process(). Each soft ring, in turn, will 296 * be associated with a distinct h/w Tx ring. 297 */ 298 299 struct mac_soft_ring_set_s { 300 /* 301 * Common elements, common to both Rx and Tx SRS type. 302 * The following block of fields are protected by srs_lock 303 */ 304 kmutex_t srs_lock; 305 uint32_t srs_type; 306 uint32_t srs_state; /* state flags */ 307 uint32_t srs_count; 308 mblk_t *srs_first; /* first mblk chain or NULL */ 309 mblk_t *srs_last; /* last mblk chain or NULL */ 310 kcondvar_t srs_async; /* cv for worker thread */ 311 kcondvar_t srs_cv; /* cv for poll thread */ 312 kcondvar_t srs_quiesce_done_cv; /* cv for removal */ 313 timeout_id_t srs_tid; /* timeout id for pending timeout */ 314 315 /* 316 * List of soft rings & processing function. 317 * The following block is protected by Rx quiescence. 318 * i.e. they can be changed only after quiescing the SRS 319 * Protected by srs_lock. 320 */ 321 mac_soft_ring_t *srs_soft_ring_head; 322 mac_soft_ring_t *srs_soft_ring_tail; 323 int srs_soft_ring_count; 324 int srs_soft_ring_quiesced_count; 325 int srs_soft_ring_condemned_count; 326 mac_soft_ring_t **srs_tcp_soft_rings; 327 int srs_tcp_ring_count; 328 mac_soft_ring_t **srs_udp_soft_rings; 329 int srs_udp_ring_count; 330 mac_soft_ring_t **srs_tcp6_soft_rings; 331 int srs_tcp6_ring_count; 332 mac_soft_ring_t **srs_udp6_soft_rings; 333 int srs_udp6_ring_count; 334 mac_soft_ring_t **srs_oth_soft_rings; 335 int srs_oth_ring_count; 336 /* 337 * srs_tx_soft_rings is used by tx_srs in 338 * when operating in multi tx ring mode. 339 */ 340 mac_soft_ring_t **srs_tx_soft_rings; 341 int srs_tx_ring_count; 342 343 /* 344 * Bandwidth control related members. 345 * They are common to both Rx- and Tx-side. 346 * Following protected by srs_lock 347 */ 348 mac_bw_ctl_t *srs_bw; 349 size_t srs_size; /* Size of packets queued in bytes */ 350 pri_t srs_pri; 351 352 mac_soft_ring_set_t *srs_next; /* mac_srs_g_lock */ 353 mac_soft_ring_set_t *srs_prev; /* mac_srs_g_lock */ 354 355 /* Attribute specific drain func (BW ctl vs non-BW ctl) */ 356 mac_srs_drain_proc_t srs_drain_func; /* Write once (WO) */ 357 358 /* 359 * If the associated ring is exclusively used by a mac client, e.g., 360 * an aggregation, this fields is used to keep a reference to the 361 * MAC client's pseudo ring. 362 */ 363 mac_resource_handle_t srs_mrh; 364 /* 365 * The following blocks are write once (WO) and valid for the life 366 * of the SRS 367 */ 368 struct mac_client_impl_s *srs_mcip; /* back ptr to mac client */ 369 void *srs_flent; /* back ptr to flent */ 370 mac_ring_t *srs_ring; /* Ring Descriptor */ 371 372 /* Teardown, disable control ops */ 373 kcondvar_t srs_client_cv; /* Client wait for the control op */ 374 375 kthread_t *srs_worker; /* WO, worker thread */ 376 kthread_t *srs_poll_thr; /* WO, poll thread */ 377 378 uint_t srs_ind; /* Round Robin indx for picking up SR */ 379 processorid_t srs_worker_cpuid; /* processor to bind to */ 380 processorid_t srs_worker_cpuid_save; /* saved cpuid during offline */ 381 processorid_t srs_poll_cpuid; /* processor to bind to */ 382 processorid_t srs_poll_cpuid_save; /* saved cpuid during offline */ 383 uint_t srs_fanout_state; 384 mac_cpus_t srs_cpu; 385 386 mac_srs_rx_t srs_rx; 387 mac_srs_tx_t srs_tx; 388 kstat_t *srs_ksp; 389 }; 390 391 /* 392 * type flags - combination allowed to process and drain the queue 393 */ 394 #define ST_RING_WORKER_ONLY 0x0001 /* Worker thread only */ 395 #define ST_RING_ANY 0x0002 /* Any thread can process the queue */ 396 #define ST_RING_TCP 0x0004 397 #define ST_RING_UDP 0x0008 398 #define ST_RING_OTH 0x0010 399 400 #define ST_RING_BW_CTL 0x0020 401 #define ST_RING_TX 0x0040 402 403 #define ST_RING_TCP6 0x0080 404 #define ST_RING_UDP6 0x0100 405 406 /* 407 * State flags. 408 */ 409 #define S_RING_PROC 0x0001 /* being processed */ 410 #define S_RING_BOUND 0x0002 /* Worker thread is bound to a cpu */ 411 #define S_RING_BLOCK 0x0004 /* No Tx descs */ 412 #define S_RING_TX_HIWAT 0x0008 /* Tx high watermark reached */ 413 414 #define S_RING_WAKEUP_CLIENT 0x0010 /* flow ctrl, client wakeup needed */ 415 #define S_RING_BLANK 0x0020 /* Has been put into polling mode */ 416 #define S_RING_CLIENT_WAIT 0x0040 /* Client waiting for control op */ 417 418 #define S_RING_CONDEMNED 0x0100 /* Being torn down */ 419 #define S_RING_CONDEMNED_DONE 0x0200 /* Being torn down */ 420 #define S_RING_QUIESCE 0x0400 /* No traffic flow, transient flag */ 421 #define S_RING_QUIESCE_DONE 0x0800 /* No traffic flow, transient flag */ 422 423 #define S_RING_RESTART 0x1000 /* Go back to normal traffic flow */ 424 #define S_RING_ENQUEUED 0x2000 /* Pkts enqueued in Tx soft ring */ 425 426 /* 427 * arguments for processors to bind to 428 */ 429 #define S_RING_BIND_NONE -1 430 431 /* 432 * defines for srs_type - identifies a link or a sub-flow 433 * and other static characteristics of a SRS like a tx 434 * srs, tcp only srs, etc. 435 */ 436 #define SRST_LINK 0x00000001 437 #define SRST_FLOW 0x00000002 438 #define SRST_NO_SOFT_RINGS 0x00000004 439 #define SRST_TCP_ONLY 0x00000008 440 441 #define SRST_FANOUT_PROTO 0x00000010 442 #define SRST_FANOUT_SRC_IP 0x00000020 443 #define SRST_FANOUT_OTH 0x00000040 444 #define SRST_DEFAULT_GRP 0x00000080 445 446 #define SRST_TX 0x00000100 447 #define SRST_BW_CONTROL 0x00000200 448 #define SRST_DIRECT_POLL 0x00000400 449 450 #define SRST_DLS_BYPASS 0x00001000 451 #define SRST_CLIENT_POLL_ENABLED 0x00002000 452 453 /* 454 * soft ring set flags. These bits are dynamic in nature and get 455 * applied to srs_state. They reflect the state of SRS at any 456 * point of time 457 */ 458 #define SRS_BLANK 0x00000001 459 #define SRS_WORKER_BOUND 0x00000002 460 #define SRS_POLL_BOUND 0x00000004 461 #define SRS_POLLING_CAPAB 0x00000008 462 463 #define SRS_PROC 0x00000010 464 #define SRS_GET_PKTS 0x00000020 465 #define SRS_POLLING 0x00000040 466 #define SRS_BW_ENFORCED 0x00000080 467 468 #define SRS_WORKER 0x00000100 469 #define SRS_ENQUEUED 0x00000200 470 #define SRS_ANY_PROCESS 0x00000400 471 #define SRS_PROC_FAST 0x00000800 472 473 #define SRS_POLL_PROC 0x00001000 474 #define SRS_TX_BLOCKED 0x00002000 /* out of Tx descs */ 475 #define SRS_TX_HIWAT 0x00004000 /* Tx count exceeds hiwat */ 476 #define SRS_TX_WAKEUP_CLIENT 0x00008000 /* Flow-ctl: wakeup client */ 477 478 #define SRS_CLIENT_PROC 0x00010000 479 #define SRS_CLIENT_WAIT 0x00020000 480 #define SRS_QUIESCE 0x00040000 481 #define SRS_QUIESCE_DONE 0x00080000 482 483 #define SRS_CONDEMNED 0x00100000 484 #define SRS_CONDEMNED_DONE 0x00200000 485 #define SRS_POLL_THR_QUIESCED 0x00400000 486 #define SRS_RESTART 0x00800000 487 488 #define SRS_RESTART_DONE 0x01000000 489 #define SRS_POLL_THR_RESTART 0x02000000 490 #define SRS_IN_GLIST 0x04000000 491 #define SRS_POLL_THR_EXITED 0x08000000 492 493 #define SRS_QUIESCE_PERM 0x10000000 494 #define SRS_LATENCY_OPT 0x20000000 495 #define SRS_SOFTRING_QUEUE 0x40000000 496 497 #define SRS_QUIESCED(srs) (srs->srs_state & SRS_QUIESCE_DONE) 498 499 /* 500 * If the SRS_QUIESCE_PERM flag is set, the SRS worker thread will not be 501 * able to be restarted. 502 */ 503 #define SRS_QUIESCED_PERMANENT(srs) (srs->srs_state & SRS_QUIESCE_PERM) 504 505 /* 506 * soft ring set (SRS) Tx modes 507 */ 508 typedef enum { 509 SRS_TX_DEFAULT = 0, 510 SRS_TX_SERIALIZE, 511 SRS_TX_FANOUT, 512 SRS_TX_BW, 513 SRS_TX_BW_FANOUT, 514 SRS_TX_AGGR, 515 SRS_TX_BW_AGGR 516 } mac_tx_srs_mode_t; 517 518 /* 519 * SRS fanout states 520 */ 521 typedef enum { 522 SRS_FANOUT_UNINIT = 0, 523 SRS_FANOUT_INIT, 524 SRS_FANOUT_REINIT 525 } mac_srs_fanout_state_t; 526 527 /* 528 * Structure for dls statistics 529 */ 530 struct dls_kstats { 531 kstat_named_t dlss_soft_ring_pkt_drop; 532 }; 533 534 extern struct dls_kstats dls_kstat; 535 536 #define DLS_BUMP_STAT(x, y) (dls_kstat.x.value.ui32 += y) 537 538 /* Turn dynamic polling off */ 539 #define MAC_SRS_POLLING_OFF(mac_srs) { \ 540 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 541 if (((mac_srs)->srs_state & (SRS_POLLING_CAPAB|SRS_POLLING)) == \ 542 (SRS_POLLING_CAPAB|SRS_POLLING)) { \ 543 (mac_srs)->srs_state &= ~SRS_POLLING; \ 544 (void) mac_hwring_enable_intr((mac_ring_handle_t) \ 545 (mac_srs)->srs_ring); \ 546 (mac_srs)->srs_rx.sr_poll_off++; \ 547 } \ 548 } 549 550 #define MAC_COUNT_CHAIN(mac_srs, head, tail, cnt, sz) { \ 551 mblk_t *tmp; \ 552 boolean_t bw_ctl = B_FALSE; \ 553 \ 554 ASSERT((head) != NULL); \ 555 cnt = 0; \ 556 sz = 0; \ 557 if ((mac_srs)->srs_type & SRST_BW_CONTROL) \ 558 bw_ctl = B_TRUE; \ 559 tmp = tail = (head); \ 560 if ((head)->b_next == NULL) { \ 561 cnt = 1; \ 562 if (bw_ctl) \ 563 sz += msgdsize(head); \ 564 } else { \ 565 while (tmp != NULL) { \ 566 tail = tmp; \ 567 cnt++; \ 568 if (bw_ctl) \ 569 sz += msgdsize(tmp); \ 570 tmp = tmp->b_next; \ 571 } \ 572 } \ 573 } 574 575 /* 576 * Decrement the cumulative packet count in SRS and its 577 * soft rings. If the srs_poll_pkt_cnt goes below lowat, then check 578 * if if the interface was left in a polling mode and no one 579 * is really processing the queue (to get the interface out 580 * of poll mode). If no one is processing the queue, then 581 * acquire the PROC and signal the poll thread to check the 582 * interface for packets and get the interface back to interrupt 583 * mode if nothing is found. 584 */ 585 #define MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt) { \ 586 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \ 587 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 588 \ 589 srs_rx->sr_poll_pkt_cnt -= cnt; \ 590 if ((srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_poll_thres) && \ 591 (((mac_srs)->srs_state & \ 592 (SRS_POLLING|SRS_PROC|SRS_GET_PKTS)) == SRS_POLLING)) \ 593 { \ 594 (mac_srs)->srs_state |= (SRS_PROC|SRS_GET_PKTS); \ 595 cv_signal(&(mac_srs)->srs_cv); \ 596 srs_rx->sr_below_hiwat++; \ 597 } \ 598 } 599 600 /* 601 * The following two macros are used to update the inbound packet and byte. 602 * count. The packet and byte count reflect the packets and bytes that are 603 * taken out of the SRS's queue, i.e. indicating they are being delivered. 604 * The srs_count and srs_size are updated in different locations as the 605 * srs_size is also used to take into account any bandwidth limits. The 606 * srs_size is updated only when a soft ring, if any, sends a packet up, 607 * as opposed to updating it when the SRS sends a packet to the SR, i.e. 608 * the srs_size reflects the packets in the SRS and SRs. These 609 * macros decrement the srs_size and srs_count and also increment the 610 * ipackets and ibytes stats resp. 611 * 612 * xxx-venu These are done under srs_lock, for now we still update 613 * mci_stat_ibytes/mci_stat_ipackets atomically, need to check if 614 * just updating them would be accurate enough. 615 * 616 * If we are updating these for a sub-flow SRS, then we need to also 617 * updated it's MAC client bandwidth info, if the MAC client is also 618 * bandwidth regulated. 619 */ 620 #define MAC_UPDATE_SRS_SIZE_LOCKED(srs, sz) { \ 621 if ((srs)->srs_type & SRST_BW_CONTROL) { \ 622 mutex_enter(&(srs)->srs_bw->mac_bw_lock); \ 623 (srs)->srs_bw->mac_bw_sz -= (sz); \ 624 (srs)->srs_bw->mac_bw_used += (sz); \ 625 mutex_exit(&(srs)->srs_bw->mac_bw_lock); \ 626 } \ 627 } 628 629 #define MAC_TX_UPDATE_BW_INFO(srs, sz) { \ 630 (srs)->srs_bw->mac_bw_sz -= (sz); \ 631 (srs)->srs_bw->mac_bw_used += (sz); \ 632 } 633 634 #define MAC_TX_SOFT_RINGS(mac_srs) ((mac_srs)->srs_tx_ring_count >= 1) 635 636 /* Soft ring flags for teardown */ 637 #define SRS_POLL_THR_OWNER (SRS_PROC | SRS_POLLING | SRS_GET_PKTS) 638 #define SRS_PAUSE (SRS_CONDEMNED | SRS_QUIESCE) 639 #define S_RING_PAUSE (S_RING_CONDEMNED | S_RING_QUIESCE) 640 641 /* Soft rings */ 642 extern void mac_soft_ring_init(void); 643 extern void mac_soft_ring_finish(void); 644 extern void mac_fanout_setup(mac_client_impl_t *, flow_entry_t *, 645 mac_resource_props_t *, mac_direct_rx_t, void *, mac_resource_handle_t, 646 cpupart_t *); 647 648 extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *); 649 extern void mac_soft_ring_blank(void *, time_t, uint_t, int); 650 extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, size_t); 651 extern void mac_soft_ring_destroy(mac_soft_ring_t *); 652 extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *); 653 654 /* Rx SRS */ 655 extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *, 656 flow_entry_t *, uint32_t, mac_direct_rx_t, void *, mac_resource_handle_t, 657 mac_ring_t *); 658 extern void mac_srs_free(mac_soft_ring_set_t *); 659 extern void mac_srs_signal(mac_soft_ring_set_t *, uint_t); 660 extern cpu_t *mac_srs_bind(mac_soft_ring_set_t *, processorid_t); 661 extern void mac_rx_srs_retarget_intr(mac_soft_ring_set_t *, processorid_t); 662 extern void mac_tx_srs_retarget_intr(mac_soft_ring_set_t *); 663 664 extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *); 665 extern void mac_srs_client_poll_enable(struct mac_client_impl_s *, 666 mac_soft_ring_set_t *); 667 extern void mac_srs_client_poll_disable(struct mac_client_impl_s *, 668 mac_soft_ring_set_t *); 669 extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *, 670 mac_soft_ring_set_t *); 671 extern void mac_srs_client_poll_restart(struct mac_client_impl_s *, 672 mac_soft_ring_set_t *); 673 extern void mac_rx_srs_quiesce(mac_soft_ring_set_t *, uint_t); 674 extern void mac_rx_srs_restart(mac_soft_ring_set_t *); 675 extern void mac_rx_srs_subflow_process(void *, mac_resource_handle_t, mblk_t *, 676 boolean_t); 677 extern void mac_tx_srs_quiesce(mac_soft_ring_set_t *, uint_t); 678 679 /* Tx SRS, Tx softring */ 680 extern void mac_tx_srs_wakeup(mac_soft_ring_set_t *, mac_ring_handle_t); 681 extern void mac_tx_srs_setup(struct mac_client_impl_s *, flow_entry_t *); 682 extern mac_tx_func_t mac_tx_get_func(uint32_t); 683 extern mblk_t *mac_tx_send(mac_client_handle_t, mac_ring_handle_t, mblk_t *, 684 mac_tx_stats_t *); 685 extern boolean_t mac_tx_srs_ring_present(mac_soft_ring_set_t *, mac_ring_t *); 686 extern mac_soft_ring_t *mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *, 687 mac_ring_t *); 688 extern void mac_tx_srs_add_ring(mac_soft_ring_set_t *, mac_ring_t *); 689 extern void mac_tx_srs_del_ring(mac_soft_ring_set_t *, mac_ring_t *); 690 extern mac_tx_cookie_t mac_tx_srs_no_desc(mac_soft_ring_set_t *, mblk_t *, 691 uint16_t, mblk_t **); 692 693 /* Subflow specific stuff */ 694 extern int mac_srs_flow_create(struct mac_client_impl_s *, flow_entry_t *, 695 mac_resource_props_t *, int, int, mac_direct_rxs_t *); 696 extern void mac_srs_update_bwlimit(flow_entry_t *, mac_resource_props_t *); 697 extern void mac_srs_adjust_subflow_bwlimit(struct mac_client_impl_s *); 698 extern void mac_srs_update_drv(struct mac_client_impl_s *); 699 extern void mac_update_srs_priority(mac_soft_ring_set_t *, pri_t); 700 extern void mac_client_update_classifier(mac_client_impl_t *, boolean_t); 701 702 extern int mac_soft_ring_intr_enable(void *); 703 extern boolean_t mac_soft_ring_intr_disable(void *); 704 extern mac_soft_ring_t *mac_soft_ring_create(int, clock_t, uint16_t, 705 pri_t, mac_client_impl_t *, mac_soft_ring_set_t *, 706 processorid_t, mac_direct_rx_t, void *, mac_resource_handle_t); 707 extern cpu_t *mac_soft_ring_bind(mac_soft_ring_t *, processorid_t); 708 extern void mac_soft_ring_unbind(mac_soft_ring_t *); 709 extern void mac_soft_ring_free(mac_soft_ring_t *); 710 extern void mac_soft_ring_signal(mac_soft_ring_t *, uint_t); 711 extern void mac_rx_soft_ring_process(mac_client_impl_t *, mac_soft_ring_t *, 712 mblk_t *, mblk_t *, int, size_t); 713 extern mac_tx_cookie_t mac_tx_soft_ring_process(mac_soft_ring_t *, 714 mblk_t *, uint16_t, mblk_t **); 715 extern void mac_srs_worker_quiesce(mac_soft_ring_set_t *); 716 extern void mac_srs_worker_restart(mac_soft_ring_set_t *); 717 extern void mac_rx_attach_flow_srs(mac_impl_t *, flow_entry_t *, 718 mac_soft_ring_set_t *, mac_ring_t *, mac_classify_type_t); 719 720 extern void mac_rx_srs_drain_bw(mac_soft_ring_set_t *, uint_t); 721 extern void mac_rx_srs_drain(mac_soft_ring_set_t *, uint_t); 722 extern void mac_rx_srs_process(void *, mac_resource_handle_t, mblk_t *, 723 boolean_t); 724 extern void mac_srs_worker(mac_soft_ring_set_t *); 725 extern void mac_rx_srs_poll_ring(mac_soft_ring_set_t *); 726 extern void mac_tx_srs_drain(mac_soft_ring_set_t *, uint_t); 727 728 extern void mac_tx_srs_restart(mac_soft_ring_set_t *); 729 extern void mac_rx_srs_remove(mac_soft_ring_set_t *); 730 731 #ifdef __cplusplus 732 } 733 #endif 734 735 #endif /* _SYS_MAC_SOFT_RING_H */ 736