1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright 2011 Joyent, Inc. All rights reserved. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/callb.h> 31 #include <sys/sdt.h> 32 #include <sys/strsubr.h> 33 #include <sys/strsun.h> 34 #include <sys/vlan.h> 35 #include <sys/stack.h> 36 #include <sys/archsystm.h> 37 #include <inet/ipsec_impl.h> 38 #include <inet/ip_impl.h> 39 #include <inet/sadb.h> 40 #include <inet/ipsecesp.h> 41 #include <inet/ipsecah.h> 42 #include <inet/ip6.h> 43 44 #include <sys/mac_impl.h> 45 #include <sys/mac_client_impl.h> 46 #include <sys/mac_client_priv.h> 47 #include <sys/mac_soft_ring.h> 48 #include <sys/mac_flow_impl.h> 49 50 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *, 51 uintptr_t, uint16_t, mblk_t **); 52 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *, 53 uintptr_t, uint16_t, mblk_t **); 54 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *, 55 uintptr_t, uint16_t, mblk_t **); 56 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *, 57 uintptr_t, uint16_t, mblk_t **); 58 static mac_tx_cookie_t mac_tx_aggr_mode(mac_soft_ring_set_t *, mblk_t *, 59 uintptr_t, uint16_t, mblk_t **); 60 61 typedef struct mac_tx_mode_s { 62 mac_tx_srs_mode_t mac_tx_mode; 63 mac_tx_func_t mac_tx_func; 64 } mac_tx_mode_t; 65 66 /* 67 * There are seven modes of operation on the Tx side. These modes get set 68 * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode, 69 * none of the other modes are user configurable. They get selected by 70 * the system depending upon whether the link (or flow) has multiple Tx 71 * rings or a bandwidth configured, or if the link is an aggr, etc. 72 * 73 * When the Tx SRS is operating in aggr mode (st_mode) or if there are 74 * multiple Tx rings owned by Tx SRS, then each Tx ring (pseudo or 75 * otherwise) will have a soft ring associated with it. These soft rings 76 * are stored in srs_tx_soft_rings[] array. 77 * 78 * Additionally in the case of aggr, there is the st_soft_rings[] array 79 * in the mac_srs_tx_t structure. This array is used to store the same 80 * set of soft rings that are present in srs_tx_soft_rings[] array but 81 * in a different manner. The soft ring associated with the pseudo Tx 82 * ring is saved at mr_index (of the pseudo ring) in st_soft_rings[] 83 * array. This helps in quickly getting the soft ring associated with the 84 * Tx ring when aggr_find_tx_ring() returns the pseudo Tx ring that is to 85 * be used for transmit. 86 */ 87 mac_tx_mode_t mac_tx_mode_list[] = { 88 {SRS_TX_DEFAULT, mac_tx_single_ring_mode}, 89 {SRS_TX_SERIALIZE, mac_tx_serializer_mode}, 90 {SRS_TX_FANOUT, mac_tx_fanout_mode}, 91 {SRS_TX_BW, mac_tx_bw_mode}, 92 {SRS_TX_BW_FANOUT, mac_tx_bw_mode}, 93 {SRS_TX_AGGR, mac_tx_aggr_mode}, 94 {SRS_TX_BW_AGGR, mac_tx_bw_mode} 95 }; 96 97 /* 98 * Soft Ring Set (SRS) - The Run time code that deals with 99 * dynamic polling from the hardware, bandwidth enforcement, 100 * fanout etc. 101 * 102 * We try to use H/W classification on NIC and assign traffic for 103 * a MAC address to a particular Rx ring or ring group. There is a 104 * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically 105 * switches the underlying Rx ring between interrupt and 106 * polling mode and enforces any specified B/W control. 107 * 108 * There is always a SRS created and tied to each H/W and S/W rule. 109 * Whenever we create a H/W rule, we always add the the same rule to 110 * S/W classifier and tie a SRS to it. 111 * 112 * In case a B/W control is specified, it is broken into bytes 113 * per ticks and as soon as the quota for a tick is exhausted, 114 * the underlying Rx ring is forced into poll mode for remainder of 115 * the tick. The SRS poll thread only polls for bytes that are 116 * allowed to come in the SRS. We typically let 4x the configured 117 * B/W worth of packets to come in the SRS (to prevent unnecessary 118 * drops due to bursts) but only process the specified amount. 119 * 120 * A MAC client (e.g. a VNIC or aggr) can have 1 or more 121 * Rx rings (and corresponding SRSs) assigned to it. The SRS 122 * in turn can have softrings to do protocol level fanout or 123 * softrings to do S/W based fanout or both. In case the NIC 124 * has no Rx rings, we do S/W classification to respective SRS. 125 * The S/W classification rule is always setup and ready. This 126 * allows the MAC layer to reassign Rx rings whenever needed 127 * but packets still continue to flow via the default path and 128 * getting S/W classified to correct SRS. 129 * 130 * The SRS's are used on both Tx and Rx side. They use the same 131 * data structure but the processing routines have slightly different 132 * semantics due to the fact that Rx side needs to do dynamic 133 * polling etc. 134 * 135 * Dynamic Polling Notes 136 * ===================== 137 * 138 * Each Soft ring set is capable of switching its Rx ring between 139 * interrupt and poll mode and actively 'polls' for packets in 140 * poll mode. If the SRS is implementing a B/W limit, it makes 141 * sure that only Max allowed packets are pulled in poll mode 142 * and goes to poll mode as soon as B/W limit is exceeded. As 143 * such, there are no overheads to implement B/W limits. 144 * 145 * In poll mode, its better to keep the pipeline going where the 146 * SRS worker thread keeps processing packets and poll thread 147 * keeps bringing more packets (specially if they get to run 148 * on different CPUs). This also prevents the overheads associated 149 * by excessive signalling (on NUMA machines, this can be 150 * pretty devastating). The exception is latency optimized case 151 * where worker thread does no work and interrupt and poll thread 152 * are allowed to do their own drain. 153 * 154 * We use the following policy to control Dynamic Polling: 155 * 1) We switch to poll mode anytime the processing 156 * thread causes a backlog to build up in SRS and 157 * its associated Soft Rings (sr_poll_pkt_cnt > 0). 158 * 2) As long as the backlog stays under the low water 159 * mark (sr_lowat), we poll the H/W for more packets. 160 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low 161 * water mark, we stay in poll mode but don't poll 162 * the H/W for more packets. 163 * 4) Anytime in polling mode, if we poll the H/W for 164 * packets and find nothing plus we have an existing 165 * backlog (sr_poll_pkt_cnt > 0), we stay in polling 166 * mode but don't poll the H/W for packets anymore 167 * (let the polling thread go to sleep). 168 * 5) Once the backlog is relived (packets are processed) 169 * we reenable polling (by signalling the poll thread) 170 * only when the backlog dips below sr_poll_thres. 171 * 6) sr_hiwat is used exclusively when we are not 172 * polling capable and is used to decide when to 173 * drop packets so the SRS queue length doesn't grow 174 * infinitely. 175 * 176 * NOTE: Also see the block level comment on top of mac_soft_ring.c 177 */ 178 179 /* 180 * mac_latency_optimize 181 * 182 * Controls whether the poll thread can process the packets inline 183 * or let the SRS worker thread do the processing. This applies if 184 * the SRS was not being processed. For latency sensitive traffic, 185 * this needs to be true to allow inline processing. For throughput 186 * under load, this should be false. 187 * 188 * This (and other similar) tunable should be rolled into a link 189 * or flow specific workload hint that can be set using dladm 190 * linkprop (instead of multiple such tunables). 191 */ 192 boolean_t mac_latency_optimize = B_TRUE; 193 194 /* 195 * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN 196 * 197 * queue a mp or chain in soft ring set and increment the 198 * local count (srs_count) for the SRS and the shared counter 199 * (srs_poll_pkt_cnt - shared between SRS and its soft rings 200 * to track the total unprocessed packets for polling to work 201 * correctly). 202 * 203 * The size (total bytes queued) counters are incremented only 204 * if we are doing B/W control. 205 */ 206 #define MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \ 207 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 208 if ((mac_srs)->srs_last != NULL) \ 209 (mac_srs)->srs_last->b_next = (head); \ 210 else \ 211 (mac_srs)->srs_first = (head); \ 212 (mac_srs)->srs_last = (tail); \ 213 (mac_srs)->srs_count += count; \ 214 } 215 216 #define MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \ 217 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \ 218 \ 219 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \ 220 srs_rx->sr_poll_pkt_cnt += count; \ 221 ASSERT(srs_rx->sr_poll_pkt_cnt > 0); \ 222 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \ 223 (mac_srs)->srs_size += (sz); \ 224 mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock); \ 225 (mac_srs)->srs_bw->mac_bw_sz += (sz); \ 226 mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock); \ 227 } \ 228 } 229 230 #define MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \ 231 mac_srs->srs_state |= SRS_ENQUEUED; \ 232 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \ 233 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \ 234 (mac_srs)->srs_size += (sz); \ 235 (mac_srs)->srs_bw->mac_bw_sz += (sz); \ 236 } \ 237 } 238 239 /* 240 * Turn polling on routines 241 */ 242 #define MAC_SRS_POLLING_ON(mac_srs) { \ 243 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 244 if (((mac_srs)->srs_state & \ 245 (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) { \ 246 (mac_srs)->srs_state |= SRS_POLLING; \ 247 (void) mac_hwring_disable_intr((mac_ring_handle_t) \ 248 (mac_srs)->srs_ring); \ 249 (mac_srs)->srs_rx.sr_poll_on++; \ 250 } \ 251 } 252 253 #define MAC_SRS_WORKER_POLLING_ON(mac_srs) { \ 254 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 255 if (((mac_srs)->srs_state & \ 256 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == \ 257 (SRS_POLLING_CAPAB|SRS_WORKER)) { \ 258 (mac_srs)->srs_state |= SRS_POLLING; \ 259 (void) mac_hwring_disable_intr((mac_ring_handle_t) \ 260 (mac_srs)->srs_ring); \ 261 (mac_srs)->srs_rx.sr_worker_poll_on++; \ 262 } \ 263 } 264 265 /* 266 * MAC_SRS_POLL_RING 267 * 268 * Signal the SRS poll thread to poll the underlying H/W ring 269 * provided it wasn't already polling (SRS_GET_PKTS was set). 270 * 271 * Poll thread gets to run only from mac_rx_srs_drain() and only 272 * if the drain was being done by the worker thread. 273 */ 274 #define MAC_SRS_POLL_RING(mac_srs) { \ 275 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \ 276 \ 277 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 278 srs_rx->sr_poll_thr_sig++; \ 279 if (((mac_srs)->srs_state & \ 280 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) == \ 281 (SRS_WORKER|SRS_POLLING_CAPAB)) { \ 282 (mac_srs)->srs_state |= SRS_GET_PKTS; \ 283 cv_signal(&(mac_srs)->srs_cv); \ 284 } else { \ 285 srs_rx->sr_poll_thr_busy++; \ 286 } \ 287 } 288 289 /* 290 * MAC_SRS_CHECK_BW_CONTROL 291 * 292 * Check to see if next tick has started so we can reset the 293 * SRS_BW_ENFORCED flag and allow more packets to come in the 294 * system. 295 */ 296 #define MAC_SRS_CHECK_BW_CONTROL(mac_srs) { \ 297 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 298 ASSERT(((mac_srs)->srs_type & SRST_TX) || \ 299 MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock)); \ 300 clock_t now = ddi_get_lbolt(); \ 301 if ((mac_srs)->srs_bw->mac_bw_curr_time != now) { \ 302 (mac_srs)->srs_bw->mac_bw_curr_time = now; \ 303 (mac_srs)->srs_bw->mac_bw_used = 0; \ 304 if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED) \ 305 (mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \ 306 } \ 307 } 308 309 /* 310 * MAC_SRS_WORKER_WAKEUP 311 * 312 * Wake up the SRS worker thread to process the queue as long as 313 * no one else is processing the queue. If we are optimizing for 314 * latency, we wake up the worker thread immediately or else we 315 * wait mac_srs_worker_wakeup_ticks before worker thread gets 316 * woken up. 317 */ 318 int mac_srs_worker_wakeup_ticks = 0; 319 #define MAC_SRS_WORKER_WAKEUP(mac_srs) { \ 320 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \ 321 if (!((mac_srs)->srs_state & SRS_PROC) && \ 322 (mac_srs)->srs_tid == NULL) { \ 323 if (((mac_srs)->srs_state & SRS_LATENCY_OPT) || \ 324 (mac_srs_worker_wakeup_ticks == 0)) \ 325 cv_signal(&(mac_srs)->srs_async); \ 326 else \ 327 (mac_srs)->srs_tid = \ 328 timeout(mac_srs_fire, (mac_srs), \ 329 mac_srs_worker_wakeup_ticks); \ 330 } \ 331 } 332 333 #define TX_BANDWIDTH_MODE(mac_srs) \ 334 ((mac_srs)->srs_tx.st_mode == SRS_TX_BW || \ 335 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT || \ 336 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_AGGR) 337 338 #define TX_SRS_TO_SOFT_RING(mac_srs, head, hint) { \ 339 if (tx_mode == SRS_TX_BW_FANOUT) \ 340 (void) mac_tx_fanout_mode(mac_srs, head, hint, 0, NULL);\ 341 else \ 342 (void) mac_tx_aggr_mode(mac_srs, head, hint, 0, NULL); \ 343 } 344 345 /* 346 * MAC_TX_SRS_BLOCK 347 * 348 * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED 349 * will be set only if srs_tx_woken_up is FALSE. If 350 * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived 351 * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to 352 * attempt to transmit again and not setting SRS_TX_BLOCKED does 353 * that. 354 */ 355 #define MAC_TX_SRS_BLOCK(srs, mp) { \ 356 ASSERT(MUTEX_HELD(&(srs)->srs_lock)); \ 357 if ((srs)->srs_tx.st_woken_up) { \ 358 (srs)->srs_tx.st_woken_up = B_FALSE; \ 359 } else { \ 360 ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED)); \ 361 (srs)->srs_state |= SRS_TX_BLOCKED; \ 362 (srs)->srs_tx.st_stat.mts_blockcnt++; \ 363 } \ 364 } 365 366 /* 367 * MAC_TX_SRS_TEST_HIWAT 368 * 369 * Called before queueing a packet onto Tx SRS to test and set 370 * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat. 371 */ 372 #define MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) { \ 373 boolean_t enqueue = 1; \ 374 \ 375 if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) { \ 376 /* \ 377 * flow-controlled. Store srs in cookie so that it \ 378 * can be returned as mac_tx_cookie_t to client \ 379 */ \ 380 (srs)->srs_state |= SRS_TX_HIWAT; \ 381 cookie = (mac_tx_cookie_t)srs; \ 382 (srs)->srs_tx.st_hiwat_cnt++; \ 383 if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) { \ 384 /* increment freed stats */ \ 385 (srs)->srs_tx.st_stat.mts_sdrops += cnt; \ 386 /* \ 387 * b_prev may be set to the fanout hint \ 388 * hence can't use freemsg directly \ 389 */ \ 390 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); \ 391 DTRACE_PROBE1(tx_queued_hiwat, \ 392 mac_soft_ring_set_t *, srs); \ 393 enqueue = 0; \ 394 } \ 395 } \ 396 if (enqueue) \ 397 MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz); \ 398 } 399 400 /* Some utility macros */ 401 #define MAC_SRS_BW_LOCK(srs) \ 402 if (!(srs->srs_type & SRST_TX)) \ 403 mutex_enter(&srs->srs_bw->mac_bw_lock); 404 405 #define MAC_SRS_BW_UNLOCK(srs) \ 406 if (!(srs->srs_type & SRST_TX)) \ 407 mutex_exit(&srs->srs_bw->mac_bw_lock); 408 409 #define MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) { \ 410 mac_pkt_drop(NULL, NULL, mp, B_FALSE); \ 411 /* increment freed stats */ \ 412 mac_srs->srs_tx.st_stat.mts_sdrops++; \ 413 cookie = (mac_tx_cookie_t)srs; \ 414 } 415 416 #define MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) { \ 417 mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT; \ 418 cookie = (mac_tx_cookie_t)srs; \ 419 *ret_mp = mp_chain; \ 420 } 421 422 /* 423 * MAC_RX_SRS_TOODEEP 424 * 425 * Macro called as part of receive-side processing to determine if handling 426 * can occur in situ (in the interrupt thread) or if it should be left to a 427 * worker thread. Note that the constant used to make this determination is 428 * not entirely made-up, and is a result of some emprical validation. That 429 * said, the constant is left as a static variable to allow it to be 430 * dynamically tuned in the field if and as needed. 431 */ 432 static uintptr_t mac_rx_srs_stack_needed = 10240; 433 static uint_t mac_rx_srs_stack_toodeep; 434 435 #ifndef STACK_GROWTH_DOWN 436 #error Downward stack growth assumed. 437 #endif 438 439 #define MAC_RX_SRS_TOODEEP() (STACK_BIAS + (uintptr_t)getfp() - \ 440 (uintptr_t)curthread->t_stkbase < mac_rx_srs_stack_needed && \ 441 ++mac_rx_srs_stack_toodeep) 442 443 444 /* 445 * Drop the rx packet and advance to the next one in the chain. 446 */ 447 static void 448 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp) 449 { 450 mac_srs_rx_t *srs_rx = &srs->srs_rx; 451 452 ASSERT(mp->b_next == NULL); 453 mutex_enter(&srs->srs_lock); 454 MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1); 455 MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp)); 456 mutex_exit(&srs->srs_lock); 457 458 srs_rx->sr_stat.mrs_sdrops++; 459 freemsg(mp); 460 } 461 462 /* DATAPATH RUNTIME ROUTINES */ 463 464 /* 465 * mac_srs_fire 466 * 467 * Timer callback routine for waking up the SRS worker thread. 468 */ 469 static void 470 mac_srs_fire(void *arg) 471 { 472 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg; 473 474 mutex_enter(&mac_srs->srs_lock); 475 if (mac_srs->srs_tid == 0) { 476 mutex_exit(&mac_srs->srs_lock); 477 return; 478 } 479 480 mac_srs->srs_tid = 0; 481 if (!(mac_srs->srs_state & SRS_PROC)) 482 cv_signal(&mac_srs->srs_async); 483 484 mutex_exit(&mac_srs->srs_lock); 485 } 486 487 /* 488 * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack, 489 * and it is used on the TX path. 490 */ 491 #define HASH_HINT(hint) \ 492 ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8)) 493 494 495 /* 496 * hash based on the src address and the port information. 497 */ 498 #define HASH_ADDR(src, ports) \ 499 (ntohl((src)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^ \ 500 ((ports) >> 8) ^ (ports)) 501 502 #define COMPUTE_INDEX(key, sz) (key % sz) 503 504 #define FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) { \ 505 if ((tail) != NULL) { \ 506 ASSERT((tail)->b_next == NULL); \ 507 (tail)->b_next = (mp); \ 508 } else { \ 509 ASSERT((head) == NULL); \ 510 (head) = (mp); \ 511 } \ 512 (tail) = (mp); \ 513 (cnt)++; \ 514 if ((bw_ctl)) \ 515 (sz) += (sz0); \ 516 } 517 518 #define MAC_FANOUT_DEFAULT 0 519 #define MAC_FANOUT_RND_ROBIN 1 520 int mac_fanout_type = MAC_FANOUT_DEFAULT; 521 522 #define MAX_SR_TYPES 3 523 /* fanout types for port based hashing */ 524 enum pkt_type { 525 V4_TCP = 0, 526 V4_UDP, 527 OTH, 528 UNDEF 529 }; 530 531 /* 532 * In general we do port based hashing to spread traffic over different 533 * softrings. The below tunable allows to override that behavior. Setting it 534 * to B_TRUE allows to do a fanout based on src ipv6 address. This behavior 535 * is also the applicable to ipv6 packets carrying multiple optional headers 536 * and other uncommon packet types. 537 */ 538 boolean_t mac_src_ipv6_fanout = B_FALSE; 539 540 /* 541 * Pair of local and remote ports in the transport header 542 */ 543 #define PORTS_SIZE 4 544 545 /* 546 * mac_rx_srs_proto_fanout 547 * 548 * This routine delivers packets destined to an SRS into one of the 549 * protocol soft rings. 550 * 551 * Given a chain of packets we need to split it up into multiple sub chains 552 * destined into TCP, UDP or OTH soft ring. Instead of entering 553 * the soft ring one packet at a time, we want to enter it in the form of a 554 * chain otherwise we get this start/stop behaviour where the worker thread 555 * goes to sleep and then next packets comes in forcing it to wake up etc. 556 */ 557 static void 558 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) 559 { 560 struct ether_header *ehp; 561 struct ether_vlan_header *evhp; 562 uint32_t sap; 563 ipha_t *ipha; 564 uint8_t *dstaddr; 565 size_t hdrsize; 566 mblk_t *mp; 567 mblk_t *headmp[MAX_SR_TYPES]; 568 mblk_t *tailmp[MAX_SR_TYPES]; 569 int cnt[MAX_SR_TYPES]; 570 size_t sz[MAX_SR_TYPES]; 571 size_t sz1; 572 boolean_t bw_ctl; 573 boolean_t hw_classified; 574 boolean_t dls_bypass; 575 boolean_t is_ether; 576 boolean_t is_unicast; 577 enum pkt_type type; 578 mac_client_impl_t *mcip = mac_srs->srs_mcip; 579 580 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER); 581 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0); 582 583 /* 584 * If we don't have a Rx ring, S/W classification would have done 585 * its job and its a packet meant for us. If we were polling on 586 * the default ring (i.e. there was a ring assigned to this SRS), 587 * then we need to make sure that the mac address really belongs 588 * to us. 589 */ 590 hw_classified = mac_srs->srs_ring != NULL && 591 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER; 592 593 /* 594 * Special clients (eg. VLAN, non ether, etc) need DLS 595 * processing in the Rx path. SRST_DLS_BYPASS will be clear for 596 * such SRSs. Another way of disabling bypass is to set the 597 * MCIS_RX_BYPASS_DISABLE flag. 598 */ 599 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) && 600 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0); 601 602 bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *)); 603 bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *)); 604 bzero(cnt, MAX_SR_TYPES * sizeof (int)); 605 bzero(sz, MAX_SR_TYPES * sizeof (size_t)); 606 607 /* 608 * We got a chain from SRS that we need to send to the soft rings. 609 * Since squeues for TCP & IPv4 sap poll their soft rings (for 610 * performance reasons), we need to separate out v4_tcp, v4_udp 611 * and the rest goes in other. 612 */ 613 while (head != NULL) { 614 mp = head; 615 head = head->b_next; 616 mp->b_next = NULL; 617 618 type = OTH; 619 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp); 620 621 if (is_ether) { 622 /* 623 * At this point we can be sure the packet at least 624 * has an ether header. 625 */ 626 if (sz1 < sizeof (struct ether_header)) { 627 mac_rx_drop_pkt(mac_srs, mp); 628 continue; 629 } 630 ehp = (struct ether_header *)mp->b_rptr; 631 632 /* 633 * Determine if this is a VLAN or non-VLAN packet. 634 */ 635 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) { 636 evhp = (struct ether_vlan_header *)mp->b_rptr; 637 sap = ntohs(evhp->ether_type); 638 hdrsize = sizeof (struct ether_vlan_header); 639 /* 640 * Check if the VID of the packet, if any, 641 * belongs to this client. 642 */ 643 if (!mac_client_check_flow_vid(mcip, 644 VLAN_ID(ntohs(evhp->ether_tci)))) { 645 mac_rx_drop_pkt(mac_srs, mp); 646 continue; 647 } 648 } else { 649 hdrsize = sizeof (struct ether_header); 650 } 651 is_unicast = 652 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0); 653 dstaddr = (uint8_t *)&ehp->ether_dhost; 654 } else { 655 mac_header_info_t mhi; 656 657 if (mac_header_info((mac_handle_t)mcip->mci_mip, 658 mp, &mhi) != 0) { 659 mac_rx_drop_pkt(mac_srs, mp); 660 continue; 661 } 662 hdrsize = mhi.mhi_hdrsize; 663 sap = mhi.mhi_bindsap; 664 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST); 665 dstaddr = (uint8_t *)mhi.mhi_daddr; 666 } 667 668 if (!dls_bypass) { 669 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], 670 cnt[type], bw_ctl, sz[type], sz1, mp); 671 continue; 672 } 673 674 if (sap == ETHERTYPE_IP) { 675 /* 676 * If we are H/W classified, but we have promisc 677 * on, then we need to check for the unicast address. 678 */ 679 if (hw_classified && mcip->mci_promisc_list != NULL) { 680 mac_address_t *map; 681 682 rw_enter(&mcip->mci_rw_lock, RW_READER); 683 map = mcip->mci_unicast; 684 if (bcmp(dstaddr, map->ma_addr, 685 map->ma_len) == 0) 686 type = UNDEF; 687 rw_exit(&mcip->mci_rw_lock); 688 } else if (is_unicast) { 689 type = UNDEF; 690 } 691 } 692 693 /* 694 * This needs to become a contract with the driver for 695 * the fast path. 696 * 697 * In the normal case the packet will have at least the L2 698 * header and the IP + Transport header in the same mblk. 699 * This is usually the case when the NIC driver sends up 700 * the packet. This is also true when the stack generates 701 * a packet that is looped back and when the stack uses the 702 * fastpath mechanism. The normal case is optimized for 703 * performance and may bypass DLS. All other cases go through 704 * the 'OTH' type path without DLS bypass. 705 */ 706 707 ipha = (ipha_t *)(mp->b_rptr + hdrsize); 708 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) 709 type = OTH; 710 711 if (type == OTH) { 712 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], 713 cnt[type], bw_ctl, sz[type], sz1, mp); 714 continue; 715 } 716 717 ASSERT(type == UNDEF); 718 /* 719 * We look for at least 4 bytes past the IP header to get 720 * the port information. If we get an IP fragment, we don't 721 * have the port information, and we use just the protocol 722 * information. 723 */ 724 switch (ipha->ipha_protocol) { 725 case IPPROTO_TCP: 726 type = V4_TCP; 727 mp->b_rptr += hdrsize; 728 break; 729 case IPPROTO_UDP: 730 type = V4_UDP; 731 mp->b_rptr += hdrsize; 732 break; 733 default: 734 type = OTH; 735 break; 736 } 737 738 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type], 739 bw_ctl, sz[type], sz1, mp); 740 } 741 742 for (type = V4_TCP; type < UNDEF; type++) { 743 if (headmp[type] != NULL) { 744 mac_soft_ring_t *softring; 745 746 ASSERT(tailmp[type]->b_next == NULL); 747 switch (type) { 748 case V4_TCP: 749 softring = mac_srs->srs_tcp_soft_rings[0]; 750 break; 751 case V4_UDP: 752 softring = mac_srs->srs_udp_soft_rings[0]; 753 break; 754 case OTH: 755 softring = mac_srs->srs_oth_soft_rings[0]; 756 } 757 mac_rx_soft_ring_process(mcip, softring, 758 headmp[type], tailmp[type], cnt[type], sz[type]); 759 } 760 } 761 } 762 763 int fanout_unalligned = 0; 764 765 /* 766 * mac_rx_srs_long_fanout 767 * 768 * The fanout routine for IPv6 769 */ 770 static int 771 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp, 772 uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx) 773 { 774 ip6_t *ip6h; 775 uint8_t *whereptr; 776 uint_t hash; 777 uint16_t remlen; 778 uint8_t nexthdr; 779 uint16_t hdr_len; 780 781 if (sap == ETHERTYPE_IPV6) { 782 boolean_t modifiable = B_TRUE; 783 784 ASSERT(MBLKL(mp) >= hdrsize); 785 786 ip6h = (ip6_t *)(mp->b_rptr + hdrsize); 787 if ((unsigned char *)ip6h == mp->b_wptr) { 788 /* 789 * The first mblk_t only includes the mac header. 790 * Note that it is safe to change the mp pointer here, 791 * as the subsequent operation does not assume mp 792 * points to the start of the mac header. 793 */ 794 mp = mp->b_cont; 795 796 /* 797 * Make sure ip6h holds the full ip6_t structure. 798 */ 799 if (mp == NULL) 800 return (-1); 801 802 if (MBLKL(mp) < IPV6_HDR_LEN) { 803 modifiable = (DB_REF(mp) == 1); 804 805 if (modifiable && 806 !pullupmsg(mp, IPV6_HDR_LEN)) { 807 return (-1); 808 } 809 } 810 811 ip6h = (ip6_t *)mp->b_rptr; 812 } 813 814 if (!modifiable || !(OK_32PTR((char *)ip6h)) || 815 ((unsigned char *)ip6h + IPV6_HDR_LEN > mp->b_wptr)) { 816 /* 817 * If either ip6h is not alligned, or ip6h does not 818 * hold the complete ip6_t structure (a pullupmsg() 819 * is not an option since it would result in an 820 * unalligned ip6h), fanout to the default ring. Note 821 * that this may cause packets reordering. 822 */ 823 *indx = 0; 824 *type = OTH; 825 fanout_unalligned++; 826 return (0); 827 } 828 829 remlen = ntohs(ip6h->ip6_plen); 830 nexthdr = ip6h->ip6_nxt; 831 832 if (remlen < MIN_EHDR_LEN) 833 return (-1); 834 /* 835 * Do src based fanout if below tunable is set to B_TRUE or 836 * when mac_ip_hdr_length_v6() fails because of malformed 837 * packets or because mblk's need to be concatenated using 838 * pullupmsg(). 839 */ 840 if (mac_src_ipv6_fanout || !mac_ip_hdr_length_v6(ip6h, 841 mp->b_wptr, &hdr_len, &nexthdr, NULL)) { 842 goto src_based_fanout; 843 } 844 whereptr = (uint8_t *)ip6h + hdr_len; 845 846 /* If the transport is one of below, we do port based fanout */ 847 switch (nexthdr) { 848 case IPPROTO_TCP: 849 case IPPROTO_UDP: 850 case IPPROTO_SCTP: 851 case IPPROTO_ESP: 852 /* 853 * If the ports in the transport header is not part of 854 * the mblk, do src_based_fanout, instead of calling 855 * pullupmsg(). 856 */ 857 if (mp->b_cont != NULL && 858 whereptr + PORTS_SIZE > mp->b_wptr) { 859 goto src_based_fanout; 860 } 861 break; 862 default: 863 break; 864 } 865 866 switch (nexthdr) { 867 case IPPROTO_TCP: 868 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src), 869 *(uint32_t *)whereptr); 870 *indx = COMPUTE_INDEX(hash, 871 mac_srs->srs_tcp_ring_count); 872 *type = OTH; 873 break; 874 875 case IPPROTO_UDP: 876 case IPPROTO_SCTP: 877 case IPPROTO_ESP: 878 if (mac_fanout_type == MAC_FANOUT_DEFAULT) { 879 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src), 880 *(uint32_t *)whereptr); 881 *indx = COMPUTE_INDEX(hash, 882 mac_srs->srs_udp_ring_count); 883 } else { 884 *indx = mac_srs->srs_ind % 885 mac_srs->srs_udp_ring_count; 886 mac_srs->srs_ind++; 887 } 888 *type = OTH; 889 break; 890 891 /* For all other protocol, do source based fanout */ 892 default: 893 goto src_based_fanout; 894 } 895 } else { 896 *indx = 0; 897 *type = OTH; 898 } 899 return (0); 900 901 src_based_fanout: 902 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src), (uint32_t)0); 903 *indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count); 904 *type = OTH; 905 return (0); 906 } 907 908 /* 909 * mac_rx_srs_fanout 910 * 911 * This routine delivers packets destined to an SRS into a soft ring member 912 * of the set. 913 * 914 * Given a chain of packets we need to split it up into multiple sub chains 915 * destined for one of the TCP, UDP or OTH soft rings. Instead of entering 916 * the soft ring one packet at a time, we want to enter it in the form of a 917 * chain otherwise we get this start/stop behaviour where the worker thread 918 * goes to sleep and then next packets comes in forcing it to wake up etc. 919 * 920 * Note: 921 * Since we know what is the maximum fanout possible, we create a 2D array 922 * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz 923 * variables so that we can enter the softrings with chain. We need the 924 * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc 925 * for each packet would be expensive). If we ever want to have the 926 * ability to have unlimited fanout, we should probably declare a head, 927 * tail, cnt, sz with each soft ring (a data struct which contains a softring 928 * along with these members) and create an array of this uber struct so we 929 * don't have to do kmem_alloc. 930 */ 931 int fanout_oth1 = 0; 932 int fanout_oth2 = 0; 933 int fanout_oth3 = 0; 934 int fanout_oth4 = 0; 935 int fanout_oth5 = 0; 936 937 static void 938 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) 939 { 940 struct ether_header *ehp; 941 struct ether_vlan_header *evhp; 942 uint32_t sap; 943 ipha_t *ipha; 944 uint8_t *dstaddr; 945 uint_t indx; 946 size_t ports_offset; 947 size_t ipha_len; 948 size_t hdrsize; 949 uint_t hash; 950 mblk_t *mp; 951 mblk_t *headmp[MAX_SR_TYPES][MAX_SR_FANOUT]; 952 mblk_t *tailmp[MAX_SR_TYPES][MAX_SR_FANOUT]; 953 int cnt[MAX_SR_TYPES][MAX_SR_FANOUT]; 954 size_t sz[MAX_SR_TYPES][MAX_SR_FANOUT]; 955 size_t sz1; 956 boolean_t bw_ctl; 957 boolean_t hw_classified; 958 boolean_t dls_bypass; 959 boolean_t is_ether; 960 boolean_t is_unicast; 961 int fanout_cnt; 962 enum pkt_type type; 963 mac_client_impl_t *mcip = mac_srs->srs_mcip; 964 965 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER); 966 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0); 967 968 /* 969 * If we don't have a Rx ring, S/W classification would have done 970 * its job and its a packet meant for us. If we were polling on 971 * the default ring (i.e. there was a ring assigned to this SRS), 972 * then we need to make sure that the mac address really belongs 973 * to us. 974 */ 975 hw_classified = mac_srs->srs_ring != NULL && 976 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER; 977 978 /* 979 * Special clients (eg. VLAN, non ether, etc) need DLS 980 * processing in the Rx path. SRST_DLS_BYPASS will be clear for 981 * such SRSs. Another way of disabling bypass is to set the 982 * MCIS_RX_BYPASS_DISABLE flag. 983 */ 984 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) && 985 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0); 986 987 /* 988 * Since the softrings are never destroyed and we always 989 * create equal number of softrings for TCP, UDP and rest, 990 * its OK to check one of them for count and use it without 991 * any lock. In future, if soft rings get destroyed because 992 * of reduction in fanout, we will need to ensure that happens 993 * behind the SRS_PROC. 994 */ 995 fanout_cnt = mac_srs->srs_tcp_ring_count; 996 997 bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *)); 998 bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *)); 999 bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int)); 1000 bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t)); 1001 1002 /* 1003 * We got a chain from SRS that we need to send to the soft rings. 1004 * Since squeues for TCP & IPv4 sap poll their soft rings (for 1005 * performance reasons), we need to separate out v4_tcp, v4_udp 1006 * and the rest goes in other. 1007 */ 1008 while (head != NULL) { 1009 mp = head; 1010 head = head->b_next; 1011 mp->b_next = NULL; 1012 1013 type = OTH; 1014 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp); 1015 1016 if (is_ether) { 1017 /* 1018 * At this point we can be sure the packet at least 1019 * has an ether header. 1020 */ 1021 if (sz1 < sizeof (struct ether_header)) { 1022 mac_rx_drop_pkt(mac_srs, mp); 1023 continue; 1024 } 1025 ehp = (struct ether_header *)mp->b_rptr; 1026 1027 /* 1028 * Determine if this is a VLAN or non-VLAN packet. 1029 */ 1030 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) { 1031 evhp = (struct ether_vlan_header *)mp->b_rptr; 1032 sap = ntohs(evhp->ether_type); 1033 hdrsize = sizeof (struct ether_vlan_header); 1034 /* 1035 * Check if the VID of the packet, if any, 1036 * belongs to this client. 1037 */ 1038 if (!mac_client_check_flow_vid(mcip, 1039 VLAN_ID(ntohs(evhp->ether_tci)))) { 1040 mac_rx_drop_pkt(mac_srs, mp); 1041 continue; 1042 } 1043 } else { 1044 hdrsize = sizeof (struct ether_header); 1045 } 1046 is_unicast = 1047 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0); 1048 dstaddr = (uint8_t *)&ehp->ether_dhost; 1049 } else { 1050 mac_header_info_t mhi; 1051 1052 if (mac_header_info((mac_handle_t)mcip->mci_mip, 1053 mp, &mhi) != 0) { 1054 mac_rx_drop_pkt(mac_srs, mp); 1055 continue; 1056 } 1057 hdrsize = mhi.mhi_hdrsize; 1058 sap = mhi.mhi_bindsap; 1059 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST); 1060 dstaddr = (uint8_t *)mhi.mhi_daddr; 1061 } 1062 1063 if (!dls_bypass) { 1064 if (mac_rx_srs_long_fanout(mac_srs, mp, sap, 1065 hdrsize, &type, &indx) == -1) { 1066 mac_rx_drop_pkt(mac_srs, mp); 1067 continue; 1068 } 1069 1070 FANOUT_ENQUEUE_MP(headmp[type][indx], 1071 tailmp[type][indx], cnt[type][indx], bw_ctl, 1072 sz[type][indx], sz1, mp); 1073 continue; 1074 } 1075 1076 1077 /* 1078 * If we are using the default Rx ring where H/W or S/W 1079 * classification has not happened, we need to verify if 1080 * this unicast packet really belongs to us. 1081 */ 1082 if (sap == ETHERTYPE_IP) { 1083 /* 1084 * If we are H/W classified, but we have promisc 1085 * on, then we need to check for the unicast address. 1086 */ 1087 if (hw_classified && mcip->mci_promisc_list != NULL) { 1088 mac_address_t *map; 1089 1090 rw_enter(&mcip->mci_rw_lock, RW_READER); 1091 map = mcip->mci_unicast; 1092 if (bcmp(dstaddr, map->ma_addr, 1093 map->ma_len) == 0) 1094 type = UNDEF; 1095 rw_exit(&mcip->mci_rw_lock); 1096 } else if (is_unicast) { 1097 type = UNDEF; 1098 } 1099 } 1100 1101 /* 1102 * This needs to become a contract with the driver for 1103 * the fast path. 1104 */ 1105 1106 ipha = (ipha_t *)(mp->b_rptr + hdrsize); 1107 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) { 1108 type = OTH; 1109 fanout_oth1++; 1110 } 1111 1112 if (type != OTH) { 1113 uint16_t frag_offset_flags; 1114 1115 switch (ipha->ipha_protocol) { 1116 case IPPROTO_TCP: 1117 case IPPROTO_UDP: 1118 case IPPROTO_SCTP: 1119 case IPPROTO_ESP: 1120 ipha_len = IPH_HDR_LENGTH(ipha); 1121 if ((uchar_t *)ipha + ipha_len + PORTS_SIZE > 1122 mp->b_wptr) { 1123 type = OTH; 1124 break; 1125 } 1126 frag_offset_flags = 1127 ntohs(ipha->ipha_fragment_offset_and_flags); 1128 if ((frag_offset_flags & 1129 (IPH_MF | IPH_OFFSET)) != 0) { 1130 type = OTH; 1131 fanout_oth3++; 1132 break; 1133 } 1134 ports_offset = hdrsize + ipha_len; 1135 break; 1136 default: 1137 type = OTH; 1138 fanout_oth4++; 1139 break; 1140 } 1141 } 1142 1143 if (type == OTH) { 1144 if (mac_rx_srs_long_fanout(mac_srs, mp, sap, 1145 hdrsize, &type, &indx) == -1) { 1146 mac_rx_drop_pkt(mac_srs, mp); 1147 continue; 1148 } 1149 1150 FANOUT_ENQUEUE_MP(headmp[type][indx], 1151 tailmp[type][indx], cnt[type][indx], bw_ctl, 1152 sz[type][indx], sz1, mp); 1153 continue; 1154 } 1155 1156 ASSERT(type == UNDEF); 1157 1158 /* 1159 * XXX-Sunay: We should hold srs_lock since ring_count 1160 * below can change. But if we are always called from 1161 * mac_rx_srs_drain and SRS_PROC is set, then we can 1162 * enforce that ring_count can't be changed i.e. 1163 * to change fanout type or ring count, the calling 1164 * thread needs to be behind SRS_PROC. 1165 */ 1166 switch (ipha->ipha_protocol) { 1167 case IPPROTO_TCP: 1168 /* 1169 * Note that for ESP, we fanout on SPI and it is at the 1170 * same offset as the 2x16-bit ports. So it is clumped 1171 * along with TCP, UDP and SCTP. 1172 */ 1173 hash = HASH_ADDR(ipha->ipha_src, 1174 *(uint32_t *)(mp->b_rptr + ports_offset)); 1175 indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count); 1176 type = V4_TCP; 1177 mp->b_rptr += hdrsize; 1178 break; 1179 case IPPROTO_UDP: 1180 case IPPROTO_SCTP: 1181 case IPPROTO_ESP: 1182 if (mac_fanout_type == MAC_FANOUT_DEFAULT) { 1183 hash = HASH_ADDR(ipha->ipha_src, 1184 *(uint32_t *)(mp->b_rptr + ports_offset)); 1185 indx = COMPUTE_INDEX(hash, 1186 mac_srs->srs_udp_ring_count); 1187 } else { 1188 indx = mac_srs->srs_ind % 1189 mac_srs->srs_udp_ring_count; 1190 mac_srs->srs_ind++; 1191 } 1192 type = V4_UDP; 1193 mp->b_rptr += hdrsize; 1194 break; 1195 default: 1196 indx = 0; 1197 type = OTH; 1198 } 1199 1200 FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx], 1201 cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp); 1202 } 1203 1204 for (type = V4_TCP; type < UNDEF; type++) { 1205 int i; 1206 1207 for (i = 0; i < fanout_cnt; i++) { 1208 if (headmp[type][i] != NULL) { 1209 mac_soft_ring_t *softring; 1210 1211 ASSERT(tailmp[type][i]->b_next == NULL); 1212 switch (type) { 1213 case V4_TCP: 1214 softring = 1215 mac_srs->srs_tcp_soft_rings[i]; 1216 break; 1217 case V4_UDP: 1218 softring = 1219 mac_srs->srs_udp_soft_rings[i]; 1220 break; 1221 case OTH: 1222 softring = 1223 mac_srs->srs_oth_soft_rings[i]; 1224 break; 1225 } 1226 mac_rx_soft_ring_process(mcip, 1227 softring, headmp[type][i], tailmp[type][i], 1228 cnt[type][i], sz[type][i]); 1229 } 1230 } 1231 } 1232 } 1233 1234 #define SRS_BYTES_TO_PICKUP 150000 1235 ssize_t max_bytes_to_pickup = SRS_BYTES_TO_PICKUP; 1236 1237 /* 1238 * mac_rx_srs_poll_ring 1239 * 1240 * This SRS Poll thread uses this routine to poll the underlying hardware 1241 * Rx ring to get a chain of packets. It can inline process that chain 1242 * if mac_latency_optimize is set (default) or signal the SRS worker thread 1243 * to do the remaining processing. 1244 * 1245 * Since packets come in the system via interrupt or poll path, we also 1246 * update the stats and deal with promiscous clients here. 1247 */ 1248 void 1249 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs) 1250 { 1251 kmutex_t *lock = &mac_srs->srs_lock; 1252 kcondvar_t *async = &mac_srs->srs_cv; 1253 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 1254 mblk_t *head, *tail, *mp; 1255 callb_cpr_t cprinfo; 1256 ssize_t bytes_to_pickup; 1257 size_t sz; 1258 int count; 1259 mac_client_impl_t *smcip; 1260 1261 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll"); 1262 mutex_enter(lock); 1263 1264 start: 1265 for (;;) { 1266 if (mac_srs->srs_state & SRS_PAUSE) 1267 goto done; 1268 1269 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1270 cv_wait(async, lock); 1271 CALLB_CPR_SAFE_END(&cprinfo, lock); 1272 1273 if (mac_srs->srs_state & SRS_PAUSE) 1274 goto done; 1275 1276 check_again: 1277 if (mac_srs->srs_type & SRST_BW_CONTROL) { 1278 /* 1279 * We pick as many bytes as we are allowed to queue. 1280 * Its possible that we will exceed the total 1281 * packets queued in case this SRS is part of the 1282 * Rx ring group since > 1 poll thread can be pulling 1283 * upto the max allowed packets at the same time 1284 * but that should be OK. 1285 */ 1286 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 1287 bytes_to_pickup = 1288 mac_srs->srs_bw->mac_bw_drop_threshold - 1289 mac_srs->srs_bw->mac_bw_sz; 1290 /* 1291 * We shouldn't have been signalled if we 1292 * have 0 or less bytes to pick but since 1293 * some of the bytes accounting is driver 1294 * dependant, we do the safety check. 1295 */ 1296 if (bytes_to_pickup < 0) 1297 bytes_to_pickup = 0; 1298 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1299 } else { 1300 /* 1301 * ToDO: Need to change the polling API 1302 * to add a packet count and a flag which 1303 * tells the driver whether we want packets 1304 * based on a count, or bytes, or all the 1305 * packets queued in the driver/HW. This 1306 * way, we never have to check the limits 1307 * on poll path. We truly let only as many 1308 * packets enter the system as we are willing 1309 * to process or queue. 1310 * 1311 * Something along the lines of 1312 * pkts_to_pickup = mac_soft_ring_max_q_cnt - 1313 * mac_srs->srs_poll_pkt_cnt 1314 */ 1315 1316 /* 1317 * Since we are not doing B/W control, pick 1318 * as many packets as allowed. 1319 */ 1320 bytes_to_pickup = max_bytes_to_pickup; 1321 } 1322 1323 /* Poll the underlying Hardware */ 1324 mutex_exit(lock); 1325 head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup); 1326 mutex_enter(lock); 1327 1328 ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) == 1329 SRS_POLL_THR_OWNER); 1330 1331 mp = tail = head; 1332 count = 0; 1333 sz = 0; 1334 while (mp != NULL) { 1335 tail = mp; 1336 sz += msgdsize(mp); 1337 mp = mp->b_next; 1338 count++; 1339 } 1340 1341 if (head != NULL) { 1342 tail->b_next = NULL; 1343 smcip = mac_srs->srs_mcip; 1344 1345 SRS_RX_STAT_UPDATE(mac_srs, pollbytes, sz); 1346 SRS_RX_STAT_UPDATE(mac_srs, pollcnt, count); 1347 1348 /* 1349 * If there are any promiscuous mode callbacks 1350 * defined for this MAC client, pass them a copy 1351 * if appropriate and also update the counters. 1352 */ 1353 if (smcip != NULL) { 1354 if (smcip->mci_mip->mi_promisc_list != NULL) { 1355 mutex_exit(lock); 1356 mac_promisc_dispatch(smcip->mci_mip, 1357 head, NULL); 1358 mutex_enter(lock); 1359 } 1360 } 1361 if (mac_srs->srs_type & SRST_BW_CONTROL) { 1362 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 1363 mac_srs->srs_bw->mac_bw_polled += sz; 1364 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1365 } 1366 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, 1367 count, sz); 1368 if (count <= 10) 1369 srs_rx->sr_stat.mrs_chaincntundr10++; 1370 else if (count > 10 && count <= 50) 1371 srs_rx->sr_stat.mrs_chaincnt10to50++; 1372 else 1373 srs_rx->sr_stat.mrs_chaincntover50++; 1374 } 1375 1376 /* 1377 * We are guaranteed that SRS_PROC will be set if we 1378 * are here. Also, poll thread gets to run only if 1379 * the drain was being done by a worker thread although 1380 * its possible that worker thread is still running 1381 * and poll thread was sent down to keep the pipeline 1382 * going instead of doing a complete drain and then 1383 * trying to poll the NIC. 1384 * 1385 * So we need to check SRS_WORKER flag to make sure 1386 * that the worker thread is not processing the queue 1387 * in parallel to us. The flags and conditions are 1388 * protected by the srs_lock to prevent any race. We 1389 * ensure that we don't drop the srs_lock from now 1390 * till the end and similarly we don't drop the srs_lock 1391 * in mac_rx_srs_drain() till similar condition check 1392 * are complete. The mac_rx_srs_drain() needs to ensure 1393 * that SRS_WORKER flag remains set as long as its 1394 * processing the queue. 1395 */ 1396 if (!(mac_srs->srs_state & SRS_WORKER) && 1397 (mac_srs->srs_first != NULL)) { 1398 /* 1399 * We have packets to process and worker thread 1400 * is not running. Check to see if poll thread is 1401 * allowed to process. 1402 */ 1403 if (mac_srs->srs_state & SRS_LATENCY_OPT) { 1404 mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC); 1405 if (!(mac_srs->srs_state & SRS_PAUSE) && 1406 srs_rx->sr_poll_pkt_cnt <= 1407 srs_rx->sr_lowat) { 1408 srs_rx->sr_poll_again++; 1409 goto check_again; 1410 } 1411 /* 1412 * We are already above low water mark 1413 * so stay in the polling mode but no 1414 * need to poll. Once we dip below 1415 * the polling threshold, the processing 1416 * thread (soft ring) will signal us 1417 * to poll again (MAC_UPDATE_SRS_COUNT) 1418 */ 1419 srs_rx->sr_poll_drain_no_poll++; 1420 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS); 1421 /* 1422 * In B/W control case, its possible 1423 * that the backlog built up due to 1424 * B/W limit being reached and packets 1425 * are queued only in SRS. In this case, 1426 * we should schedule worker thread 1427 * since no one else will wake us up. 1428 */ 1429 if ((mac_srs->srs_type & SRST_BW_CONTROL) && 1430 (mac_srs->srs_tid == NULL)) { 1431 mac_srs->srs_tid = 1432 timeout(mac_srs_fire, mac_srs, 1); 1433 srs_rx->sr_poll_worker_wakeup++; 1434 } 1435 } else { 1436 /* 1437 * Wakeup the worker thread for more processing. 1438 * We optimize for throughput in this case. 1439 */ 1440 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS); 1441 MAC_SRS_WORKER_WAKEUP(mac_srs); 1442 srs_rx->sr_poll_sig_worker++; 1443 } 1444 } else if ((mac_srs->srs_first == NULL) && 1445 !(mac_srs->srs_state & SRS_WORKER)) { 1446 /* 1447 * There is nothing queued in SRS and 1448 * no worker thread running. Plus we 1449 * didn't get anything from the H/W 1450 * as well (head == NULL); 1451 */ 1452 ASSERT(head == NULL); 1453 mac_srs->srs_state &= 1454 ~(SRS_PROC|SRS_GET_PKTS); 1455 1456 /* 1457 * If we have a packets in soft ring, don't allow 1458 * more packets to come into this SRS by keeping the 1459 * interrupts off but not polling the H/W. The 1460 * poll thread will get signaled as soon as 1461 * srs_poll_pkt_cnt dips below poll threshold. 1462 */ 1463 if (srs_rx->sr_poll_pkt_cnt == 0) { 1464 srs_rx->sr_poll_intr_enable++; 1465 MAC_SRS_POLLING_OFF(mac_srs); 1466 } else { 1467 /* 1468 * We know nothing is queued in SRS 1469 * since we are here after checking 1470 * srs_first is NULL. The backlog 1471 * is entirely due to packets queued 1472 * in Soft ring which will wake us up 1473 * and get the interface out of polling 1474 * mode once the backlog dips below 1475 * sr_poll_thres. 1476 */ 1477 srs_rx->sr_poll_no_poll++; 1478 } 1479 } else { 1480 /* 1481 * Worker thread is already running. 1482 * Nothing much to do. If the polling 1483 * was enabled, worker thread will deal 1484 * with that. 1485 */ 1486 mac_srs->srs_state &= ~SRS_GET_PKTS; 1487 srs_rx->sr_poll_goto_sleep++; 1488 } 1489 } 1490 done: 1491 mac_srs->srs_state |= SRS_POLL_THR_QUIESCED; 1492 cv_signal(&mac_srs->srs_async); 1493 /* 1494 * If this is a temporary quiesce then wait for the restart signal 1495 * from the srs worker. Then clear the flags and signal the srs worker 1496 * to ensure a positive handshake and go back to start. 1497 */ 1498 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART))) 1499 cv_wait(async, lock); 1500 if (mac_srs->srs_state & SRS_POLL_THR_RESTART) { 1501 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED)); 1502 mac_srs->srs_state &= 1503 ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART); 1504 cv_signal(&mac_srs->srs_async); 1505 goto start; 1506 } else { 1507 mac_srs->srs_state |= SRS_POLL_THR_EXITED; 1508 cv_signal(&mac_srs->srs_async); 1509 CALLB_CPR_EXIT(&cprinfo); 1510 thread_exit(); 1511 } 1512 } 1513 1514 /* 1515 * mac_srs_pick_chain 1516 * 1517 * In Bandwidth control case, checks how many packets can be processed 1518 * and return them in a sub chain. 1519 */ 1520 static mblk_t * 1521 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail, 1522 size_t *chain_sz, int *chain_cnt) 1523 { 1524 mblk_t *head = NULL; 1525 mblk_t *tail = NULL; 1526 size_t sz; 1527 size_t tsz = 0; 1528 int cnt = 0; 1529 mblk_t *mp; 1530 1531 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 1532 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 1533 if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <= 1534 mac_srs->srs_bw->mac_bw_limit) || 1535 (mac_srs->srs_bw->mac_bw_limit == 0)) { 1536 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1537 head = mac_srs->srs_first; 1538 mac_srs->srs_first = NULL; 1539 *chain_tail = mac_srs->srs_last; 1540 mac_srs->srs_last = NULL; 1541 *chain_sz = mac_srs->srs_size; 1542 *chain_cnt = mac_srs->srs_count; 1543 mac_srs->srs_count = 0; 1544 mac_srs->srs_size = 0; 1545 return (head); 1546 } 1547 1548 /* 1549 * Can't clear the entire backlog. 1550 * Need to find how many packets to pick 1551 */ 1552 ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock)); 1553 while ((mp = mac_srs->srs_first) != NULL) { 1554 sz = msgdsize(mp); 1555 if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) > 1556 mac_srs->srs_bw->mac_bw_limit) { 1557 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) 1558 mac_srs->srs_bw->mac_bw_state |= 1559 SRS_BW_ENFORCED; 1560 break; 1561 } 1562 1563 /* 1564 * The _size & cnt is decremented from the softrings 1565 * when they send up the packet for polling to work 1566 * properly. 1567 */ 1568 tsz += sz; 1569 cnt++; 1570 mac_srs->srs_count--; 1571 mac_srs->srs_size -= sz; 1572 if (tail != NULL) 1573 tail->b_next = mp; 1574 else 1575 head = mp; 1576 tail = mp; 1577 mac_srs->srs_first = mac_srs->srs_first->b_next; 1578 } 1579 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1580 if (mac_srs->srs_first == NULL) 1581 mac_srs->srs_last = NULL; 1582 1583 if (tail != NULL) 1584 tail->b_next = NULL; 1585 *chain_tail = tail; 1586 *chain_cnt = cnt; 1587 *chain_sz = tsz; 1588 1589 return (head); 1590 } 1591 1592 /* 1593 * mac_rx_srs_drain 1594 * 1595 * The SRS drain routine. Gets to run to clear the queue. Any thread 1596 * (worker, interrupt, poll) can call this based on processing model. 1597 * The first thing we do is disable interrupts if possible and then 1598 * drain the queue. we also try to poll the underlying hardware if 1599 * there is a dedicated hardware Rx ring assigned to this SRS. 1600 * 1601 * There is a equivalent drain routine in bandwidth control mode 1602 * mac_rx_srs_drain_bw. There is some code duplication between the two 1603 * routines but they are highly performance sensitive and are easier 1604 * to read/debug if they stay separate. Any code changes here might 1605 * also apply to mac_rx_srs_drain_bw as well. 1606 */ 1607 void 1608 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type) 1609 { 1610 mblk_t *head; 1611 mblk_t *tail; 1612 timeout_id_t tid; 1613 int cnt = 0; 1614 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1615 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 1616 1617 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 1618 ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL)); 1619 1620 /* If we are blanked i.e. can't do upcalls, then we are done */ 1621 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) { 1622 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) || 1623 (mac_srs->srs_state & SRS_PAUSE)); 1624 goto out; 1625 } 1626 1627 if (mac_srs->srs_first == NULL) 1628 goto out; 1629 1630 if (!(mac_srs->srs_state & SRS_LATENCY_OPT) && 1631 (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) { 1632 /* 1633 * In the normal case, the SRS worker thread does no 1634 * work and we wait for a backlog to build up before 1635 * we switch into polling mode. In case we are 1636 * optimizing for throughput, we use the worker thread 1637 * as well. The goal is to let worker thread process 1638 * the queue and poll thread to feed packets into 1639 * the queue. As such, we should signal the poll 1640 * thread to try and get more packets. 1641 * 1642 * We could have pulled this check in the POLL_RING 1643 * macro itself but keeping it explicit here makes 1644 * the architecture more human understandable. 1645 */ 1646 MAC_SRS_POLL_RING(mac_srs); 1647 } 1648 1649 again: 1650 head = mac_srs->srs_first; 1651 mac_srs->srs_first = NULL; 1652 tail = mac_srs->srs_last; 1653 mac_srs->srs_last = NULL; 1654 cnt = mac_srs->srs_count; 1655 mac_srs->srs_count = 0; 1656 1657 ASSERT(head != NULL); 1658 ASSERT(tail != NULL); 1659 1660 if ((tid = mac_srs->srs_tid) != 0) 1661 mac_srs->srs_tid = 0; 1662 1663 mac_srs->srs_state |= (SRS_PROC|proc_type); 1664 1665 1666 /* 1667 * mcip is NULL for broadcast and multicast flows. The promisc 1668 * callbacks for broadcast and multicast packets are delivered from 1669 * mac_rx() and we don't need to worry about that case in this path 1670 */ 1671 if (mcip != NULL) { 1672 if (mcip->mci_promisc_list != NULL) { 1673 mutex_exit(&mac_srs->srs_lock); 1674 mac_promisc_client_dispatch(mcip, head); 1675 mutex_enter(&mac_srs->srs_lock); 1676 } 1677 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) { 1678 mutex_exit(&mac_srs->srs_lock); 1679 mac_protect_intercept_dhcp(mcip, head); 1680 mutex_enter(&mac_srs->srs_lock); 1681 } 1682 } 1683 1684 /* 1685 * Check if SRS itself is doing the processing 1686 * This direct path does not apply when subflows are present. In this 1687 * case, packets need to be dispatched to a soft ring according to the 1688 * flow's bandwidth and other resources contraints. 1689 */ 1690 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) { 1691 mac_direct_rx_t proc; 1692 void *arg1; 1693 mac_resource_handle_t arg2; 1694 1695 /* 1696 * This is the case when a Rx is directly 1697 * assigned and we have a fully classified 1698 * protocol chain. We can deal with it in 1699 * one shot. 1700 */ 1701 proc = srs_rx->sr_func; 1702 arg1 = srs_rx->sr_arg1; 1703 arg2 = srs_rx->sr_arg2; 1704 1705 mac_srs->srs_state |= SRS_CLIENT_PROC; 1706 mutex_exit(&mac_srs->srs_lock); 1707 if (tid != 0) { 1708 (void) untimeout(tid); 1709 tid = 0; 1710 } 1711 1712 proc(arg1, arg2, head, NULL); 1713 /* 1714 * Decrement the size and count here itelf 1715 * since the packet has been processed. 1716 */ 1717 mutex_enter(&mac_srs->srs_lock); 1718 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt); 1719 if (mac_srs->srs_state & SRS_CLIENT_WAIT) 1720 cv_signal(&mac_srs->srs_client_cv); 1721 mac_srs->srs_state &= ~SRS_CLIENT_PROC; 1722 } else { 1723 /* Some kind of softrings based fanout is required */ 1724 mutex_exit(&mac_srs->srs_lock); 1725 if (tid != 0) { 1726 (void) untimeout(tid); 1727 tid = 0; 1728 } 1729 1730 /* 1731 * Since the fanout routines can deal with chains, 1732 * shoot the entire chain up. 1733 */ 1734 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP) 1735 mac_rx_srs_fanout(mac_srs, head); 1736 else 1737 mac_rx_srs_proto_fanout(mac_srs, head); 1738 mutex_enter(&mac_srs->srs_lock); 1739 } 1740 1741 if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) && 1742 (mac_srs->srs_first != NULL)) { 1743 /* 1744 * More packets arrived while we were clearing the 1745 * SRS. This can be possible because of one of 1746 * three conditions below: 1747 * 1) The driver is using multiple worker threads 1748 * to send the packets to us. 1749 * 2) The driver has a race in switching 1750 * between interrupt and polling mode or 1751 * 3) Packets are arriving in this SRS via the 1752 * S/W classification as well. 1753 * 1754 * We should switch to polling mode and see if we 1755 * need to send the poll thread down. Also, signal 1756 * the worker thread to process whats just arrived. 1757 */ 1758 MAC_SRS_POLLING_ON(mac_srs); 1759 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) { 1760 srs_rx->sr_drain_poll_sig++; 1761 MAC_SRS_POLL_RING(mac_srs); 1762 } 1763 1764 /* 1765 * If we didn't signal the poll thread, we need 1766 * to deal with the pending packets ourselves. 1767 */ 1768 if (proc_type == SRS_WORKER) { 1769 srs_rx->sr_drain_again++; 1770 goto again; 1771 } else { 1772 srs_rx->sr_drain_worker_sig++; 1773 cv_signal(&mac_srs->srs_async); 1774 } 1775 } 1776 1777 out: 1778 if (mac_srs->srs_state & SRS_GET_PKTS) { 1779 /* 1780 * Poll thread is already running. Leave the 1781 * SRS_RPOC set and hand over the control to 1782 * poll thread. 1783 */ 1784 mac_srs->srs_state &= ~proc_type; 1785 srs_rx->sr_drain_poll_running++; 1786 return; 1787 } 1788 1789 /* 1790 * Even if there are no packets queued in SRS, we 1791 * need to make sure that the shared counter is 1792 * clear and any associated softrings have cleared 1793 * all the backlog. Otherwise, leave the interface 1794 * in polling mode and the poll thread will get 1795 * signalled once the count goes down to zero. 1796 * 1797 * If someone is already draining the queue (SRS_PROC is 1798 * set) when the srs_poll_pkt_cnt goes down to zero, 1799 * then it means that drain is already running and we 1800 * will turn off polling at that time if there is 1801 * no backlog. 1802 * 1803 * As long as there are packets queued either 1804 * in soft ring set or its soft rings, we will leave 1805 * the interface in polling mode (even if the drain 1806 * was done being the interrupt thread). We signal 1807 * the poll thread as well if we have dipped below 1808 * low water mark. 1809 * 1810 * NOTE: We can't use the MAC_SRS_POLLING_ON macro 1811 * since that turn polling on only for worker thread. 1812 * Its not worth turning polling on for interrupt 1813 * thread (since NIC will not issue another interrupt) 1814 * unless a backlog builds up. 1815 */ 1816 if ((srs_rx->sr_poll_pkt_cnt > 0) && 1817 (mac_srs->srs_state & SRS_POLLING_CAPAB)) { 1818 mac_srs->srs_state &= ~(SRS_PROC|proc_type); 1819 srs_rx->sr_drain_keep_polling++; 1820 MAC_SRS_POLLING_ON(mac_srs); 1821 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) 1822 MAC_SRS_POLL_RING(mac_srs); 1823 return; 1824 } 1825 1826 /* Nothing else to do. Get out of poll mode */ 1827 MAC_SRS_POLLING_OFF(mac_srs); 1828 mac_srs->srs_state &= ~(SRS_PROC|proc_type); 1829 srs_rx->sr_drain_finish_intr++; 1830 } 1831 1832 /* 1833 * mac_rx_srs_drain_bw 1834 * 1835 * The SRS BW drain routine. Gets to run to clear the queue. Any thread 1836 * (worker, interrupt, poll) can call this based on processing model. 1837 * The first thing we do is disable interrupts if possible and then 1838 * drain the queue. we also try to poll the underlying hardware if 1839 * there is a dedicated hardware Rx ring assigned to this SRS. 1840 * 1841 * There is a equivalent drain routine in non bandwidth control mode 1842 * mac_rx_srs_drain. There is some code duplication between the two 1843 * routines but they are highly performance sensitive and are easier 1844 * to read/debug if they stay separate. Any code changes here might 1845 * also apply to mac_rx_srs_drain as well. 1846 */ 1847 void 1848 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type) 1849 { 1850 mblk_t *head; 1851 mblk_t *tail; 1852 timeout_id_t tid; 1853 size_t sz = 0; 1854 int cnt = 0; 1855 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1856 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 1857 clock_t now; 1858 1859 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 1860 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL); 1861 again: 1862 /* Check if we are doing B/W control */ 1863 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 1864 now = ddi_get_lbolt(); 1865 if (mac_srs->srs_bw->mac_bw_curr_time != now) { 1866 mac_srs->srs_bw->mac_bw_curr_time = now; 1867 mac_srs->srs_bw->mac_bw_used = 0; 1868 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) 1869 mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; 1870 } else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) { 1871 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1872 goto done; 1873 } else if (mac_srs->srs_bw->mac_bw_used > 1874 mac_srs->srs_bw->mac_bw_limit) { 1875 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED; 1876 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1877 goto done; 1878 } 1879 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1880 1881 /* If we are blanked i.e. can't do upcalls, then we are done */ 1882 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) { 1883 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) || 1884 (mac_srs->srs_state & SRS_PAUSE)); 1885 goto done; 1886 } 1887 1888 sz = 0; 1889 cnt = 0; 1890 if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) { 1891 /* 1892 * We couldn't pick up a single packet. 1893 */ 1894 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 1895 if ((mac_srs->srs_bw->mac_bw_used == 0) && 1896 (mac_srs->srs_size != 0) && 1897 !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) { 1898 /* 1899 * Seems like configured B/W doesn't 1900 * even allow processing of 1 packet 1901 * per tick. 1902 * 1903 * XXX: raise the limit to processing 1904 * at least 1 packet per tick. 1905 */ 1906 mac_srs->srs_bw->mac_bw_limit += 1907 mac_srs->srs_bw->mac_bw_limit; 1908 mac_srs->srs_bw->mac_bw_drop_threshold += 1909 mac_srs->srs_bw->mac_bw_drop_threshold; 1910 cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) " 1911 "raised B/W limit to %d since not even a " 1912 "single packet can be processed per " 1913 "tick %d\n", (void *)mac_srs, 1914 (int)mac_srs->srs_bw->mac_bw_limit, 1915 (int)msgdsize(mac_srs->srs_first)); 1916 } 1917 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1918 goto done; 1919 } 1920 1921 ASSERT(head != NULL); 1922 ASSERT(tail != NULL); 1923 1924 /* zero bandwidth: drop all and return to interrupt mode */ 1925 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 1926 if (mac_srs->srs_bw->mac_bw_limit == 0) { 1927 srs_rx->sr_stat.mrs_sdrops += cnt; 1928 ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz); 1929 mac_srs->srs_bw->mac_bw_sz -= sz; 1930 mac_srs->srs_bw->mac_bw_drop_bytes += sz; 1931 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1932 mac_pkt_drop(NULL, NULL, head, B_FALSE); 1933 goto leave_poll; 1934 } else { 1935 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 1936 } 1937 1938 if ((tid = mac_srs->srs_tid) != 0) 1939 mac_srs->srs_tid = 0; 1940 1941 mac_srs->srs_state |= (SRS_PROC|proc_type); 1942 MAC_SRS_WORKER_POLLING_ON(mac_srs); 1943 1944 /* 1945 * mcip is NULL for broadcast and multicast flows. The promisc 1946 * callbacks for broadcast and multicast packets are delivered from 1947 * mac_rx() and we don't need to worry about that case in this path 1948 */ 1949 if (mcip != NULL) { 1950 if (mcip->mci_promisc_list != NULL) { 1951 mutex_exit(&mac_srs->srs_lock); 1952 mac_promisc_client_dispatch(mcip, head); 1953 mutex_enter(&mac_srs->srs_lock); 1954 } 1955 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) { 1956 mutex_exit(&mac_srs->srs_lock); 1957 mac_protect_intercept_dhcp(mcip, head); 1958 mutex_enter(&mac_srs->srs_lock); 1959 } 1960 } 1961 1962 /* 1963 * Check if SRS itself is doing the processing 1964 * This direct path does not apply when subflows are present. In this 1965 * case, packets need to be dispatched to a soft ring according to the 1966 * flow's bandwidth and other resources contraints. 1967 */ 1968 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) { 1969 mac_direct_rx_t proc; 1970 void *arg1; 1971 mac_resource_handle_t arg2; 1972 1973 /* 1974 * This is the case when a Rx is directly 1975 * assigned and we have a fully classified 1976 * protocol chain. We can deal with it in 1977 * one shot. 1978 */ 1979 proc = srs_rx->sr_func; 1980 arg1 = srs_rx->sr_arg1; 1981 arg2 = srs_rx->sr_arg2; 1982 1983 mac_srs->srs_state |= SRS_CLIENT_PROC; 1984 mutex_exit(&mac_srs->srs_lock); 1985 if (tid != 0) { 1986 (void) untimeout(tid); 1987 tid = 0; 1988 } 1989 1990 proc(arg1, arg2, head, NULL); 1991 /* 1992 * Decrement the size and count here itelf 1993 * since the packet has been processed. 1994 */ 1995 mutex_enter(&mac_srs->srs_lock); 1996 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt); 1997 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz); 1998 1999 if (mac_srs->srs_state & SRS_CLIENT_WAIT) 2000 cv_signal(&mac_srs->srs_client_cv); 2001 mac_srs->srs_state &= ~SRS_CLIENT_PROC; 2002 } else { 2003 /* Some kind of softrings based fanout is required */ 2004 mutex_exit(&mac_srs->srs_lock); 2005 if (tid != 0) { 2006 (void) untimeout(tid); 2007 tid = 0; 2008 } 2009 2010 /* 2011 * Since the fanout routines can deal with chains, 2012 * shoot the entire chain up. 2013 */ 2014 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP) 2015 mac_rx_srs_fanout(mac_srs, head); 2016 else 2017 mac_rx_srs_proto_fanout(mac_srs, head); 2018 mutex_enter(&mac_srs->srs_lock); 2019 } 2020 2021 /* 2022 * Send the poll thread to pick up any packets arrived 2023 * so far. This also serves as the last check in case 2024 * nothing else is queued in the SRS. The poll thread 2025 * is signalled only in the case the drain was done 2026 * by the worker thread and SRS_WORKER is set. The 2027 * worker thread can run in parallel as long as the 2028 * SRS_WORKER flag is set. We we have nothing else to 2029 * process, we can exit while leaving SRS_PROC set 2030 * which gives the poll thread control to process and 2031 * cleanup once it returns from the NIC. 2032 * 2033 * If we have nothing else to process, we need to 2034 * ensure that we keep holding the srs_lock till 2035 * all the checks below are done and control is 2036 * handed to the poll thread if it was running. 2037 */ 2038 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 2039 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) { 2040 if (mac_srs->srs_first != NULL) { 2041 if (proc_type == SRS_WORKER) { 2042 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 2043 if (srs_rx->sr_poll_pkt_cnt <= 2044 srs_rx->sr_lowat) 2045 MAC_SRS_POLL_RING(mac_srs); 2046 goto again; 2047 } else { 2048 cv_signal(&mac_srs->srs_async); 2049 } 2050 } 2051 } 2052 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 2053 2054 done: 2055 2056 if (mac_srs->srs_state & SRS_GET_PKTS) { 2057 /* 2058 * Poll thread is already running. Leave the 2059 * SRS_RPOC set and hand over the control to 2060 * poll thread. 2061 */ 2062 mac_srs->srs_state &= ~proc_type; 2063 return; 2064 } 2065 2066 /* 2067 * If we can't process packets because we have exceeded 2068 * B/W limit for this tick, just set the timeout 2069 * and leave. 2070 * 2071 * Even if there are no packets queued in SRS, we 2072 * need to make sure that the shared counter is 2073 * clear and any associated softrings have cleared 2074 * all the backlog. Otherwise, leave the interface 2075 * in polling mode and the poll thread will get 2076 * signalled once the count goes down to zero. 2077 * 2078 * If someone is already draining the queue (SRS_PROC is 2079 * set) when the srs_poll_pkt_cnt goes down to zero, 2080 * then it means that drain is already running and we 2081 * will turn off polling at that time if there is 2082 * no backlog. As long as there are packets queued either 2083 * is soft ring set or its soft rings, we will leave 2084 * the interface in polling mode. 2085 */ 2086 mutex_enter(&mac_srs->srs_bw->mac_bw_lock); 2087 if ((mac_srs->srs_state & SRS_POLLING_CAPAB) && 2088 ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) || 2089 (srs_rx->sr_poll_pkt_cnt > 0))) { 2090 MAC_SRS_POLLING_ON(mac_srs); 2091 mac_srs->srs_state &= ~(SRS_PROC|proc_type); 2092 if ((mac_srs->srs_first != NULL) && 2093 (mac_srs->srs_tid == NULL)) 2094 mac_srs->srs_tid = timeout(mac_srs_fire, 2095 mac_srs, 1); 2096 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 2097 return; 2098 } 2099 mutex_exit(&mac_srs->srs_bw->mac_bw_lock); 2100 2101 leave_poll: 2102 2103 /* Nothing else to do. Get out of poll mode */ 2104 MAC_SRS_POLLING_OFF(mac_srs); 2105 mac_srs->srs_state &= ~(SRS_PROC|proc_type); 2106 } 2107 2108 /* 2109 * mac_srs_worker 2110 * 2111 * The SRS worker routine. Drains the queue when no one else is 2112 * processing it. 2113 */ 2114 void 2115 mac_srs_worker(mac_soft_ring_set_t *mac_srs) 2116 { 2117 kmutex_t *lock = &mac_srs->srs_lock; 2118 kcondvar_t *async = &mac_srs->srs_async; 2119 callb_cpr_t cprinfo; 2120 boolean_t bw_ctl_flag; 2121 2122 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker"); 2123 mutex_enter(lock); 2124 2125 start: 2126 for (;;) { 2127 bw_ctl_flag = B_FALSE; 2128 if (mac_srs->srs_type & SRST_BW_CONTROL) { 2129 MAC_SRS_BW_LOCK(mac_srs); 2130 MAC_SRS_CHECK_BW_CONTROL(mac_srs); 2131 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) 2132 bw_ctl_flag = B_TRUE; 2133 MAC_SRS_BW_UNLOCK(mac_srs); 2134 } 2135 /* 2136 * The SRS_BW_ENFORCED flag may change since we have dropped 2137 * the mac_bw_lock. However the drain function can handle both 2138 * a drainable SRS or a bandwidth controlled SRS, and the 2139 * effect of scheduling a timeout is to wakeup the worker 2140 * thread which in turn will call the drain function. Since 2141 * we release the srs_lock atomically only in the cv_wait there 2142 * isn't a fear of waiting for ever. 2143 */ 2144 while (((mac_srs->srs_state & SRS_PROC) || 2145 (mac_srs->srs_first == NULL) || bw_ctl_flag || 2146 (mac_srs->srs_state & SRS_TX_BLOCKED)) && 2147 !(mac_srs->srs_state & SRS_PAUSE)) { 2148 /* 2149 * If we have packets queued and we are here 2150 * because B/W control is in place, we better 2151 * schedule the worker wakeup after 1 tick 2152 * to see if bandwidth control can be relaxed. 2153 */ 2154 if (bw_ctl_flag && mac_srs->srs_tid == NULL) { 2155 /* 2156 * We need to ensure that a timer is already 2157 * scheduled or we force schedule one for 2158 * later so that we can continue processing 2159 * after this quanta is over. 2160 */ 2161 mac_srs->srs_tid = timeout(mac_srs_fire, 2162 mac_srs, 1); 2163 } 2164 wait: 2165 CALLB_CPR_SAFE_BEGIN(&cprinfo); 2166 cv_wait(async, lock); 2167 CALLB_CPR_SAFE_END(&cprinfo, lock); 2168 2169 if (mac_srs->srs_state & SRS_PAUSE) 2170 goto done; 2171 if (mac_srs->srs_state & SRS_PROC) 2172 goto wait; 2173 2174 if (mac_srs->srs_first != NULL && 2175 mac_srs->srs_type & SRST_BW_CONTROL) { 2176 MAC_SRS_BW_LOCK(mac_srs); 2177 if (mac_srs->srs_bw->mac_bw_state & 2178 SRS_BW_ENFORCED) { 2179 MAC_SRS_CHECK_BW_CONTROL(mac_srs); 2180 } 2181 bw_ctl_flag = mac_srs->srs_bw->mac_bw_state & 2182 SRS_BW_ENFORCED; 2183 MAC_SRS_BW_UNLOCK(mac_srs); 2184 } 2185 } 2186 2187 if (mac_srs->srs_state & SRS_PAUSE) 2188 goto done; 2189 mac_srs->srs_drain_func(mac_srs, SRS_WORKER); 2190 } 2191 done: 2192 /* 2193 * The Rx SRS quiesce logic first cuts off packet supply to the SRS 2194 * from both hard and soft classifications and waits for such threads 2195 * to finish before signaling the worker. So at this point the only 2196 * thread left that could be competing with the worker is the poll 2197 * thread. In the case of Tx, there shouldn't be any thread holding 2198 * SRS_PROC at this point. 2199 */ 2200 if (!(mac_srs->srs_state & SRS_PROC)) { 2201 mac_srs->srs_state |= SRS_PROC; 2202 } else { 2203 ASSERT((mac_srs->srs_type & SRST_TX) == 0); 2204 /* 2205 * Poll thread still owns the SRS and is still running 2206 */ 2207 ASSERT((mac_srs->srs_poll_thr == NULL) || 2208 ((mac_srs->srs_state & SRS_POLL_THR_OWNER) == 2209 SRS_POLL_THR_OWNER)); 2210 } 2211 mac_srs_worker_quiesce(mac_srs); 2212 /* 2213 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator 2214 * of the quiesce operation 2215 */ 2216 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART))) 2217 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2218 2219 if (mac_srs->srs_state & SRS_RESTART) { 2220 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED)); 2221 mac_srs_worker_restart(mac_srs); 2222 mac_srs->srs_state &= ~SRS_PROC; 2223 goto start; 2224 } 2225 2226 if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE)) 2227 mac_srs_worker_quiesce(mac_srs); 2228 2229 mac_srs->srs_state &= ~SRS_PROC; 2230 /* The macro drops the srs_lock */ 2231 CALLB_CPR_EXIT(&cprinfo); 2232 thread_exit(); 2233 } 2234 2235 /* 2236 * mac_rx_srs_subflow_process 2237 * 2238 * Receive side routine called from interrupt path when there are 2239 * sub flows present on this SRS. 2240 */ 2241 /* ARGSUSED */ 2242 void 2243 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs, 2244 mblk_t *mp_chain, boolean_t loopback) 2245 { 2246 flow_entry_t *flent = NULL; 2247 flow_entry_t *prev_flent = NULL; 2248 mblk_t *mp = NULL; 2249 mblk_t *tail = NULL; 2250 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs; 2251 mac_client_impl_t *mcip; 2252 2253 mcip = mac_srs->srs_mcip; 2254 ASSERT(mcip != NULL); 2255 2256 /* 2257 * We need to determine the SRS for every packet 2258 * by walking the flow table, if we don't get any, 2259 * then we proceed using the SRS we came with. 2260 */ 2261 mp = tail = mp_chain; 2262 while (mp != NULL) { 2263 2264 /* 2265 * We will increment the stats for the mactching subflow. 2266 * when we get the bytes/pkt count for the classified packets 2267 * later in mac_rx_srs_process. 2268 */ 2269 (void) mac_flow_lookup(mcip->mci_subflow_tab, mp, 2270 FLOW_INBOUND, &flent); 2271 2272 if (mp == mp_chain || flent == prev_flent) { 2273 if (prev_flent != NULL) 2274 FLOW_REFRELE(prev_flent); 2275 prev_flent = flent; 2276 flent = NULL; 2277 tail = mp; 2278 mp = mp->b_next; 2279 continue; 2280 } 2281 tail->b_next = NULL; 2282 /* 2283 * A null indicates, this is for the mac_srs itself. 2284 * XXX-venu : probably assert for fe_rx_srs_cnt == 0. 2285 */ 2286 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) { 2287 mac_rx_srs_process(arg, 2288 (mac_resource_handle_t)mac_srs, mp_chain, 2289 loopback); 2290 } else { 2291 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1, 2292 prev_flent->fe_cb_arg2, mp_chain, loopback); 2293 FLOW_REFRELE(prev_flent); 2294 } 2295 prev_flent = flent; 2296 flent = NULL; 2297 mp_chain = mp; 2298 tail = mp; 2299 mp = mp->b_next; 2300 } 2301 /* Last chain */ 2302 ASSERT(mp_chain != NULL); 2303 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) { 2304 mac_rx_srs_process(arg, 2305 (mac_resource_handle_t)mac_srs, mp_chain, loopback); 2306 } else { 2307 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1, 2308 prev_flent->fe_cb_arg2, mp_chain, loopback); 2309 FLOW_REFRELE(prev_flent); 2310 } 2311 } 2312 2313 /* 2314 * mac_rx_srs_process 2315 * 2316 * Receive side routine called from the interrupt path. 2317 * 2318 * loopback is set to force a context switch on the loopback 2319 * path between MAC clients. 2320 */ 2321 /* ARGSUSED */ 2322 void 2323 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain, 2324 boolean_t loopback) 2325 { 2326 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs; 2327 mblk_t *mp, *tail, *head; 2328 int count = 0; 2329 int count1; 2330 size_t sz = 0; 2331 size_t chain_sz, sz1; 2332 mac_bw_ctl_t *mac_bw; 2333 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 2334 2335 /* 2336 * Set the tail, count and sz. We set the sz irrespective 2337 * of whether we are doing B/W control or not for the 2338 * purpose of updating the stats. 2339 */ 2340 mp = tail = mp_chain; 2341 while (mp != NULL) { 2342 tail = mp; 2343 count++; 2344 sz += msgdsize(mp); 2345 mp = mp->b_next; 2346 } 2347 2348 mutex_enter(&mac_srs->srs_lock); 2349 2350 if (loopback) { 2351 SRS_RX_STAT_UPDATE(mac_srs, lclbytes, sz); 2352 SRS_RX_STAT_UPDATE(mac_srs, lclcnt, count); 2353 2354 } else { 2355 SRS_RX_STAT_UPDATE(mac_srs, intrbytes, sz); 2356 SRS_RX_STAT_UPDATE(mac_srs, intrcnt, count); 2357 } 2358 2359 /* 2360 * If the SRS in already being processed; has been blanked; 2361 * can be processed by worker thread only; or the B/W limit 2362 * has been reached, then queue the chain and check if 2363 * worker thread needs to be awakend. 2364 */ 2365 if (mac_srs->srs_type & SRST_BW_CONTROL) { 2366 mac_bw = mac_srs->srs_bw; 2367 ASSERT(mac_bw != NULL); 2368 mutex_enter(&mac_bw->mac_bw_lock); 2369 mac_bw->mac_bw_intr += sz; 2370 if (mac_bw->mac_bw_limit == 0) { 2371 /* zero bandwidth: drop all */ 2372 srs_rx->sr_stat.mrs_sdrops += count; 2373 mac_bw->mac_bw_drop_bytes += sz; 2374 mutex_exit(&mac_bw->mac_bw_lock); 2375 mutex_exit(&mac_srs->srs_lock); 2376 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); 2377 return; 2378 } else { 2379 if ((mac_bw->mac_bw_sz + sz) <= 2380 mac_bw->mac_bw_drop_threshold) { 2381 mutex_exit(&mac_bw->mac_bw_lock); 2382 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, 2383 tail, count, sz); 2384 } else { 2385 mp = mp_chain; 2386 chain_sz = 0; 2387 count1 = 0; 2388 tail = NULL; 2389 head = NULL; 2390 while (mp != NULL) { 2391 sz1 = msgdsize(mp); 2392 if (mac_bw->mac_bw_sz + chain_sz + sz1 > 2393 mac_bw->mac_bw_drop_threshold) 2394 break; 2395 chain_sz += sz1; 2396 count1++; 2397 tail = mp; 2398 mp = mp->b_next; 2399 } 2400 mutex_exit(&mac_bw->mac_bw_lock); 2401 if (tail != NULL) { 2402 head = tail->b_next; 2403 tail->b_next = NULL; 2404 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, 2405 mp_chain, tail, count1, chain_sz); 2406 sz -= chain_sz; 2407 count -= count1; 2408 } else { 2409 /* Can't pick up any */ 2410 head = mp_chain; 2411 } 2412 if (head != NULL) { 2413 /* Drop any packet over the threshold */ 2414 srs_rx->sr_stat.mrs_sdrops += count; 2415 mutex_enter(&mac_bw->mac_bw_lock); 2416 mac_bw->mac_bw_drop_bytes += sz; 2417 mutex_exit(&mac_bw->mac_bw_lock); 2418 freemsgchain(head); 2419 } 2420 } 2421 MAC_SRS_WORKER_WAKEUP(mac_srs); 2422 mutex_exit(&mac_srs->srs_lock); 2423 return; 2424 } 2425 } 2426 2427 /* 2428 * If the total number of packets queued in the SRS and 2429 * its associated soft rings exceeds the max allowed, 2430 * then drop the chain. If we are polling capable, this 2431 * shouldn't be happening. 2432 */ 2433 if (!(mac_srs->srs_type & SRST_BW_CONTROL) && 2434 (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) { 2435 mac_bw = mac_srs->srs_bw; 2436 srs_rx->sr_stat.mrs_sdrops += count; 2437 mutex_enter(&mac_bw->mac_bw_lock); 2438 mac_bw->mac_bw_drop_bytes += sz; 2439 mutex_exit(&mac_bw->mac_bw_lock); 2440 freemsgchain(mp_chain); 2441 mutex_exit(&mac_srs->srs_lock); 2442 return; 2443 } 2444 2445 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz); 2446 2447 if (!(mac_srs->srs_state & SRS_PROC)) { 2448 /* 2449 * If we are coming via loopback, if we are not optimizing for 2450 * latency, or if our stack is running deep, we should signal 2451 * the worker thread. 2452 */ 2453 if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT) || 2454 MAC_RX_SRS_TOODEEP()) { 2455 /* 2456 * For loopback, We need to let the worker take 2457 * over as we don't want to continue in the same 2458 * thread even if we can. This could lead to stack 2459 * overflows and may also end up using 2460 * resources (cpu) incorrectly. 2461 */ 2462 cv_signal(&mac_srs->srs_async); 2463 } else { 2464 /* 2465 * Seems like no one is processing the SRS and 2466 * there is no backlog. We also inline process 2467 * our packet if its a single packet in non 2468 * latency optimized case (in latency optimized 2469 * case, we inline process chains of any size). 2470 */ 2471 mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST); 2472 } 2473 } 2474 mutex_exit(&mac_srs->srs_lock); 2475 } 2476 2477 /* TX SIDE ROUTINES (RUNTIME) */ 2478 2479 /* 2480 * mac_tx_srs_no_desc 2481 * 2482 * This routine is called by Tx single ring default mode 2483 * when Tx ring runs out of descs. 2484 */ 2485 mac_tx_cookie_t 2486 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 2487 uint16_t flag, mblk_t **ret_mp) 2488 { 2489 mac_tx_cookie_t cookie = NULL; 2490 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 2491 boolean_t wakeup_worker = B_TRUE; 2492 uint32_t tx_mode = srs_tx->st_mode; 2493 int cnt, sz; 2494 mblk_t *tail; 2495 2496 ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW); 2497 if (flag & MAC_DROP_ON_NO_DESC) { 2498 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie); 2499 } else { 2500 if (mac_srs->srs_first != NULL) 2501 wakeup_worker = B_FALSE; 2502 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz); 2503 if (flag & MAC_TX_NO_ENQUEUE) { 2504 /* 2505 * If TX_QUEUED is not set, queue the 2506 * packet and let mac_tx_srs_drain() 2507 * set the TX_BLOCKED bit for the 2508 * reasons explained above. Otherwise, 2509 * return the mblks. 2510 */ 2511 if (wakeup_worker) { 2512 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, 2513 mp_chain, tail, cnt, sz); 2514 } else { 2515 MAC_TX_SET_NO_ENQUEUE(mac_srs, 2516 mp_chain, ret_mp, cookie); 2517 } 2518 } else { 2519 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain, 2520 tail, cnt, sz, cookie); 2521 } 2522 if (wakeup_worker) 2523 cv_signal(&mac_srs->srs_async); 2524 } 2525 return (cookie); 2526 } 2527 2528 /* 2529 * mac_tx_srs_enqueue 2530 * 2531 * This routine is called when Tx SRS is operating in either serializer 2532 * or bandwidth mode. In serializer mode, a packet will get enqueued 2533 * when a thread cannot enter SRS exclusively. In bandwidth mode, 2534 * packets gets queued if allowed byte-count limit for a tick is 2535 * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and 2536 * MAC_TX_NO_ENQUEUE is set is different than when operaing in either 2537 * the default mode or fanout mode. Here packets get dropped or 2538 * returned back to the caller only after hi-watermark worth of data 2539 * is queued. 2540 */ 2541 static mac_tx_cookie_t 2542 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 2543 uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp) 2544 { 2545 mac_tx_cookie_t cookie = NULL; 2546 int cnt, sz; 2547 mblk_t *tail; 2548 boolean_t wakeup_worker = B_TRUE; 2549 2550 /* 2551 * Ignore fanout hint if we don't have multiple tx rings. 2552 */ 2553 if (!MAC_TX_SOFT_RINGS(mac_srs)) 2554 fanout_hint = 0; 2555 2556 if (mac_srs->srs_first != NULL) 2557 wakeup_worker = B_FALSE; 2558 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz); 2559 if (flag & MAC_DROP_ON_NO_DESC) { 2560 if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) { 2561 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie); 2562 } else { 2563 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, 2564 mp_chain, tail, cnt, sz); 2565 } 2566 } else if (flag & MAC_TX_NO_ENQUEUE) { 2567 if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) || 2568 (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) { 2569 MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain, 2570 ret_mp, cookie); 2571 } else { 2572 mp_chain->b_prev = (mblk_t *)fanout_hint; 2573 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, 2574 mp_chain, tail, cnt, sz); 2575 } 2576 } else { 2577 /* 2578 * If you are BW_ENFORCED, just enqueue the 2579 * packet. srs_worker will drain it at the 2580 * prescribed rate. Before enqueueing, save 2581 * the fanout hint. 2582 */ 2583 mp_chain->b_prev = (mblk_t *)fanout_hint; 2584 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain, 2585 tail, cnt, sz, cookie); 2586 } 2587 if (wakeup_worker) 2588 cv_signal(&mac_srs->srs_async); 2589 return (cookie); 2590 } 2591 2592 /* 2593 * There are seven tx modes: 2594 * 2595 * 1) Default mode (SRS_TX_DEFAULT) 2596 * 2) Serialization mode (SRS_TX_SERIALIZE) 2597 * 3) Fanout mode (SRS_TX_FANOUT) 2598 * 4) Bandwdith mode (SRS_TX_BW) 2599 * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT) 2600 * 6) aggr Tx mode (SRS_TX_AGGR) 2601 * 7) aggr Tx bw mode (SRS_TX_BW_AGGR) 2602 * 2603 * The tx mode in which an SRS operates is decided in mac_tx_srs_setup() 2604 * based on the number of Tx rings requested for an SRS and whether 2605 * bandwidth control is requested or not. 2606 * 2607 * The default mode (i.e., no fanout/no bandwidth) is used when the 2608 * underlying NIC does not have Tx rings or just one Tx ring. In this mode, 2609 * the SRS acts as a pass-thru. Packets will go directly to mac_tx_send(). 2610 * When the underlying Tx ring runs out of Tx descs, it starts queueing up 2611 * packets in SRS. When flow-control is relieved, the srs_worker drains 2612 * the queued packets and informs blocked clients to restart sending 2613 * packets. 2614 * 2615 * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized. This 2616 * mode is used when the link has no Tx rings or only one Tx ring. 2617 * 2618 * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple 2619 * Tx rings. Each Tx ring will have a soft ring associated with it. 2620 * These soft rings will be hung off the Tx SRS. Queueing if it happens 2621 * due to lack of Tx desc will be in individual soft ring (and not srs) 2622 * associated with Tx ring. 2623 * 2624 * In the TX_BW mode, tx srs will allow packets to go down to Tx ring 2625 * only if bw is available. Otherwise the packets will be queued in 2626 * SRS. If fanout to multiple Tx rings is configured, the packets will 2627 * be fanned out among the soft rings associated with the Tx rings. 2628 * 2629 * In SRS_TX_AGGR mode, mac_tx_aggr_mode() routine is called. This routine 2630 * invokes an aggr function, aggr_find_tx_ring(), to find a pseudo Tx ring 2631 * belonging to a port on which the packet has to be sent. Aggr will 2632 * always have a pseudo Tx ring associated with it even when it is an 2633 * aggregation over a single NIC that has no Tx rings. Even in such a 2634 * case, the single pseudo Tx ring will have a soft ring associated with 2635 * it and the soft ring will hang off the SRS. 2636 * 2637 * If a bandwidth is specified for an aggr, SRS_TX_BW_AGGR mode is used. 2638 * In this mode, the bandwidth is first applied on the outgoing packets 2639 * and later mac_tx_addr_mode() function is called to send the packet out 2640 * of one of the pseudo Tx rings. 2641 * 2642 * Four flags are used in srs_state for indicating flow control 2643 * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT. 2644 * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the 2645 * driver below. 2646 * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat 2647 * and flow-control pressure is applied back to clients. The clients expect 2648 * wakeup when flow-control is relieved. 2649 * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk 2650 * got returned back to client either due to lack of Tx descs or due to bw 2651 * control reasons. The clients expect a wakeup when condition is relieved. 2652 * 2653 * The fourth argument to mac_tx() is the flag. Normally it will be 0 but 2654 * some clients set the following values too: MAC_DROP_ON_NO_DESC, 2655 * MAC_TX_NO_ENQUEUE 2656 * Mac clients that do not want packets to be enqueued in the mac layer set 2657 * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or 2658 * Tx soft rings but instead get dropped when the NIC runs out of desc. The 2659 * behaviour of this flag is different when the Tx is running in serializer 2660 * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet 2661 * get dropped when Tx high watermark is reached. 2662 * There are some mac clients like vsw, aggr that want the mblks to be 2663 * returned back to clients instead of being queued in Tx SRS (or Tx soft 2664 * rings) under flow-control (i.e., out of desc or exceeding bw limits) 2665 * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set. 2666 * In the default and Tx fanout mode, the un-transmitted mblks will be 2667 * returned back to the clients when the driver runs out of Tx descs. 2668 * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or 2669 * soft ring) so that the clients can be woken up when Tx desc become 2670 * available. When running in serializer or bandwidth mode mode, 2671 * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached. 2672 */ 2673 2674 mac_tx_func_t 2675 mac_tx_get_func(uint32_t mode) 2676 { 2677 return (mac_tx_mode_list[mode].mac_tx_func); 2678 } 2679 2680 /* ARGSUSED */ 2681 static mac_tx_cookie_t 2682 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 2683 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp) 2684 { 2685 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 2686 mac_tx_stats_t stats; 2687 mac_tx_cookie_t cookie = NULL; 2688 2689 ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT); 2690 2691 /* Regular case with a single Tx ring */ 2692 /* 2693 * SRS_TX_BLOCKED is set when underlying NIC runs 2694 * out of Tx descs and messages start getting 2695 * queued. It won't get reset until 2696 * tx_srs_drain() completely drains out the 2697 * messages. 2698 */ 2699 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) { 2700 /* Tx descs/resources not available */ 2701 mutex_enter(&mac_srs->srs_lock); 2702 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) { 2703 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, 2704 flag, ret_mp); 2705 mutex_exit(&mac_srs->srs_lock); 2706 return (cookie); 2707 } 2708 /* 2709 * While we were computing mblk count, the 2710 * flow control condition got relieved. 2711 * Continue with the transmission. 2712 */ 2713 mutex_exit(&mac_srs->srs_lock); 2714 } 2715 2716 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2, 2717 mp_chain, &stats); 2718 2719 /* 2720 * Multiple threads could be here sending packets. 2721 * Under such conditions, it is not possible to 2722 * automically set SRS_TX_BLOCKED bit to indicate 2723 * out of tx desc condition. To atomically set 2724 * this, we queue the returned packet and do 2725 * the setting of SRS_TX_BLOCKED in 2726 * mac_tx_srs_drain(). 2727 */ 2728 if (mp_chain != NULL) { 2729 mutex_enter(&mac_srs->srs_lock); 2730 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp); 2731 mutex_exit(&mac_srs->srs_lock); 2732 return (cookie); 2733 } 2734 SRS_TX_STATS_UPDATE(mac_srs, &stats); 2735 2736 return (NULL); 2737 } 2738 2739 /* 2740 * mac_tx_serialize_mode 2741 * 2742 * This is an experimental mode implemented as per the request of PAE. 2743 * In this mode, all callers attempting to send a packet to the NIC 2744 * will get serialized. Only one thread at any time will access the 2745 * NIC to send the packet out. 2746 */ 2747 /* ARGSUSED */ 2748 static mac_tx_cookie_t 2749 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 2750 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp) 2751 { 2752 mac_tx_stats_t stats; 2753 mac_tx_cookie_t cookie = NULL; 2754 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 2755 2756 /* Single ring, serialize below */ 2757 ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE); 2758 mutex_enter(&mac_srs->srs_lock); 2759 if ((mac_srs->srs_first != NULL) || 2760 (mac_srs->srs_state & SRS_PROC)) { 2761 /* 2762 * In serialization mode, queue all packets until 2763 * TX_HIWAT is set. 2764 * If drop bit is set, drop if TX_HIWAT is set. 2765 * If no_enqueue is set, still enqueue until hiwat 2766 * is set and return mblks after TX_HIWAT is set. 2767 */ 2768 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, 2769 flag, NULL, ret_mp); 2770 mutex_exit(&mac_srs->srs_lock); 2771 return (cookie); 2772 } 2773 /* 2774 * No packets queued, nothing on proc and no flow 2775 * control condition. Fast-path, ok. Do inline 2776 * processing. 2777 */ 2778 mac_srs->srs_state |= SRS_PROC; 2779 mutex_exit(&mac_srs->srs_lock); 2780 2781 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2, 2782 mp_chain, &stats); 2783 2784 mutex_enter(&mac_srs->srs_lock); 2785 mac_srs->srs_state &= ~SRS_PROC; 2786 if (mp_chain != NULL) { 2787 cookie = mac_tx_srs_enqueue(mac_srs, 2788 mp_chain, flag, NULL, ret_mp); 2789 } 2790 if (mac_srs->srs_first != NULL) { 2791 /* 2792 * We processed inline our packet and a new 2793 * packet/s got queued while we were 2794 * processing. Wakeup srs worker 2795 */ 2796 cv_signal(&mac_srs->srs_async); 2797 } 2798 mutex_exit(&mac_srs->srs_lock); 2799 2800 if (cookie == NULL) 2801 SRS_TX_STATS_UPDATE(mac_srs, &stats); 2802 2803 return (cookie); 2804 } 2805 2806 /* 2807 * mac_tx_fanout_mode 2808 * 2809 * In this mode, the SRS will have access to multiple Tx rings to send 2810 * the packet out. The fanout hint that is passed as an argument is 2811 * used to find an appropriate ring to fanout the traffic. Each Tx 2812 * ring, in turn, will have a soft ring associated with it. If a Tx 2813 * ring runs out of Tx desc's the returned packet will be queued in 2814 * the soft ring associated with that Tx ring. The srs itself will not 2815 * queue any packets. 2816 */ 2817 2818 #define MAC_TX_SOFT_RING_PROCESS(chain) { \ 2819 index = COMPUTE_INDEX(hash, mac_srs->srs_tx_ring_count), \ 2820 softring = mac_srs->srs_tx_soft_rings[index]; \ 2821 cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \ 2822 DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index); \ 2823 } 2824 2825 static mac_tx_cookie_t 2826 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 2827 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp) 2828 { 2829 mac_soft_ring_t *softring; 2830 uint64_t hash; 2831 uint_t index; 2832 mac_tx_cookie_t cookie = NULL; 2833 2834 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT || 2835 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT); 2836 if (fanout_hint != 0) { 2837 /* 2838 * The hint is specified by the caller, simply pass the 2839 * whole chain to the soft ring. 2840 */ 2841 hash = HASH_HINT(fanout_hint); 2842 MAC_TX_SOFT_RING_PROCESS(mp_chain); 2843 } else { 2844 mblk_t *last_mp, *cur_mp, *sub_chain; 2845 uint64_t last_hash = 0; 2846 uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media; 2847 2848 /* 2849 * Compute the hash from the contents (headers) of the 2850 * packets of the mblk chain. Split the chains into 2851 * subchains of the same conversation. 2852 * 2853 * Since there may be more than one ring used for 2854 * sub-chains of the same call, and since the caller 2855 * does not maintain per conversation state since it 2856 * passed a zero hint, unsent subchains will be 2857 * dropped. 2858 */ 2859 2860 flag |= MAC_DROP_ON_NO_DESC; 2861 ret_mp = NULL; 2862 2863 ASSERT(ret_mp == NULL); 2864 2865 sub_chain = NULL; 2866 last_mp = NULL; 2867 2868 for (cur_mp = mp_chain; cur_mp != NULL; 2869 cur_mp = cur_mp->b_next) { 2870 hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4, 2871 B_TRUE); 2872 if (last_hash != 0 && hash != last_hash) { 2873 /* 2874 * Starting a different subchain, send current 2875 * chain out. 2876 */ 2877 ASSERT(last_mp != NULL); 2878 last_mp->b_next = NULL; 2879 MAC_TX_SOFT_RING_PROCESS(sub_chain); 2880 sub_chain = NULL; 2881 } 2882 2883 /* add packet to subchain */ 2884 if (sub_chain == NULL) 2885 sub_chain = cur_mp; 2886 last_mp = cur_mp; 2887 last_hash = hash; 2888 } 2889 2890 if (sub_chain != NULL) { 2891 /* send last subchain */ 2892 ASSERT(last_mp != NULL); 2893 last_mp->b_next = NULL; 2894 MAC_TX_SOFT_RING_PROCESS(sub_chain); 2895 } 2896 2897 cookie = NULL; 2898 } 2899 2900 return (cookie); 2901 } 2902 2903 /* 2904 * mac_tx_bw_mode 2905 * 2906 * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring 2907 * only if bw is available. Otherwise the packets will be queued in 2908 * SRS. If the SRS has multiple Tx rings, then packets will get fanned 2909 * out to a Tx rings. 2910 */ 2911 static mac_tx_cookie_t 2912 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 2913 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp) 2914 { 2915 int cnt, sz; 2916 mblk_t *tail; 2917 mac_tx_cookie_t cookie = NULL; 2918 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 2919 clock_t now; 2920 2921 ASSERT(TX_BANDWIDTH_MODE(mac_srs)); 2922 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL); 2923 mutex_enter(&mac_srs->srs_lock); 2924 if (mac_srs->srs_bw->mac_bw_limit == 0) { 2925 /* 2926 * zero bandwidth, no traffic is sent: drop the packets, 2927 * or return the whole chain if the caller requests all 2928 * unsent packets back. 2929 */ 2930 if (flag & MAC_TX_NO_ENQUEUE) { 2931 cookie = (mac_tx_cookie_t)mac_srs; 2932 *ret_mp = mp_chain; 2933 } else { 2934 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie); 2935 } 2936 mutex_exit(&mac_srs->srs_lock); 2937 return (cookie); 2938 } else if ((mac_srs->srs_first != NULL) || 2939 (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) { 2940 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag, 2941 fanout_hint, ret_mp); 2942 mutex_exit(&mac_srs->srs_lock); 2943 return (cookie); 2944 } 2945 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz); 2946 now = ddi_get_lbolt(); 2947 if (mac_srs->srs_bw->mac_bw_curr_time != now) { 2948 mac_srs->srs_bw->mac_bw_curr_time = now; 2949 mac_srs->srs_bw->mac_bw_used = 0; 2950 } else if (mac_srs->srs_bw->mac_bw_used > 2951 mac_srs->srs_bw->mac_bw_limit) { 2952 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED; 2953 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, 2954 mp_chain, tail, cnt, sz); 2955 /* 2956 * Wakeup worker thread. Note that worker 2957 * thread has to be woken up so that it 2958 * can fire up the timer to be woken up 2959 * on the next tick. Also once 2960 * BW_ENFORCED is set, it can only be 2961 * reset by srs_worker thread. Until then 2962 * all packets will get queued up in SRS 2963 * and hence this this code path won't be 2964 * entered until BW_ENFORCED is reset. 2965 */ 2966 cv_signal(&mac_srs->srs_async); 2967 mutex_exit(&mac_srs->srs_lock); 2968 return (cookie); 2969 } 2970 2971 mac_srs->srs_bw->mac_bw_used += sz; 2972 mutex_exit(&mac_srs->srs_lock); 2973 2974 if (srs_tx->st_mode == SRS_TX_BW_FANOUT) { 2975 mac_soft_ring_t *softring; 2976 uint_t indx, hash; 2977 2978 hash = HASH_HINT(fanout_hint); 2979 indx = COMPUTE_INDEX(hash, 2980 mac_srs->srs_tx_ring_count); 2981 softring = mac_srs->srs_tx_soft_rings[indx]; 2982 return (mac_tx_soft_ring_process(softring, mp_chain, flag, 2983 ret_mp)); 2984 } else if (srs_tx->st_mode == SRS_TX_BW_AGGR) { 2985 return (mac_tx_aggr_mode(mac_srs, mp_chain, 2986 fanout_hint, flag, ret_mp)); 2987 } else { 2988 mac_tx_stats_t stats; 2989 2990 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2, 2991 mp_chain, &stats); 2992 2993 if (mp_chain != NULL) { 2994 mutex_enter(&mac_srs->srs_lock); 2995 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz); 2996 if (mac_srs->srs_bw->mac_bw_used > sz) 2997 mac_srs->srs_bw->mac_bw_used -= sz; 2998 else 2999 mac_srs->srs_bw->mac_bw_used = 0; 3000 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag, 3001 fanout_hint, ret_mp); 3002 mutex_exit(&mac_srs->srs_lock); 3003 return (cookie); 3004 } 3005 SRS_TX_STATS_UPDATE(mac_srs, &stats); 3006 3007 return (NULL); 3008 } 3009 } 3010 3011 /* 3012 * mac_tx_aggr_mode 3013 * 3014 * This routine invokes an aggr function, aggr_find_tx_ring(), to find 3015 * a (pseudo) Tx ring belonging to a port on which the packet has to 3016 * be sent. aggr_find_tx_ring() first finds the outgoing port based on 3017 * L2/L3/L4 policy and then uses the fanout_hint passed to it to pick 3018 * a Tx ring from the selected port. 3019 * 3020 * Note that a port can be deleted from the aggregation. In such a case, 3021 * the aggregation layer first separates the port from the rest of the 3022 * ports making sure that port (and thus any Tx rings associated with 3023 * it) won't get selected in the call to aggr_find_tx_ring() function. 3024 * Later calls are made to mac_group_rem_ring() passing pseudo Tx ring 3025 * handles one by one which in turn will quiesce the Tx SRS and remove 3026 * the soft ring associated with the pseudo Tx ring. Unlike Rx side 3027 * where a cookie is used to protect against mac_rx_ring() calls on 3028 * rings that have been removed, no such cookie is needed on the Tx 3029 * side as the pseudo Tx ring won't be available anymore to 3030 * aggr_find_tx_ring() once the port has been removed. 3031 */ 3032 static mac_tx_cookie_t 3033 mac_tx_aggr_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain, 3034 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp) 3035 { 3036 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 3037 mac_tx_ring_fn_t find_tx_ring_fn; 3038 mac_ring_handle_t ring = NULL; 3039 void *arg; 3040 mac_soft_ring_t *sringp; 3041 3042 find_tx_ring_fn = srs_tx->st_capab_aggr.mca_find_tx_ring_fn; 3043 arg = srs_tx->st_capab_aggr.mca_arg; 3044 if (find_tx_ring_fn(arg, mp_chain, fanout_hint, &ring) == NULL) 3045 return (NULL); 3046 sringp = srs_tx->st_soft_rings[((mac_ring_t *)ring)->mr_index]; 3047 return (mac_tx_soft_ring_process(sringp, mp_chain, flag, ret_mp)); 3048 } 3049 3050 void 3051 mac_tx_invoke_callbacks(mac_client_impl_t *mcip, mac_tx_cookie_t cookie) 3052 { 3053 mac_cb_t *mcb; 3054 mac_tx_notify_cb_t *mtnfp; 3055 3056 /* Wakeup callback registered clients */ 3057 MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info); 3058 for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL; 3059 mcb = mcb->mcb_nextp) { 3060 mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp; 3061 mtnfp->mtnf_fn(mtnfp->mtnf_arg, cookie); 3062 } 3063 MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info, 3064 &mcip->mci_tx_notify_cb_list); 3065 } 3066 3067 /* ARGSUSED */ 3068 void 3069 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type) 3070 { 3071 mblk_t *head, *tail; 3072 size_t sz; 3073 uint32_t tx_mode; 3074 uint_t saved_pkt_count; 3075 mac_tx_stats_t stats; 3076 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 3077 clock_t now; 3078 3079 saved_pkt_count = 0; 3080 ASSERT(mutex_owned(&mac_srs->srs_lock)); 3081 ASSERT(!(mac_srs->srs_state & SRS_PROC)); 3082 3083 mac_srs->srs_state |= SRS_PROC; 3084 3085 tx_mode = srs_tx->st_mode; 3086 if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) { 3087 if (mac_srs->srs_first != NULL) { 3088 head = mac_srs->srs_first; 3089 tail = mac_srs->srs_last; 3090 saved_pkt_count = mac_srs->srs_count; 3091 mac_srs->srs_first = NULL; 3092 mac_srs->srs_last = NULL; 3093 mac_srs->srs_count = 0; 3094 mutex_exit(&mac_srs->srs_lock); 3095 3096 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2, 3097 head, &stats); 3098 3099 mutex_enter(&mac_srs->srs_lock); 3100 if (head != NULL) { 3101 /* Device out of tx desc, set block */ 3102 if (head->b_next == NULL) 3103 VERIFY(head == tail); 3104 tail->b_next = mac_srs->srs_first; 3105 mac_srs->srs_first = head; 3106 mac_srs->srs_count += 3107 (saved_pkt_count - stats.mts_opackets); 3108 if (mac_srs->srs_last == NULL) 3109 mac_srs->srs_last = tail; 3110 MAC_TX_SRS_BLOCK(mac_srs, head); 3111 } else { 3112 srs_tx->st_woken_up = B_FALSE; 3113 SRS_TX_STATS_UPDATE(mac_srs, &stats); 3114 } 3115 } 3116 } else if (tx_mode == SRS_TX_BW) { 3117 /* 3118 * We are here because the timer fired and we have some data 3119 * to tranmit. Also mac_tx_srs_worker should have reset 3120 * SRS_BW_ENFORCED flag 3121 */ 3122 ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)); 3123 head = tail = mac_srs->srs_first; 3124 while (mac_srs->srs_first != NULL) { 3125 tail = mac_srs->srs_first; 3126 tail->b_prev = NULL; 3127 mac_srs->srs_first = tail->b_next; 3128 if (mac_srs->srs_first == NULL) 3129 mac_srs->srs_last = NULL; 3130 mac_srs->srs_count--; 3131 sz = msgdsize(tail); 3132 mac_srs->srs_size -= sz; 3133 saved_pkt_count++; 3134 MAC_TX_UPDATE_BW_INFO(mac_srs, sz); 3135 3136 if (mac_srs->srs_bw->mac_bw_used < 3137 mac_srs->srs_bw->mac_bw_limit) 3138 continue; 3139 3140 now = ddi_get_lbolt(); 3141 if (mac_srs->srs_bw->mac_bw_curr_time != now) { 3142 mac_srs->srs_bw->mac_bw_curr_time = now; 3143 mac_srs->srs_bw->mac_bw_used = sz; 3144 continue; 3145 } 3146 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED; 3147 break; 3148 } 3149 3150 ASSERT((head == NULL && tail == NULL) || 3151 (head != NULL && tail != NULL)); 3152 if (tail != NULL) { 3153 tail->b_next = NULL; 3154 mutex_exit(&mac_srs->srs_lock); 3155 3156 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2, 3157 head, &stats); 3158 3159 mutex_enter(&mac_srs->srs_lock); 3160 if (head != NULL) { 3161 uint_t size_sent; 3162 3163 /* Device out of tx desc, set block */ 3164 if (head->b_next == NULL) 3165 VERIFY(head == tail); 3166 tail->b_next = mac_srs->srs_first; 3167 mac_srs->srs_first = head; 3168 mac_srs->srs_count += 3169 (saved_pkt_count - stats.mts_opackets); 3170 if (mac_srs->srs_last == NULL) 3171 mac_srs->srs_last = tail; 3172 size_sent = sz - stats.mts_obytes; 3173 mac_srs->srs_size += size_sent; 3174 mac_srs->srs_bw->mac_bw_sz += size_sent; 3175 if (mac_srs->srs_bw->mac_bw_used > size_sent) { 3176 mac_srs->srs_bw->mac_bw_used -= 3177 size_sent; 3178 } else { 3179 mac_srs->srs_bw->mac_bw_used = 0; 3180 } 3181 MAC_TX_SRS_BLOCK(mac_srs, head); 3182 } else { 3183 srs_tx->st_woken_up = B_FALSE; 3184 SRS_TX_STATS_UPDATE(mac_srs, &stats); 3185 } 3186 } 3187 } else if (tx_mode == SRS_TX_BW_FANOUT || tx_mode == SRS_TX_BW_AGGR) { 3188 mblk_t *prev; 3189 uint64_t hint; 3190 3191 /* 3192 * We are here because the timer fired and we 3193 * have some quota to tranmit. 3194 */ 3195 prev = NULL; 3196 head = tail = mac_srs->srs_first; 3197 while (mac_srs->srs_first != NULL) { 3198 tail = mac_srs->srs_first; 3199 mac_srs->srs_first = tail->b_next; 3200 if (mac_srs->srs_first == NULL) 3201 mac_srs->srs_last = NULL; 3202 mac_srs->srs_count--; 3203 sz = msgdsize(tail); 3204 mac_srs->srs_size -= sz; 3205 mac_srs->srs_bw->mac_bw_used += sz; 3206 if (prev == NULL) 3207 hint = (ulong_t)tail->b_prev; 3208 if (hint != (ulong_t)tail->b_prev) { 3209 prev->b_next = NULL; 3210 mutex_exit(&mac_srs->srs_lock); 3211 TX_SRS_TO_SOFT_RING(mac_srs, head, hint); 3212 head = tail; 3213 hint = (ulong_t)tail->b_prev; 3214 mutex_enter(&mac_srs->srs_lock); 3215 } 3216 3217 prev = tail; 3218 tail->b_prev = NULL; 3219 if (mac_srs->srs_bw->mac_bw_used < 3220 mac_srs->srs_bw->mac_bw_limit) 3221 continue; 3222 3223 now = ddi_get_lbolt(); 3224 if (mac_srs->srs_bw->mac_bw_curr_time != now) { 3225 mac_srs->srs_bw->mac_bw_curr_time = now; 3226 mac_srs->srs_bw->mac_bw_used = 0; 3227 continue; 3228 } 3229 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED; 3230 break; 3231 } 3232 ASSERT((head == NULL && tail == NULL) || 3233 (head != NULL && tail != NULL)); 3234 if (tail != NULL) { 3235 tail->b_next = NULL; 3236 mutex_exit(&mac_srs->srs_lock); 3237 TX_SRS_TO_SOFT_RING(mac_srs, head, hint); 3238 mutex_enter(&mac_srs->srs_lock); 3239 } 3240 } 3241 /* 3242 * SRS_TX_FANOUT case not considered here because packets 3243 * won't be queued in the SRS for this case. Packets will 3244 * be sent directly to soft rings underneath and if there 3245 * is any queueing at all, it would be in Tx side soft 3246 * rings. 3247 */ 3248 3249 /* 3250 * When srs_count becomes 0, reset SRS_TX_HIWAT and 3251 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients. 3252 */ 3253 if (mac_srs->srs_count == 0 && (mac_srs->srs_state & 3254 (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) { 3255 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3256 boolean_t wakeup_required = B_FALSE; 3257 3258 if (mac_srs->srs_state & 3259 (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) { 3260 wakeup_required = B_TRUE; 3261 } 3262 mac_srs->srs_state &= ~(SRS_TX_HIWAT | 3263 SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED); 3264 mutex_exit(&mac_srs->srs_lock); 3265 if (wakeup_required) { 3266 mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)mac_srs); 3267 /* 3268 * If the client is not the primary MAC client, then we 3269 * need to send the notification to the clients upper 3270 * MAC, i.e. mci_upper_mip. 3271 */ 3272 mac_tx_notify(mcip->mci_upper_mip != NULL ? 3273 mcip->mci_upper_mip : mcip->mci_mip); 3274 } 3275 mutex_enter(&mac_srs->srs_lock); 3276 } 3277 mac_srs->srs_state &= ~SRS_PROC; 3278 } 3279 3280 /* 3281 * Given a packet, get the flow_entry that identifies the flow 3282 * to which that packet belongs. The flow_entry will contain 3283 * the transmit function to be used to send the packet. If the 3284 * function returns NULL, the packet should be sent using the 3285 * underlying NIC. 3286 */ 3287 static flow_entry_t * 3288 mac_tx_classify(mac_impl_t *mip, mblk_t *mp) 3289 { 3290 flow_entry_t *flent = NULL; 3291 mac_client_impl_t *mcip; 3292 int err; 3293 3294 /* 3295 * Do classification on the packet. 3296 */ 3297 err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent); 3298 if (err != 0) 3299 return (NULL); 3300 3301 /* 3302 * This flent might just be an additional one on the MAC client, 3303 * i.e. for classification purposes (different fdesc), however 3304 * the resources, SRS et. al., are in the mci_flent, so if 3305 * this isn't the mci_flent, we need to get it. 3306 */ 3307 if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) { 3308 FLOW_REFRELE(flent); 3309 flent = mcip->mci_flent; 3310 FLOW_TRY_REFHOLD(flent, err); 3311 if (err != 0) 3312 return (NULL); 3313 } 3314 3315 return (flent); 3316 } 3317 3318 /* 3319 * This macro is only meant to be used by mac_tx_send(). 3320 */ 3321 #define CHECK_VID_AND_ADD_TAG(mp) { \ 3322 if (vid_check) { \ 3323 int err = 0; \ 3324 \ 3325 MAC_VID_CHECK(src_mcip, (mp), err); \ 3326 if (err != 0) { \ 3327 freemsg((mp)); \ 3328 (mp) = next; \ 3329 oerrors++; \ 3330 continue; \ 3331 } \ 3332 } \ 3333 if (add_tag) { \ 3334 (mp) = mac_add_vlan_tag((mp), 0, vid); \ 3335 if ((mp) == NULL) { \ 3336 (mp) = next; \ 3337 oerrors++; \ 3338 continue; \ 3339 } \ 3340 } \ 3341 } 3342 3343 mblk_t * 3344 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain, 3345 mac_tx_stats_t *stats) 3346 { 3347 mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch; 3348 mac_impl_t *mip = src_mcip->mci_mip; 3349 uint_t obytes = 0, opackets = 0, oerrors = 0; 3350 mblk_t *mp = NULL, *next; 3351 boolean_t vid_check, add_tag; 3352 uint16_t vid = 0; 3353 3354 if (mip->mi_nclients > 1) { 3355 vid_check = MAC_VID_CHECK_NEEDED(src_mcip); 3356 add_tag = MAC_TAG_NEEDED(src_mcip); 3357 if (add_tag) 3358 vid = mac_client_vid(mch); 3359 } else { 3360 ASSERT(mip->mi_nclients == 1); 3361 vid_check = add_tag = B_FALSE; 3362 } 3363 3364 /* 3365 * Fastpath: if there's only one client, we simply send 3366 * the packet down to the underlying NIC. 3367 */ 3368 if (mip->mi_nactiveclients == 1) { 3369 DTRACE_PROBE2(fastpath, 3370 mac_client_impl_t *, src_mcip, mblk_t *, mp_chain); 3371 3372 mp = mp_chain; 3373 while (mp != NULL) { 3374 next = mp->b_next; 3375 mp->b_next = NULL; 3376 opackets++; 3377 obytes += (mp->b_cont == NULL ? MBLKL(mp) : 3378 msgdsize(mp)); 3379 3380 CHECK_VID_AND_ADD_TAG(mp); 3381 MAC_TX(mip, ring, mp, src_mcip); 3382 3383 /* 3384 * If the driver is out of descriptors and does a 3385 * partial send it will return a chain of unsent 3386 * mblks. Adjust the accounting stats. 3387 */ 3388 if (mp != NULL) { 3389 opackets--; 3390 obytes -= msgdsize(mp); 3391 mp->b_next = next; 3392 break; 3393 } 3394 mp = next; 3395 } 3396 goto done; 3397 } 3398 3399 /* 3400 * No fastpath, we either have more than one MAC client 3401 * defined on top of the same MAC, or one or more MAC 3402 * client promiscuous callbacks. 3403 */ 3404 DTRACE_PROBE3(slowpath, mac_client_impl_t *, 3405 src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain); 3406 3407 mp = mp_chain; 3408 while (mp != NULL) { 3409 flow_entry_t *dst_flow_ent; 3410 void *flow_cookie; 3411 size_t pkt_size; 3412 mblk_t *mp1; 3413 3414 next = mp->b_next; 3415 mp->b_next = NULL; 3416 opackets++; 3417 pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp)); 3418 obytes += pkt_size; 3419 CHECK_VID_AND_ADD_TAG(mp); 3420 3421 /* 3422 * Find the destination. 3423 */ 3424 dst_flow_ent = mac_tx_classify(mip, mp); 3425 3426 if (dst_flow_ent != NULL) { 3427 size_t hdrsize; 3428 int err = 0; 3429 3430 if (mip->mi_info.mi_nativemedia == DL_ETHER) { 3431 struct ether_vlan_header *evhp = 3432 (struct ether_vlan_header *)mp->b_rptr; 3433 3434 if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN) 3435 hdrsize = sizeof (*evhp); 3436 else 3437 hdrsize = sizeof (struct ether_header); 3438 } else { 3439 mac_header_info_t mhi; 3440 3441 err = mac_header_info((mac_handle_t)mip, 3442 mp, &mhi); 3443 if (err == 0) 3444 hdrsize = mhi.mhi_hdrsize; 3445 } 3446 3447 /* 3448 * Got a matching flow. It's either another 3449 * MAC client, or a broadcast/multicast flow. 3450 * Make sure the packet size is within the 3451 * allowed size. If not drop the packet and 3452 * move to next packet. 3453 */ 3454 if (err != 0 || 3455 (pkt_size - hdrsize) > mip->mi_sdu_max) { 3456 oerrors++; 3457 DTRACE_PROBE2(loopback__drop, size_t, pkt_size, 3458 mblk_t *, mp); 3459 freemsg(mp); 3460 mp = next; 3461 FLOW_REFRELE(dst_flow_ent); 3462 continue; 3463 } 3464 flow_cookie = mac_flow_get_client_cookie(dst_flow_ent); 3465 if (flow_cookie != NULL) { 3466 /* 3467 * The vnic_bcast_send function expects 3468 * to receive the sender MAC client 3469 * as value for arg2. 3470 */ 3471 mac_bcast_send(flow_cookie, src_mcip, mp, 3472 B_TRUE); 3473 } else { 3474 /* 3475 * loopback the packet to a local MAC 3476 * client. We force a context switch 3477 * if both source and destination MAC 3478 * clients are used by IP, i.e. 3479 * bypass is set. 3480 */ 3481 boolean_t do_switch; 3482 mac_client_impl_t *dst_mcip = 3483 dst_flow_ent->fe_mcip; 3484 3485 /* 3486 * Check if there are promiscuous mode 3487 * callbacks defined. This check is 3488 * done here in the 'else' case and 3489 * not in other cases because this 3490 * path is for local loopback 3491 * communication which does not go 3492 * through MAC_TX(). For paths that go 3493 * through MAC_TX(), the promisc_list 3494 * check is done inside the MAC_TX() 3495 * macro. 3496 */ 3497 if (mip->mi_promisc_list != NULL) 3498 mac_promisc_dispatch(mip, mp, src_mcip); 3499 3500 do_switch = ((src_mcip->mci_state_flags & 3501 dst_mcip->mci_state_flags & 3502 MCIS_CLIENT_POLL_CAPABLE) != 0); 3503 3504 if ((mp1 = mac_fix_cksum(mp)) != NULL) { 3505 (dst_flow_ent->fe_cb_fn)( 3506 dst_flow_ent->fe_cb_arg1, 3507 dst_flow_ent->fe_cb_arg2, 3508 mp1, do_switch); 3509 } 3510 } 3511 FLOW_REFRELE(dst_flow_ent); 3512 } else { 3513 /* 3514 * Unknown destination, send via the underlying 3515 * NIC. 3516 */ 3517 MAC_TX(mip, ring, mp, src_mcip); 3518 if (mp != NULL) { 3519 /* 3520 * Adjust for the last packet that 3521 * could not be transmitted 3522 */ 3523 opackets--; 3524 obytes -= pkt_size; 3525 mp->b_next = next; 3526 break; 3527 } 3528 } 3529 mp = next; 3530 } 3531 3532 done: 3533 stats->mts_obytes = obytes; 3534 stats->mts_opackets = opackets; 3535 stats->mts_oerrors = oerrors; 3536 return (mp); 3537 } 3538 3539 /* 3540 * mac_tx_srs_ring_present 3541 * 3542 * Returns whether the specified ring is part of the specified SRS. 3543 */ 3544 boolean_t 3545 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring) 3546 { 3547 int i; 3548 mac_soft_ring_t *soft_ring; 3549 3550 if (srs->srs_tx.st_arg2 == tx_ring) 3551 return (B_TRUE); 3552 3553 for (i = 0; i < srs->srs_tx_ring_count; i++) { 3554 soft_ring = srs->srs_tx_soft_rings[i]; 3555 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3556 return (B_TRUE); 3557 } 3558 3559 return (B_FALSE); 3560 } 3561 3562 /* 3563 * mac_tx_srs_get_soft_ring 3564 * 3565 * Returns the TX soft ring associated with the given ring, if present. 3566 */ 3567 mac_soft_ring_t * 3568 mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring) 3569 { 3570 int i; 3571 mac_soft_ring_t *soft_ring; 3572 3573 if (srs->srs_tx.st_arg2 == tx_ring) 3574 return (NULL); 3575 3576 for (i = 0; i < srs->srs_tx_ring_count; i++) { 3577 soft_ring = srs->srs_tx_soft_rings[i]; 3578 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3579 return (soft_ring); 3580 } 3581 3582 return (NULL); 3583 } 3584 3585 /* 3586 * mac_tx_srs_wakeup 3587 * 3588 * Called when Tx desc become available. Wakeup the appropriate worker 3589 * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the 3590 * state field. 3591 */ 3592 void 3593 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring) 3594 { 3595 int i; 3596 mac_soft_ring_t *sringp; 3597 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx; 3598 3599 mutex_enter(&mac_srs->srs_lock); 3600 /* 3601 * srs_tx_ring_count == 0 is the single ring mode case. In 3602 * this mode, there will not be Tx soft rings associated 3603 * with the SRS. 3604 */ 3605 if (!MAC_TX_SOFT_RINGS(mac_srs)) { 3606 if (srs_tx->st_arg2 == ring && 3607 mac_srs->srs_state & SRS_TX_BLOCKED) { 3608 mac_srs->srs_state &= ~SRS_TX_BLOCKED; 3609 srs_tx->st_stat.mts_unblockcnt++; 3610 cv_signal(&mac_srs->srs_async); 3611 } 3612 /* 3613 * A wakeup can come before tx_srs_drain() could 3614 * grab srs lock and set SRS_TX_BLOCKED. So 3615 * always set woken_up flag when we come here. 3616 */ 3617 srs_tx->st_woken_up = B_TRUE; 3618 mutex_exit(&mac_srs->srs_lock); 3619 return; 3620 } 3621 3622 /* 3623 * If you are here, it is for FANOUT, BW_FANOUT, 3624 * AGGR_MODE or AGGR_BW_MODE case 3625 */ 3626 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 3627 sringp = mac_srs->srs_tx_soft_rings[i]; 3628 mutex_enter(&sringp->s_ring_lock); 3629 if (sringp->s_ring_tx_arg2 == ring) { 3630 if (sringp->s_ring_state & S_RING_BLOCK) { 3631 sringp->s_ring_state &= ~S_RING_BLOCK; 3632 sringp->s_st_stat.mts_unblockcnt++; 3633 cv_signal(&sringp->s_ring_async); 3634 } 3635 sringp->s_ring_tx_woken_up = B_TRUE; 3636 } 3637 mutex_exit(&sringp->s_ring_lock); 3638 } 3639 mutex_exit(&mac_srs->srs_lock); 3640 } 3641 3642 /* 3643 * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash 3644 * the blocked clients again. 3645 */ 3646 void 3647 mac_tx_notify(mac_impl_t *mip) 3648 { 3649 i_mac_notify(mip, MAC_NOTE_TX); 3650 } 3651 3652 /* 3653 * RX SOFTRING RELATED FUNCTIONS 3654 * 3655 * These functions really belong in mac_soft_ring.c and here for 3656 * a short period. 3657 */ 3658 3659 #define SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \ 3660 /* \ 3661 * Enqueue our mblk chain. \ 3662 */ \ 3663 ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock)); \ 3664 \ 3665 if ((ringp)->s_ring_last != NULL) \ 3666 (ringp)->s_ring_last->b_next = (mp); \ 3667 else \ 3668 (ringp)->s_ring_first = (mp); \ 3669 (ringp)->s_ring_last = (tail); \ 3670 (ringp)->s_ring_count += (cnt); \ 3671 ASSERT((ringp)->s_ring_count > 0); \ 3672 if ((ringp)->s_ring_type & ST_RING_BW_CTL) { \ 3673 (ringp)->s_ring_size += sz; \ 3674 } \ 3675 } 3676 3677 /* 3678 * Default entry point to deliver a packet chain to a MAC client. 3679 * If the MAC client has flows, do the classification with these 3680 * flows as well. 3681 */ 3682 /* ARGSUSED */ 3683 void 3684 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain, 3685 mac_header_info_t *arg3) 3686 { 3687 mac_client_impl_t *mcip = arg1; 3688 3689 if (mcip->mci_nvids == 1 && 3690 !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) { 3691 /* 3692 * If the client has exactly one VID associated with it 3693 * and striping of VLAN header is not disabled, 3694 * remove the VLAN tag from the packet before 3695 * passing it on to the client's receive callback. 3696 * Note that this needs to be done after we dispatch 3697 * the packet to the promiscuous listeners of the 3698 * client, since they expect to see the whole 3699 * frame including the VLAN headers. 3700 */ 3701 mp_chain = mac_strip_vlan_tag_chain(mp_chain); 3702 } 3703 3704 mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE); 3705 } 3706 3707 /* 3708 * mac_rx_soft_ring_process 3709 * 3710 * process a chain for a given soft ring. The number of packets queued 3711 * in the SRS and its associated soft rings (including this one) is 3712 * very small (tracked by srs_poll_pkt_cnt), then allow the entering 3713 * thread (interrupt or poll thread) to do inline processing. This 3714 * helps keep the latency down under low load. 3715 * 3716 * The proc and arg for each mblk is already stored in the mblk in 3717 * appropriate places. 3718 */ 3719 /* ARGSUSED */ 3720 void 3721 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp, 3722 mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz) 3723 { 3724 mac_direct_rx_t proc; 3725 void *arg1; 3726 mac_resource_handle_t arg2; 3727 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set; 3728 3729 ASSERT(ringp != NULL); 3730 ASSERT(mp_chain != NULL); 3731 ASSERT(tail != NULL); 3732 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock)); 3733 3734 mutex_enter(&ringp->s_ring_lock); 3735 ringp->s_ring_total_inpkt += cnt; 3736 ringp->s_ring_total_rbytes += sz; 3737 if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) && 3738 !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) { 3739 /* If on processor or blanking on, then enqueue and return */ 3740 if (ringp->s_ring_state & S_RING_BLANK || 3741 ringp->s_ring_state & S_RING_PROC) { 3742 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); 3743 mutex_exit(&ringp->s_ring_lock); 3744 return; 3745 } 3746 proc = ringp->s_ring_rx_func; 3747 arg1 = ringp->s_ring_rx_arg1; 3748 arg2 = ringp->s_ring_rx_arg2; 3749 /* 3750 * See if anything is already queued. If we are the 3751 * first packet, do inline processing else queue the 3752 * packet and do the drain. 3753 */ 3754 if (ringp->s_ring_first == NULL) { 3755 /* 3756 * Fast-path, ok to process and nothing queued. 3757 */ 3758 ringp->s_ring_run = curthread; 3759 ringp->s_ring_state |= (S_RING_PROC); 3760 3761 mutex_exit(&ringp->s_ring_lock); 3762 3763 /* 3764 * We are the chain of 1 packet so 3765 * go through this fast path. 3766 */ 3767 ASSERT(mp_chain->b_next == NULL); 3768 3769 (*proc)(arg1, arg2, mp_chain, NULL); 3770 3771 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock)); 3772 /* 3773 * If we have a soft ring set which is doing 3774 * bandwidth control, we need to decrement 3775 * srs_size and count so it the SRS can have a 3776 * accurate idea of what is the real data 3777 * queued between SRS and its soft rings. We 3778 * decrement the counters only when the packet 3779 * gets processed by both SRS and the soft ring. 3780 */ 3781 mutex_enter(&mac_srs->srs_lock); 3782 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt); 3783 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz); 3784 mutex_exit(&mac_srs->srs_lock); 3785 3786 mutex_enter(&ringp->s_ring_lock); 3787 ringp->s_ring_run = NULL; 3788 ringp->s_ring_state &= ~S_RING_PROC; 3789 if (ringp->s_ring_state & S_RING_CLIENT_WAIT) 3790 cv_signal(&ringp->s_ring_client_cv); 3791 3792 if ((ringp->s_ring_first == NULL) || 3793 (ringp->s_ring_state & S_RING_BLANK)) { 3794 /* 3795 * We processed inline our packet and 3796 * nothing new has arrived or our 3797 * receiver doesn't want to receive 3798 * any packets. We are done. 3799 */ 3800 mutex_exit(&ringp->s_ring_lock); 3801 return; 3802 } 3803 } else { 3804 SOFT_RING_ENQUEUE_CHAIN(ringp, 3805 mp_chain, tail, cnt, sz); 3806 } 3807 3808 /* 3809 * We are here because either we couldn't do inline 3810 * processing (because something was already 3811 * queued), or we had a chain of more than one 3812 * packet, or something else arrived after we were 3813 * done with inline processing. 3814 */ 3815 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); 3816 ASSERT(ringp->s_ring_first != NULL); 3817 3818 ringp->s_ring_drain_func(ringp); 3819 mutex_exit(&ringp->s_ring_lock); 3820 return; 3821 } else { 3822 /* ST_RING_WORKER_ONLY case */ 3823 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); 3824 mac_soft_ring_worker_wakeup(ringp); 3825 mutex_exit(&ringp->s_ring_lock); 3826 } 3827 } 3828 3829 /* 3830 * TX SOFTRING RELATED FUNCTIONS 3831 * 3832 * These functions really belong in mac_soft_ring.c and here for 3833 * a short period. 3834 */ 3835 3836 #define TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \ 3837 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); \ 3838 ringp->s_ring_state |= S_RING_ENQUEUED; \ 3839 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); \ 3840 } 3841 3842 /* 3843 * mac_tx_sring_queued 3844 * 3845 * When we are out of transmit descriptors and we already have a 3846 * queue that exceeds hiwat (or the client called us with 3847 * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the 3848 * soft ring pointer as the opaque cookie for the client enable 3849 * flow control. 3850 */ 3851 static mac_tx_cookie_t 3852 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag, 3853 mblk_t **ret_mp) 3854 { 3855 int cnt; 3856 size_t sz; 3857 mblk_t *tail; 3858 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set; 3859 mac_tx_cookie_t cookie = NULL; 3860 boolean_t wakeup_worker = B_TRUE; 3861 3862 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); 3863 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz); 3864 if (flag & MAC_DROP_ON_NO_DESC) { 3865 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); 3866 /* increment freed stats */ 3867 ringp->s_ring_drops += cnt; 3868 cookie = (mac_tx_cookie_t)ringp; 3869 } else { 3870 if (ringp->s_ring_first != NULL) 3871 wakeup_worker = B_FALSE; 3872 3873 if (flag & MAC_TX_NO_ENQUEUE) { 3874 /* 3875 * If QUEUED is not set, queue the packet 3876 * and let mac_tx_soft_ring_drain() set 3877 * the TX_BLOCKED bit for the reasons 3878 * explained above. Otherwise, return the 3879 * mblks. 3880 */ 3881 if (wakeup_worker) { 3882 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, 3883 mp_chain, tail, cnt, sz); 3884 } else { 3885 ringp->s_ring_state |= S_RING_WAKEUP_CLIENT; 3886 cookie = (mac_tx_cookie_t)ringp; 3887 *ret_mp = mp_chain; 3888 } 3889 } else { 3890 boolean_t enqueue = B_TRUE; 3891 3892 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) { 3893 /* 3894 * flow-controlled. Store ringp in cookie 3895 * so that it can be returned as 3896 * mac_tx_cookie_t to client 3897 */ 3898 ringp->s_ring_state |= S_RING_TX_HIWAT; 3899 cookie = (mac_tx_cookie_t)ringp; 3900 ringp->s_ring_hiwat_cnt++; 3901 if (ringp->s_ring_count > 3902 ringp->s_ring_tx_max_q_cnt) { 3903 /* increment freed stats */ 3904 ringp->s_ring_drops += cnt; 3905 /* 3906 * b_prev may be set to the fanout hint 3907 * hence can't use freemsg directly 3908 */ 3909 mac_pkt_drop(NULL, NULL, 3910 mp_chain, B_FALSE); 3911 DTRACE_PROBE1(tx_queued_hiwat, 3912 mac_soft_ring_t *, ringp); 3913 enqueue = B_FALSE; 3914 } 3915 } 3916 if (enqueue) { 3917 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, 3918 tail, cnt, sz); 3919 } 3920 } 3921 if (wakeup_worker) 3922 cv_signal(&ringp->s_ring_async); 3923 } 3924 return (cookie); 3925 } 3926 3927 3928 /* 3929 * mac_tx_soft_ring_process 3930 * 3931 * This routine is called when fanning out outgoing traffic among 3932 * multipe Tx rings. 3933 * Note that a soft ring is associated with a h/w Tx ring. 3934 */ 3935 mac_tx_cookie_t 3936 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain, 3937 uint16_t flag, mblk_t **ret_mp) 3938 { 3939 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set; 3940 int cnt; 3941 size_t sz; 3942 mblk_t *tail; 3943 mac_tx_cookie_t cookie = NULL; 3944 3945 ASSERT(ringp != NULL); 3946 ASSERT(mp_chain != NULL); 3947 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock)); 3948 /* 3949 * The following modes can come here: SRS_TX_BW_FANOUT, 3950 * SRS_TX_FANOUT, SRS_TX_AGGR, SRS_TX_BW_AGGR. 3951 */ 3952 ASSERT(MAC_TX_SOFT_RINGS(mac_srs)); 3953 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT || 3954 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT || 3955 mac_srs->srs_tx.st_mode == SRS_TX_AGGR || 3956 mac_srs->srs_tx.st_mode == SRS_TX_BW_AGGR); 3957 3958 if (ringp->s_ring_type & ST_RING_WORKER_ONLY) { 3959 /* Serialization mode */ 3960 3961 mutex_enter(&ringp->s_ring_lock); 3962 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) { 3963 cookie = mac_tx_sring_enqueue(ringp, mp_chain, 3964 flag, ret_mp); 3965 mutex_exit(&ringp->s_ring_lock); 3966 return (cookie); 3967 } 3968 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz); 3969 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); 3970 if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) { 3971 /* 3972 * If ring is blocked due to lack of Tx 3973 * descs, just return. Worker thread 3974 * will get scheduled when Tx desc's 3975 * become available. 3976 */ 3977 mutex_exit(&ringp->s_ring_lock); 3978 return (cookie); 3979 } 3980 mac_soft_ring_worker_wakeup(ringp); 3981 mutex_exit(&ringp->s_ring_lock); 3982 return (cookie); 3983 } else { 3984 /* Default fanout mode */ 3985 /* 3986 * S_RING_BLOCKED is set when underlying NIC runs 3987 * out of Tx descs and messages start getting 3988 * queued. It won't get reset until 3989 * tx_srs_drain() completely drains out the 3990 * messages. 3991 */ 3992 mac_tx_stats_t stats; 3993 3994 if (ringp->s_ring_state & S_RING_ENQUEUED) { 3995 /* Tx descs/resources not available */ 3996 mutex_enter(&ringp->s_ring_lock); 3997 if (ringp->s_ring_state & S_RING_ENQUEUED) { 3998 cookie = mac_tx_sring_enqueue(ringp, mp_chain, 3999 flag, ret_mp); 4000 mutex_exit(&ringp->s_ring_lock); 4001 return (cookie); 4002 } 4003 /* 4004 * While we were computing mblk count, the 4005 * flow control condition got relieved. 4006 * Continue with the transmission. 4007 */ 4008 mutex_exit(&ringp->s_ring_lock); 4009 } 4010 4011 mp_chain = mac_tx_send(ringp->s_ring_tx_arg1, 4012 ringp->s_ring_tx_arg2, mp_chain, &stats); 4013 4014 /* 4015 * Multiple threads could be here sending packets. 4016 * Under such conditions, it is not possible to 4017 * automically set S_RING_BLOCKED bit to indicate 4018 * out of tx desc condition. To atomically set 4019 * this, we queue the returned packet and do 4020 * the setting of S_RING_BLOCKED in 4021 * mac_tx_soft_ring_drain(). 4022 */ 4023 if (mp_chain != NULL) { 4024 mutex_enter(&ringp->s_ring_lock); 4025 cookie = 4026 mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp); 4027 mutex_exit(&ringp->s_ring_lock); 4028 return (cookie); 4029 } 4030 SRS_TX_STATS_UPDATE(mac_srs, &stats); 4031 SOFTRING_TX_STATS_UPDATE(ringp, &stats); 4032 4033 return (NULL); 4034 } 4035 } 4036