1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/callb.h> 28 #include <sys/sdt.h> 29 #include <sys/strsubr.h> 30 #include <sys/strsun.h> 31 #include <sys/vlan.h> 32 #include <inet/ipsec_impl.h> 33 #include <inet/ip_impl.h> 34 #include <inet/sadb.h> 35 #include <inet/ipsecesp.h> 36 #include <inet/ipsecah.h> 37 38 #include <sys/mac_impl.h> 39 #include <sys/mac_client_impl.h> 40 #include <sys/mac_client_priv.h> 41 #include <sys/mac_soft_ring.h> 42 #include <sys/mac_flow_impl.h> 43 44 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 45 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 46 static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 47 static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 48 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 49 50 static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 51 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 52 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 53 static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 54 static void mac_srs_add_glist(mac_soft_ring_set_t *); 55 static void mac_srs_remove_glist(mac_soft_ring_set_t *); 56 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 57 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 58 59 static int mac_compute_soft_ring_count(flow_entry_t *, int); 60 static void mac_walk_srs_and_bind(int); 61 static void mac_walk_srs_and_unbind(int); 62 63 extern mac_group_t *mac_reserve_rx_group(mac_client_impl_t *, uint8_t *, 64 mac_rx_group_reserve_type_t); 65 extern void mac_release_rx_group(mac_client_impl_t *, mac_group_t *); 66 67 extern boolean_t mac_latency_optimize; 68 69 static kmem_cache_t *mac_srs_cache; 70 kmem_cache_t *mac_soft_ring_cache; 71 72 /* 73 * The duration in msec we wait before signalling the soft ring 74 * worker thread in case packets get queued. 75 */ 76 uint32_t mac_soft_ring_worker_wait = 0; 77 78 /* 79 * A global tunable for turning polling on/off. By default, dynamic 80 * polling is always on and is always very beneficial. It should be 81 * turned off with absolute care and for the rare workload (very 82 * low latency sensitive traffic). 83 */ 84 int mac_poll_enable = B_TRUE; 85 86 /* 87 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 88 * Large values could end up in consuming lot of system memory and cause 89 * system hang. 90 */ 91 int mac_soft_ring_max_q_cnt = 1024; 92 int mac_soft_ring_min_q_cnt = 256; 93 int mac_soft_ring_poll_thres = 16; 94 95 /* 96 * Default value of number of TX rings to be assigned to a MAC client. 97 * If less than 'mac_tx_ring_count' worth of Tx rings is available, then 98 * as many as is available will be assigned to the newly created MAC client. 99 * If no TX rings are available, then MAC client(s) will be assigned the 100 * default Tx ring. Default Tx ring can be shared among multiple MAC clients. 101 */ 102 uint32_t mac_tx_ring_count = 32; 103 boolean_t mac_tx_serialize = B_FALSE; 104 105 /* 106 * mac_tx_srs_hiwat is the queue depth threshold at which callers of 107 * mac_tx() will be notified of flow control condition. 108 * 109 * TCP does not honour flow control condition sent up by mac_tx(). 110 * Thus provision is made for TCP to allow more packets to be queued 111 * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 112 * 113 * Note that mac_tx_srs_hiwat is always be lesser than 114 * mac_tx_srs_max_q_cnt. 115 */ 116 uint32_t mac_tx_srs_max_q_cnt = 100000; 117 uint32_t mac_tx_srs_hiwat = 1000; 118 119 /* 120 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 121 * 122 * Global tunables that determines the number of soft rings to be used for 123 * fanning out incoming traffic on a link. These count will be used only 124 * when no explicit set of CPUs was assigned to the data-links. 125 * 126 * mac_rx_soft_ring_count tunable will come into effect only if 127 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 128 * default only for sun4v platforms. 129 * 130 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 131 * 10Gbps link and is not dependent upon mac_soft_ring_enable. 132 * 133 * The number of soft rings for fanout for a link or a flow is determined 134 * by mac_compute_soft_ring_count() routine. This routine will take into 135 * account mac_soft_ring_enable, mac_rx_soft_ring_count and 136 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 137 * 138 * If a bandwidth is specified, the determination of the number of soft 139 * rings is based on specified bandwidth, CPU speed and number of CPUs in 140 * the system. 141 */ 142 uint_t mac_rx_soft_ring_count = 8; 143 uint_t mac_rx_soft_ring_10gig_count = 8; 144 145 /* 146 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 147 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 148 * list is used to walk the list of all MAC threads when a CPU is 149 * coming online or going offline. 150 */ 151 static mac_soft_ring_set_t *mac_srs_g_list = NULL; 152 static krwlock_t mac_srs_g_lock; 153 154 /* 155 * Whether the SRS threads should be bound, or not. 156 */ 157 boolean_t mac_srs_thread_bind = B_TRUE; 158 159 /* 160 * CPU to fallback to, used by mac_next_bind_cpu(). 161 */ 162 processorid_t srs_bind_cpu = 0; 163 164 /* 165 * If cpu bindings are specified by user, then Tx SRS and its soft 166 * rings should also be bound to the CPUs specified by user. The 167 * CPUs for Tx bindings are at the end of the cpu list provided by 168 * the user. If enough CPUs are not available (for Tx and Rx 169 * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 170 */ 171 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 172 processorid_t cpuid; \ 173 int i, j; \ 174 mac_soft_ring_t *softring; \ 175 \ 176 cpuid = mrp->mrp_cpu[mrp->mrp_ncpus - 1]; \ 177 mac_srs_worker_bind(mac_tx_srs, cpuid); \ 178 if (TX_MULTI_RING_MODE(mac_tx_srs)) { \ 179 j = mrp->mrp_ncpus - 1; \ 180 for (i = 0; \ 181 i < mac_tx_srs->srs_oth_ring_count; i++, j--) { \ 182 if (j < 0) \ 183 j = mrp->mrp_ncpus - 1; \ 184 cpuid = mrp->mrp_cpu[j]; \ 185 softring = mac_tx_srs->srs_oth_soft_rings[i]; \ 186 (void) mac_soft_ring_bind(softring, cpuid); \ 187 } \ 188 } \ 189 } 190 191 /* INIT and FINI ROUTINES */ 192 193 void 194 mac_soft_ring_init(void) 195 { 196 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 197 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 198 199 mac_srs_cache = kmem_cache_create("mac_srs_cache", 200 sizeof (mac_soft_ring_set_t), 201 64, NULL, NULL, NULL, NULL, NULL, 0); 202 203 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 204 mutex_enter(&cpu_lock); 205 register_cpu_setup_func(mac_srs_cpu_setup, NULL); 206 mutex_exit(&cpu_lock); 207 } 208 209 void 210 mac_soft_ring_finish(void) 211 { 212 mutex_enter(&cpu_lock); 213 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 214 mutex_exit(&cpu_lock); 215 rw_destroy(&mac_srs_g_lock); 216 kmem_cache_destroy(mac_soft_ring_cache); 217 kmem_cache_destroy(mac_srs_cache); 218 } 219 220 static void 221 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs, boolean_t release_tx_ring) 222 { 223 mac_soft_ring_t *softring, *next, *head; 224 225 /* 226 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 227 * DR. The callbacks from DR are called with cpu_lock held, and hence 228 * can't wait to grab the mac perimeter. The soft ring list is hence 229 * protected for read access by srs_lock. Changing the soft ring list 230 * needs the mac perimeter and the srs_lock. 231 */ 232 mutex_enter(&mac_srs->srs_lock); 233 234 head = mac_srs->srs_soft_ring_head; 235 mac_srs->srs_soft_ring_head = NULL; 236 mac_srs->srs_soft_ring_tail = NULL; 237 mac_srs->srs_soft_ring_count = 0; 238 239 mutex_exit(&mac_srs->srs_lock); 240 241 for (softring = head; softring != NULL; softring = next) { 242 next = softring->s_ring_next; 243 mac_soft_ring_free(softring, release_tx_ring); 244 } 245 } 246 247 static void 248 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 249 { 250 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 251 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 252 253 rw_enter(&mac_srs_g_lock, RW_WRITER); 254 mutex_enter(&mac_srs->srs_lock); 255 256 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 257 258 if (mac_srs_g_list == NULL) { 259 mac_srs_g_list = mac_srs; 260 } else { 261 mac_srs->srs_next = mac_srs_g_list; 262 mac_srs_g_list->srs_prev = mac_srs; 263 mac_srs->srs_prev = NULL; 264 mac_srs_g_list = mac_srs; 265 } 266 mac_srs->srs_state |= SRS_IN_GLIST; 267 268 mutex_exit(&mac_srs->srs_lock); 269 rw_exit(&mac_srs_g_lock); 270 } 271 272 static void 273 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 274 { 275 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 276 277 rw_enter(&mac_srs_g_lock, RW_WRITER); 278 mutex_enter(&mac_srs->srs_lock); 279 280 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 281 282 if (mac_srs == mac_srs_g_list) { 283 mac_srs_g_list = mac_srs->srs_next; 284 if (mac_srs_g_list != NULL) 285 mac_srs_g_list->srs_prev = NULL; 286 } else { 287 mac_srs->srs_prev->srs_next = mac_srs->srs_next; 288 if (mac_srs->srs_next != NULL) 289 mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 290 } 291 mac_srs->srs_state &= ~SRS_IN_GLIST; 292 293 mutex_exit(&mac_srs->srs_lock); 294 rw_exit(&mac_srs_g_lock); 295 } 296 297 /* POLLING SETUP AND TEAR DOWN ROUTINES */ 298 299 /* 300 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 301 * 302 * These routines are used to call back into the upper layer 303 * (primarily TCP squeue) to stop polling the soft rings or 304 * restart polling. 305 */ 306 void 307 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 308 mac_soft_ring_set_t *mac_srs) 309 { 310 mac_soft_ring_t *softring; 311 312 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 313 314 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 315 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 316 return; 317 } 318 319 for (softring = mac_srs->srs_soft_ring_head; 320 softring != NULL; softring = softring->s_ring_next) { 321 if ((softring->s_ring_type & ST_RING_TCP) && 322 (softring->s_ring_rx_arg2 != NULL)) { 323 mcip->mci_resource_quiesce(mcip->mci_resource_arg, 324 softring->s_ring_rx_arg2); 325 } 326 } 327 } 328 329 void 330 mac_srs_client_poll_restart(mac_client_impl_t *mcip, 331 mac_soft_ring_set_t *mac_srs) 332 { 333 mac_soft_ring_t *softring; 334 335 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 336 337 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 338 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 339 return; 340 } 341 342 for (softring = mac_srs->srs_soft_ring_head; 343 softring != NULL; softring = softring->s_ring_next) { 344 if ((softring->s_ring_type & ST_RING_TCP) && 345 (softring->s_ring_rx_arg2 != NULL)) { 346 mcip->mci_resource_restart(mcip->mci_resource_arg, 347 softring->s_ring_rx_arg2); 348 } 349 } 350 } 351 352 /* 353 * Register the given SRS and associated soft rings with the consumer and 354 * enable the polling interface used by the consumer.(i.e IP) over this 355 * SRS and associated soft rings. 356 */ 357 void 358 mac_srs_client_poll_enable(mac_client_impl_t *mcip, 359 mac_soft_ring_set_t *mac_srs) 360 { 361 mac_rx_fifo_t mrf; 362 mac_soft_ring_t *softring; 363 364 ASSERT(mac_srs->srs_mcip == mcip); 365 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 366 367 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 368 return; 369 370 bzero(&mrf, sizeof (mac_rx_fifo_t)); 371 mrf.mrf_type = MAC_RX_FIFO; 372 373 /* 374 * A SRS is capable of acting as a soft ring for cases 375 * where no fanout is needed. This is the case for userland 376 * flows. 377 */ 378 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 379 return; 380 381 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 382 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 383 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 384 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 385 386 softring = mac_srs->srs_soft_ring_head; 387 while (softring != NULL) { 388 if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 389 /* 390 * TCP and UDP support DLS bypass. Squeue polling 391 * support implies DLS bypass since the squeue poll 392 * path does not have DLS processing. 393 */ 394 mac_soft_ring_dls_bypass(softring, 395 mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 396 } 397 /* 398 * Non-TCP protocols don't support squeues. Hence we don't 399 * make any ring addition callbacks for non-TCP rings 400 */ 401 if (!(softring->s_ring_type & ST_RING_TCP)) { 402 softring->s_ring_rx_arg2 = NULL; 403 softring = softring->s_ring_next; 404 continue; 405 } 406 mrf.mrf_rx_arg = softring; 407 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 408 mrf.mrf_cpu_id = softring->s_ring_cpuid; 409 mrf.mrf_flow_priority = mac_srs->srs_pri; 410 411 softring->s_ring_rx_arg2 = mcip->mci_resource_add( 412 mcip->mci_resource_arg, (mac_resource_t *)&mrf); 413 414 softring = softring->s_ring_next; 415 } 416 } 417 418 /* 419 * Unregister the given SRS and associated soft rings with the consumer and 420 * disable the polling interface used by the consumer.(i.e IP) over this 421 * SRS and associated soft rings. 422 */ 423 void 424 mac_srs_client_poll_disable(mac_client_impl_t *mcip, 425 mac_soft_ring_set_t *mac_srs) 426 { 427 mac_soft_ring_t *softring; 428 429 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 430 431 /* 432 * A SRS is capable of acting as a soft ring for cases 433 * where no protocol fanout is needed. This is the case 434 * for userland flows. Nothing to do here. 435 */ 436 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 437 return; 438 439 mutex_enter(&mac_srs->srs_lock); 440 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 441 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 442 mutex_exit(&mac_srs->srs_lock); 443 return; 444 } 445 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 446 mutex_exit(&mac_srs->srs_lock); 447 448 /* 449 * DLS bypass is now disabled in the case of both TCP and UDP. 450 * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 451 * callback. In addition, in the case of TCP, invoke IP's callback 452 * for ring removal. 453 */ 454 for (softring = mac_srs->srs_soft_ring_head; 455 softring != NULL; softring = softring->s_ring_next) { 456 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 457 continue; 458 459 if ((softring->s_ring_type & ST_RING_TCP) && 460 softring->s_ring_rx_arg2 != NULL) { 461 mcip->mci_resource_remove(mcip->mci_resource_arg, 462 softring->s_ring_rx_arg2); 463 } 464 465 mutex_enter(&softring->s_ring_lock); 466 while (softring->s_ring_state & S_RING_PROC) { 467 softring->s_ring_state |= S_RING_CLIENT_WAIT; 468 cv_wait(&softring->s_ring_client_cv, 469 &softring->s_ring_lock); 470 } 471 softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 472 softring->s_ring_rx_arg2 = NULL; 473 softring->s_ring_rx_func = mac_rx_deliver; 474 softring->s_ring_rx_arg1 = mcip; 475 mutex_exit(&softring->s_ring_lock); 476 } 477 } 478 479 /* 480 * Enable or disable poll capability of the SRS on the underlying Rx ring. 481 * 482 * There is a need to enable or disable the poll capability of an SRS over an 483 * Rx ring depending on the number of mac clients sharing the ring and also 484 * whether user flows are configured on it. However the poll state is actively 485 * manipulated by the SRS worker and poll threads and uncoordinated changes by 486 * yet another thread to the underlying capability can surprise them leading 487 * to assert failures. Instead we quiesce the SRS, make the changes and then 488 * restart the SRS. 489 */ 490 static void 491 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 492 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 493 { 494 boolean_t need_restart = B_FALSE; 495 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 496 mac_ring_t *ring; 497 498 if (!SRS_QUIESCED(mac_srs)) { 499 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 500 need_restart = B_TRUE; 501 } 502 503 ring = mac_srs->srs_ring; 504 if ((ring != NULL) && 505 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 506 if (turn_off_poll_capab) 507 mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 508 else if (mac_poll_enable) 509 mac_srs->srs_state |= SRS_POLLING_CAPAB; 510 } 511 srs_rx->sr_lower_proc = rx_func; 512 513 if (need_restart) 514 mac_rx_srs_restart(mac_srs); 515 } 516 517 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 518 519 /* 520 * Return the next CPU to be used to bind a MAC kernel thread. 521 */ 522 static processorid_t 523 mac_next_bind_cpu(void) 524 { 525 static processorid_t srs_curr_cpu = -1; 526 cpu_t *cp; 527 528 ASSERT(MUTEX_HELD(&cpu_lock)); 529 530 srs_curr_cpu++; 531 cp = cpu_get(srs_curr_cpu); 532 if (cp == NULL || !cpu_is_online(cp)) 533 srs_curr_cpu = srs_bind_cpu; 534 535 return (srs_curr_cpu); 536 } 537 538 /* ARGSUSED */ 539 static int 540 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 541 { 542 ASSERT(MUTEX_HELD(&cpu_lock)); 543 switch (what) { 544 case CPU_CONFIG: 545 case CPU_ON: 546 case CPU_CPUPART_IN: 547 mac_walk_srs_and_bind(id); 548 break; 549 550 case CPU_UNCONFIG: 551 case CPU_OFF: 552 case CPU_CPUPART_OUT: 553 mac_walk_srs_and_unbind(id); 554 break; 555 556 default: 557 break; 558 } 559 return (0); 560 } 561 562 /* 563 * mac_compute_soft_ring_count(): 564 * 565 * This routine computes the number of soft rings needed to handle incoming 566 * load given a flow_entry. 567 * 568 * The routine does the following: 569 * 1) soft rings will be created if mac_soft_ring_enable is set. 570 * 2) If the underlying link is a 10Gbps link, then soft rings will be 571 * created even if mac_soft_ring_enable is not set. The number of soft 572 * rings, so created, will equal mac_rx_soft_ring_10gig_count. 573 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 574 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 575 * 10Gbps link. 576 * 577 * If a bandwidth limit is specified, the number that gets computed is 578 * dependent upon CPU speed, the number of Rx rings configured, and 579 * the bandwidth limit. 580 * If more Rx rings are available, less number of soft rings is needed. 581 * 582 * mac_use_bw_heuristic is another "hidden" variable that can be used to 583 * override the default use of soft ring count computation. Depending upon 584 * the usefulness of it, mac_use_bw_heuristic can later be made into a 585 * data-link property or removed altogether. 586 * 587 * TODO: Cleanup and tighten some of the assumptions. 588 */ 589 boolean_t mac_use_bw_heuristic = B_TRUE; 590 static int 591 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt) 592 { 593 uint64_t cpu_speed, bw = 0; 594 int srings = 0; 595 boolean_t bw_enabled = B_FALSE; 596 597 ASSERT(!(flent->fe_type & FLOW_USER)); 598 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 599 mac_use_bw_heuristic) { 600 /* bandwidth enabled */ 601 bw_enabled = B_TRUE; 602 bw = flent->fe_resource_props.mrp_maxbw; 603 } 604 if (!bw_enabled) { 605 /* No bandwidth enabled */ 606 if (mac_soft_ring_enable) 607 srings = mac_rx_soft_ring_count; 608 609 /* Is this a 10Gig link? */ 610 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 611 MAC_STAT_IFSPEED); 612 /* convert to Mbps */ 613 if (((flent->fe_nic_speed)/1000000) > 1000 && 614 mac_rx_soft_ring_10gig_count > 0) { 615 /* This is a 10Gig link */ 616 srings = mac_rx_soft_ring_10gig_count; 617 /* 618 * Use 2 times mac_rx_soft_ring_10gig_count for 619 * sun4v systems. 620 */ 621 if (mac_soft_ring_enable) 622 srings = srings * 2; 623 } 624 } else { 625 /* 626 * Soft ring computation using CPU speed and specified 627 * bandwidth limit. 628 */ 629 /* Assumption: all CPUs have the same frequency */ 630 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 631 632 /* cpu_speed is in MHz; make bw in units of Mbps. */ 633 bw = bw/1000000; 634 635 if (bw >= 1000) { 636 /* 637 * bw is greater than or equal to 1Gbps. 638 * The number of soft rings required is a function 639 * of bandwidth and CPU speed. To keep this simple, 640 * let's use this rule: 1GHz CPU can handle 1Gbps. 641 * If bw is less than 1 Gbps, then there is no need 642 * for soft rings. Assumption is that CPU speeds 643 * (on modern systems) are at least 1GHz. 644 */ 645 srings = bw/cpu_speed; 646 if (srings <= 1 && mac_soft_ring_enable) { 647 /* 648 * Give at least 2 soft rings 649 * for sun4v systems 650 */ 651 srings = 2; 652 } 653 } 654 } 655 /* 656 * If the flent has multiple Rx SRSs, then each SRS need not 657 * have that many soft rings on top of it. The number of 658 * soft rings for each Rx SRS is found by dividing srings by 659 * rx_srs_cnt. 660 */ 661 if (rx_srs_cnt > 1) { 662 int remainder; 663 664 remainder = srings%rx_srs_cnt; 665 srings = srings/rx_srs_cnt; 666 if (remainder != 0) 667 srings++; 668 /* 669 * Fanning out to 1 soft ring is not very useful. 670 * Set it as well to 0 and mac_srs_fanout_init() 671 * will take care of creating a single soft ring 672 * for proto fanout. 673 */ 674 if (srings == 1) 675 srings = 0; 676 } 677 /* Do some more massaging */ 678 srings = min(srings, ncpus); 679 srings = min(srings, MAX_SR_FANOUT); 680 return (srings); 681 } 682 683 /* 684 * Assignment of user specified CPUs to a link. 685 * 686 * Minimum CPUs required to get an optimal assignmet: 687 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 688 * flag is set -- one for polling, one for fanout soft ring. 689 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 690 * for polling, one for SRS worker thread and one for fanout soft ring. 691 * 692 * The CPUs needed for Tx side is equal to the number of Tx rings 693 * the link is using. 694 * 695 * mac_flow_user_cpu_init() categorizes the CPU assignment depending 696 * upon the number of CPUs in 3 different buckets. 697 * 698 * In the first bucket, the most optimal case is handled. The user has 699 * passed enough number of CPUs and every thread gets its own CPU. 700 * 701 * The second and third are the sub-optimal cases. Enough CPUs are not 702 * available. 703 * 704 * The second bucket handles the case where atleast one distinct CPU is 705 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 706 * SRS or soft rings). 707 * 708 * In the third case (worst case scenario), specified CPU count is less 709 * than the Rx rings configured for the link. In this case, we round 710 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 711 */ 712 static void 713 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 714 { 715 mac_soft_ring_set_t *rx_srs, *tx_srs; 716 int i, srs_cnt; 717 mac_cpus_t *srs_cpu; 718 int no_of_cpus, cpu_cnt; 719 int rx_srs_cnt, reqd_rx_cpu_cnt; 720 int fanout_cpu_cnt, reqd_tx_cpu_cnt; 721 int reqd_poll_worker_cnt, fanout_cnt_per_srs; 722 723 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 724 /* 725 * The check for nbc_ncpus to be within limits for 726 * the user specified case was done earlier and if 727 * not within limits, an error would have been 728 * returned to the user. 729 */ 730 ASSERT(mrp->mrp_ncpus > 0 && mrp->mrp_ncpus <= MAX_SR_FANOUT); 731 732 no_of_cpus = mrp->mrp_ncpus; 733 734 if (mrp->mrp_intr_cpu != -1) { 735 /* 736 * interrupt has been re-targetted. Poll 737 * thread needs to be bound to interrupt 738 * CPU. Presently only fixed interrupts 739 * are re-targetted, MSI-x aren't. 740 * 741 * Find where in the list is the intr 742 * CPU and swap it with the first one. 743 * We will be using the first CPU in the 744 * list for poll. 745 */ 746 for (i = 0; i < no_of_cpus; i++) { 747 if (mrp->mrp_cpu[i] == mrp->mrp_intr_cpu) 748 break; 749 } 750 mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 751 mrp->mrp_cpu[0] = mrp->mrp_intr_cpu; 752 } 753 754 /* 755 * Requirements: 756 * The number of CPUs that each Rx ring needs is dependent 757 * upon mac_latency_optimize flag. 758 * 1) If set, atleast 2 CPUs are needed -- one for 759 * polling, one for fanout soft ring. 760 * 2) If not set, then atleast 3 CPUs are needed -- one 761 * for polling, one for srs worker thread, and one for 762 * fanout soft ring. 763 */ 764 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 765 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 766 reqd_rx_cpu_cnt = mac_latency_optimize ? 767 (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 768 769 /* How many CPUs are needed for Tx side? */ 770 tx_srs = flent->fe_tx_srs; 771 reqd_tx_cpu_cnt = TX_MULTI_RING_MODE(tx_srs) ? 772 tx_srs->srs_oth_ring_count : 1; 773 774 /* CPUs needed for Rx SRSes poll and worker threads */ 775 reqd_poll_worker_cnt = mac_latency_optimize ? 776 rx_srs_cnt : rx_srs_cnt * 2; 777 778 /* Has the user provided enough CPUs? */ 779 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 780 /* 781 * Best case scenario. There is enough CPUs. All 782 * Rx rings will get their own set of CPUs plus 783 * Tx soft rings will get their own. 784 */ 785 /* 786 * fanout_cpu_cnt is the number of CPUs available 787 * for Rx side fanout soft rings. 788 */ 789 fanout_cpu_cnt = no_of_cpus - 790 reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 791 792 /* 793 * Divide fanout_cpu_cnt by rx_srs_cnt to find 794 * out how many fanout soft rings each Rx SRS 795 * can have. 796 */ 797 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 798 799 /* Do the assignment for the default Rx ring */ 800 cpu_cnt = 0; 801 rx_srs = flent->fe_rx_srs[0]; 802 ASSERT(rx_srs->srs_ring == NULL); 803 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 804 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 805 srs_cpu = &rx_srs->srs_cpu; 806 srs_cpu->mc_ncpus = no_of_cpus; 807 bcopy(mrp->mrp_cpu, 808 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 809 srs_cpu->mc_fanout_cnt = fanout_cnt_per_srs; 810 srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt++]; 811 srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 812 srs_cpu->mc_workerid = srs_cpu->mc_pollid; 813 if (!mac_latency_optimize) 814 srs_cpu->mc_workerid = mrp->mrp_cpu[cpu_cnt++]; 815 for (i = 0; i < fanout_cnt_per_srs; i++) 816 srs_cpu->mc_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 817 818 /* Do the assignment for h/w Rx SRSes */ 819 if (flent->fe_rx_srs_cnt > 1) { 820 cpu_cnt = 0; 821 for (srs_cnt = 1; 822 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 823 rx_srs = flent->fe_rx_srs[srs_cnt]; 824 ASSERT(rx_srs->srs_ring != NULL); 825 if (rx_srs->srs_fanout_state == 826 SRS_FANOUT_INIT) { 827 rx_srs->srs_fanout_state = 828 SRS_FANOUT_REINIT; 829 } 830 srs_cpu = &rx_srs->srs_cpu; 831 srs_cpu->mc_ncpus = no_of_cpus; 832 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 833 sizeof (srs_cpu->mc_cpus)); 834 srs_cpu->mc_fanout_cnt = fanout_cnt_per_srs; 835 /* The first CPU in the list is the intr CPU */ 836 srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt++]; 837 srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 838 srs_cpu->mc_workerid = srs_cpu->mc_pollid; 839 if (!mac_latency_optimize) { 840 srs_cpu->mc_workerid = 841 mrp->mrp_cpu[cpu_cnt++]; 842 } 843 for (i = 0; i < fanout_cnt_per_srs; i++) { 844 srs_cpu->mc_fanout_cpus[i] = 845 mrp->mrp_cpu[cpu_cnt++]; 846 } 847 ASSERT(cpu_cnt <= no_of_cpus); 848 } 849 } 850 return; 851 } 852 853 /* 854 * Sub-optimal case. 855 * We have the following information: 856 * no_of_cpus - no. of cpus that user passed. 857 * rx_srs_cnt - no. of rx rings. 858 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 859 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 860 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 861 */ 862 /* 863 * If we bind the Rx fanout soft rings to the same CPUs 864 * as poll/worker, would that be enough? 865 */ 866 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 867 boolean_t worker_assign = B_FALSE; 868 869 /* 870 * If mac_latency_optimize is not set, are there 871 * enough CPUs to assign a CPU for worker also? 872 */ 873 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 874 worker_assign = B_TRUE; 875 /* 876 * Zero'th Rx SRS is the default Rx ring. It is not 877 * associated with h/w Rx ring. 878 */ 879 rx_srs = flent->fe_rx_srs[0]; 880 ASSERT(rx_srs->srs_ring == NULL); 881 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 882 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 883 cpu_cnt = 0; 884 srs_cpu = &rx_srs->srs_cpu; 885 srs_cpu->mc_ncpus = no_of_cpus; 886 bcopy(mrp->mrp_cpu, 887 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 888 srs_cpu->mc_fanout_cnt = 1; 889 srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt++]; 890 srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 891 srs_cpu->mc_workerid = srs_cpu->mc_pollid; 892 if (!mac_latency_optimize && worker_assign) 893 srs_cpu->mc_workerid = mrp->mrp_cpu[cpu_cnt++]; 894 srs_cpu->mc_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 895 896 /* Do CPU bindings for SRSes having h/w Rx rings */ 897 if (flent->fe_rx_srs_cnt > 1) { 898 cpu_cnt = 0; 899 for (srs_cnt = 1; 900 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 901 rx_srs = flent->fe_rx_srs[srs_cnt]; 902 ASSERT(rx_srs->srs_ring != NULL); 903 if (rx_srs->srs_fanout_state == 904 SRS_FANOUT_INIT) { 905 rx_srs->srs_fanout_state = 906 SRS_FANOUT_REINIT; 907 } 908 srs_cpu = &rx_srs->srs_cpu; 909 srs_cpu->mc_ncpus = no_of_cpus; 910 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 911 sizeof (srs_cpu->mc_cpus)); 912 srs_cpu->mc_pollid = 913 mrp->mrp_cpu[cpu_cnt]; 914 srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 915 srs_cpu->mc_workerid = srs_cpu->mc_pollid; 916 if (!mac_latency_optimize && worker_assign) { 917 srs_cpu->mc_workerid = 918 mrp->mrp_cpu[++cpu_cnt]; 919 } 920 srs_cpu->mc_fanout_cnt = 1; 921 srs_cpu->mc_fanout_cpus[0] = 922 mrp->mrp_cpu[cpu_cnt]; 923 cpu_cnt++; 924 ASSERT(cpu_cnt <= no_of_cpus); 925 } 926 } 927 return; 928 } 929 930 /* 931 * Real sub-optimal case. Not enough CPUs for poll and 932 * Tx soft rings. Do a round robin assignment where 933 * each Rx SRS will get the same CPU for poll, worker 934 * and fanout soft ring. 935 */ 936 cpu_cnt = 0; 937 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 938 rx_srs = flent->fe_rx_srs[srs_cnt]; 939 srs_cpu = &rx_srs->srs_cpu; 940 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 941 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 942 srs_cpu->mc_ncpus = no_of_cpus; 943 bcopy(mrp->mrp_cpu, 944 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 945 srs_cpu->mc_fanout_cnt = 1; 946 srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt]; 947 srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 948 srs_cpu->mc_workerid = mrp->mrp_cpu[cpu_cnt]; 949 srs_cpu->mc_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 950 if (++cpu_cnt >= no_of_cpus) 951 cpu_cnt = 0; 952 } 953 } 954 955 /* 956 * mac_flow_cpu_init(): 957 * 958 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 959 * the CPU binding information in srs_cpu for all Rx SRSes associated 960 * with a flent. 961 */ 962 static void 963 mac_flow_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 964 { 965 mac_soft_ring_set_t *rx_srs; 966 processorid_t cpuid; 967 int j, srs_cnt, soft_ring_cnt = 0; 968 mac_cpus_t *srs_cpu; 969 970 if (mrp->mrp_mask & MRP_CPUS_USERSPEC) { 971 mac_flow_user_cpu_init(flent, mrp); 972 } else { 973 /* 974 * Compute the number of soft rings needed on top for each Rx 975 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 976 * associated with h/w Rx rings. Soft ring count needed for 977 * each h/w Rx SRS is computed and the same is applied to 978 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 979 * is the software classified Rx SRS. 980 */ 981 soft_ring_cnt = mac_compute_soft_ring_count(flent, 982 flent->fe_rx_srs_cnt - 1); 983 if (soft_ring_cnt == 0) { 984 /* 985 * Even when soft_ring_cnt is 0, we still need 986 * to create a soft ring for TCP, UDP and 987 * OTHER. So set it to 1. 988 */ 989 soft_ring_cnt = 1; 990 } 991 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 992 rx_srs = flent->fe_rx_srs[srs_cnt]; 993 srs_cpu = &rx_srs->srs_cpu; 994 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) { 995 if (soft_ring_cnt == srs_cpu->mc_fanout_cnt) 996 continue; 997 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 998 } 999 srs_cpu->mc_ncpus = soft_ring_cnt; 1000 srs_cpu->mc_fanout_cnt = soft_ring_cnt; 1001 mutex_enter(&cpu_lock); 1002 for (j = 0; j < soft_ring_cnt; j++) { 1003 cpuid = mac_next_bind_cpu(); 1004 srs_cpu->mc_cpus[j] = cpuid; 1005 srs_cpu->mc_fanout_cpus[j] = cpuid; 1006 } 1007 cpuid = mac_next_bind_cpu(); 1008 srs_cpu->mc_pollid = cpuid; 1009 /* increment ncpus to account for polling cpu */ 1010 srs_cpu->mc_ncpus++; 1011 srs_cpu->mc_cpus[j++] = cpuid; 1012 if (!mac_latency_optimize) { 1013 cpuid = mac_next_bind_cpu(); 1014 srs_cpu->mc_ncpus++; 1015 srs_cpu->mc_cpus[j++] = cpuid; 1016 } 1017 srs_cpu->mc_workerid = cpuid; 1018 mutex_exit(&cpu_lock); 1019 } 1020 } 1021 } 1022 1023 /* 1024 * DATAPATH SETUP ROUTINES 1025 * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1026 */ 1027 1028 static void 1029 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1030 { 1031 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1032 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, KM_SLEEP); 1033 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1034 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, KM_SLEEP); 1035 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1036 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, KM_SLEEP); 1037 } 1038 1039 static void 1040 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1041 { 1042 cpu_t *cp; 1043 boolean_t clear = B_FALSE; 1044 1045 ASSERT(MUTEX_HELD(&cpu_lock)); 1046 1047 if (!mac_srs_thread_bind) 1048 return; 1049 1050 cp = cpu_get(cpuid); 1051 if (cp == NULL || !cpu_is_online(cp)) 1052 return; 1053 1054 mutex_enter(&mac_srs->srs_lock); 1055 mac_srs->srs_state |= SRS_WORKER_BOUND; 1056 if (mac_srs->srs_worker_cpuid != -1) 1057 clear = B_TRUE; 1058 mac_srs->srs_worker_cpuid = cpuid; 1059 mutex_exit(&mac_srs->srs_lock); 1060 1061 if (clear) 1062 thread_affinity_clear(mac_srs->srs_worker); 1063 1064 thread_affinity_set(mac_srs->srs_worker, cpuid); 1065 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1066 } 1067 1068 static void 1069 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1070 { 1071 cpu_t *cp; 1072 boolean_t clear = B_FALSE; 1073 1074 ASSERT(MUTEX_HELD(&cpu_lock)); 1075 1076 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1077 return; 1078 1079 cp = cpu_get(cpuid); 1080 if (cp == NULL || !cpu_is_online(cp)) 1081 return; 1082 1083 mutex_enter(&mac_srs->srs_lock); 1084 mac_srs->srs_state |= SRS_POLL_BOUND; 1085 if (mac_srs->srs_poll_cpuid != -1) 1086 clear = B_TRUE; 1087 mac_srs->srs_poll_cpuid = cpuid; 1088 mutex_exit(&mac_srs->srs_lock); 1089 1090 if (clear) 1091 thread_affinity_clear(mac_srs->srs_poll_thr); 1092 1093 thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1094 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1095 } 1096 1097 /* 1098 * When a CPU comes back online, bind the MAC kernel threads which 1099 * were previously bound to that CPU, and had to be unbound because 1100 * the CPU was going away. 1101 * 1102 * These functions are called with cpu_lock held and hence we can't 1103 * cv_wait to grab the mac perimeter. Since these functions walk the soft 1104 * ring list of an SRS without being in the perimeter, the list itself 1105 * is protected by the SRS lock. 1106 */ 1107 static void 1108 mac_walk_srs_and_bind(int cpuid) 1109 { 1110 mac_soft_ring_set_t *mac_srs; 1111 mac_soft_ring_t *soft_ring; 1112 1113 rw_enter(&mac_srs_g_lock, RW_READER); 1114 1115 if ((mac_srs = mac_srs_g_list) == NULL) 1116 goto done; 1117 1118 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1119 if (mac_srs->srs_worker_cpuid == -1 && 1120 mac_srs->srs_worker_cpuid_save == cpuid) { 1121 mac_srs->srs_worker_cpuid_save = -1; 1122 mac_srs_worker_bind(mac_srs, cpuid); 1123 } 1124 1125 if (!(mac_srs->srs_type & SRST_TX)) { 1126 if (mac_srs->srs_poll_cpuid == -1 && 1127 mac_srs->srs_poll_cpuid_save == cpuid) { 1128 mac_srs->srs_poll_cpuid_save = -1; 1129 mac_srs_poll_bind(mac_srs, cpuid); 1130 } 1131 } 1132 1133 /* Next tackle the soft rings associated with the srs */ 1134 mutex_enter(&mac_srs->srs_lock); 1135 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1136 soft_ring = soft_ring->s_ring_next) { 1137 if (soft_ring->s_ring_cpuid == -1 && 1138 soft_ring->s_ring_cpuid_save == cpuid) { 1139 soft_ring->s_ring_cpuid_save = -1; 1140 (void) mac_soft_ring_bind(soft_ring, cpuid); 1141 } 1142 } 1143 mutex_exit(&mac_srs->srs_lock); 1144 } 1145 done: 1146 rw_exit(&mac_srs_g_lock); 1147 } 1148 1149 /* 1150 * Change the priority of the SRS's poll and worker thread. Additionally, 1151 * update the priority of the worker threads for the SRS's soft rings. 1152 * Need to modify any associated squeue threads. 1153 */ 1154 void 1155 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1156 { 1157 mac_soft_ring_t *ringp; 1158 1159 mac_srs->srs_pri = prival; 1160 thread_lock(mac_srs->srs_worker); 1161 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1162 thread_unlock(mac_srs->srs_worker); 1163 if (mac_srs->srs_poll_thr != NULL) { 1164 thread_lock(mac_srs->srs_poll_thr); 1165 (void) thread_change_pri(mac_srs->srs_poll_thr, 1166 mac_srs->srs_pri, 0); 1167 thread_unlock(mac_srs->srs_poll_thr); 1168 } 1169 if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1170 return; 1171 while (ringp != mac_srs->srs_soft_ring_tail) { 1172 thread_lock(ringp->s_ring_worker); 1173 (void) thread_change_pri(ringp->s_ring_worker, 1174 mac_srs->srs_pri, 0); 1175 thread_unlock(ringp->s_ring_worker); 1176 ringp = ringp->s_ring_next; 1177 } 1178 ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1179 thread_lock(ringp->s_ring_worker); 1180 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1181 thread_unlock(ringp->s_ring_worker); 1182 } 1183 1184 /* 1185 * Change the receive bandwidth limit. 1186 */ 1187 static void 1188 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1189 { 1190 mac_soft_ring_t *softring; 1191 1192 mutex_enter(&srs->srs_lock); 1193 mutex_enter(&srs->srs_bw->mac_bw_lock); 1194 1195 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1196 /* Reset bandwidth limit */ 1197 if (srs->srs_type & SRST_BW_CONTROL) { 1198 softring = srs->srs_soft_ring_head; 1199 while (softring != NULL) { 1200 softring->s_ring_type &= ~ST_RING_BW_CTL; 1201 softring = softring->s_ring_next; 1202 } 1203 srs->srs_type &= ~SRST_BW_CONTROL; 1204 srs->srs_drain_func = mac_rx_srs_drain; 1205 } 1206 } else { 1207 /* Set/Modify bandwidth limit */ 1208 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1209 /* 1210 * Give twice the queuing capability before 1211 * dropping packets. The unit is bytes/tick. 1212 */ 1213 srs->srs_bw->mac_bw_drop_threshold = 1214 srs->srs_bw->mac_bw_limit << 1; 1215 if (!(srs->srs_type & SRST_BW_CONTROL)) { 1216 softring = srs->srs_soft_ring_head; 1217 while (softring != NULL) { 1218 softring->s_ring_type |= ST_RING_BW_CTL; 1219 softring = softring->s_ring_next; 1220 } 1221 srs->srs_type |= SRST_BW_CONTROL; 1222 srs->srs_drain_func = mac_rx_srs_drain_bw; 1223 } 1224 } 1225 done: 1226 mutex_exit(&srs->srs_bw->mac_bw_lock); 1227 mutex_exit(&srs->srs_lock); 1228 } 1229 1230 /* Change the transmit bandwidth limit */ 1231 static void 1232 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1233 { 1234 uint32_t tx_mode; 1235 mac_srs_tx_t *srs_tx = &srs->srs_tx; 1236 mac_client_impl_t *mcip = srs->srs_mcip; 1237 mac_impl_t *mip = mcip->mci_mip; 1238 1239 /* 1240 * We need to quiesce/restart the client here because mac_tx() and 1241 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1242 * st_mode and related fields, which are modified by the code below. 1243 */ 1244 mac_tx_client_quiesce(mcip, SRS_QUIESCE); 1245 1246 mutex_enter(&srs->srs_lock); 1247 mutex_enter(&srs->srs_bw->mac_bw_lock); 1248 1249 tx_mode = srs_tx->st_mode; 1250 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1251 /* Reset bandwidth limit */ 1252 if (tx_mode == SRS_TX_BW) { 1253 if (mac_tx_serialize || 1254 (mip->mi_v12n_level & MAC_VIRT_SERIALIZE)) { 1255 srs_tx->st_mode = SRS_TX_SERIALIZE; 1256 } else { 1257 srs_tx->st_mode = SRS_TX_DEFAULT; 1258 } 1259 } else if (tx_mode == SRS_TX_BW_FANOUT) { 1260 srs_tx->st_mode = SRS_TX_FANOUT; 1261 } 1262 srs->srs_type &= ~SRST_BW_CONTROL; 1263 } else { 1264 /* Set/Modify bandwidth limit */ 1265 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1266 /* 1267 * Give twice the queuing capability before 1268 * dropping packets. The unit is bytes/tick. 1269 */ 1270 srs->srs_bw->mac_bw_drop_threshold = 1271 srs->srs_bw->mac_bw_limit << 1; 1272 srs->srs_type |= SRST_BW_CONTROL; 1273 if (tx_mode != SRS_TX_BW && 1274 tx_mode != SRS_TX_BW_FANOUT) { 1275 if (tx_mode == SRS_TX_SERIALIZE || 1276 tx_mode == SRS_TX_DEFAULT) { 1277 srs_tx->st_mode = SRS_TX_BW; 1278 } else if (tx_mode == SRS_TX_FANOUT) { 1279 srs_tx->st_mode = SRS_TX_BW_FANOUT; 1280 } else { 1281 ASSERT(0); 1282 } 1283 } 1284 } 1285 done: 1286 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1287 mutex_exit(&srs->srs_bw->mac_bw_lock); 1288 mutex_exit(&srs->srs_lock); 1289 1290 mac_tx_client_restart(mcip); 1291 } 1292 1293 /* 1294 * The uber function that deals with any update to bandwidth limits. 1295 */ 1296 void 1297 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1298 { 1299 int count; 1300 1301 for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1302 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1303 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1304 } 1305 1306 void 1307 mac_srs_change_upcall(void *arg, mac_direct_rx_t rx_func, void *rx_arg1) 1308 { 1309 mac_soft_ring_set_t *mac_srs = arg; 1310 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 1311 mac_soft_ring_t *softring; 1312 1313 mutex_enter(&mac_srs->srs_lock); 1314 ASSERT((mac_srs->srs_type & SRST_TX) == 0); 1315 srs_rx->sr_func = rx_func; 1316 srs_rx->sr_arg1 = rx_arg1; 1317 1318 softring = mac_srs->srs_soft_ring_head; 1319 while (softring != NULL) { 1320 mutex_enter(&softring->s_ring_lock); 1321 softring->s_ring_rx_func = rx_func; 1322 softring->s_ring_rx_arg1 = rx_arg1; 1323 mutex_exit(&softring->s_ring_lock); 1324 softring = softring->s_ring_next; 1325 } 1326 1327 mutex_exit(&mac_srs->srs_lock); 1328 } 1329 1330 /* 1331 * When the first sub-flow is added to a link, we disable polling on the 1332 * link and also modify the entry point to mac_rx_srs_subflow_process. 1333 * (polling is disabled because with the subflow added, accounting 1334 * for polling needs additional logic, it is assumed that when a subflow is 1335 * added, we can take some hit as a result of disabling polling rather than 1336 * adding more complexity - if this becomes a perf. issue we need to 1337 * re-rvaluate this logic). When the last subflow is removed, we turn back 1338 * polling and also reset the entry point to mac_rx_srs_process. 1339 * 1340 * In the future if there are multiple SRS, we can simply 1341 * take one and give it to the flow rather than disabling polling and 1342 * resetting the entry point. 1343 */ 1344 void 1345 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1346 { 1347 flow_entry_t *flent = mcip->mci_flent; 1348 int i; 1349 mac_impl_t *mip = mcip->mci_mip; 1350 mac_rx_func_t rx_func; 1351 uint_t rx_srs_cnt; 1352 boolean_t enable_classifier; 1353 1354 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1355 1356 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1357 1358 rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1359 mac_rx_srs_process; 1360 1361 /* Tell mac_srs_poll_state_change to disable polling if necessary */ 1362 if (mip->mi_state_flags & MIS_POLL_DISABLE) 1363 enable_classifier = B_TRUE; 1364 1365 /* 1366 * If receive function has already been configured correctly for 1367 * current subflow configuration, do nothing. 1368 */ 1369 if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1370 return; 1371 1372 rx_srs_cnt = flent->fe_rx_srs_cnt; 1373 for (i = 0; i < rx_srs_cnt; i++) { 1374 ASSERT(flent->fe_rx_srs[i] != NULL); 1375 mac_srs_poll_state_change(flent->fe_rx_srs[i], 1376 enable_classifier, rx_func); 1377 } 1378 1379 /* 1380 * Change the S/W classifier so that we can land in the 1381 * correct processing function with correct argument. 1382 * If all subflows have been removed we can revert to 1383 * mac_rx_srsprocess, else we need mac_rx_srs_subflow_process. 1384 */ 1385 mutex_enter(&flent->fe_lock); 1386 flent->fe_cb_fn = (flow_fn_t)rx_func; 1387 flent->fe_cb_arg1 = (void *)mip; 1388 flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1389 mutex_exit(&flent->fe_lock); 1390 } 1391 1392 static void 1393 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1394 { 1395 int tcp_count = 0; 1396 int udp_count = 0; 1397 int oth_count = 0; 1398 mac_soft_ring_t *softring; 1399 1400 softring = mac_srs->srs_soft_ring_head; 1401 if (softring == NULL) { 1402 ASSERT(mac_srs->srs_soft_ring_count == 0); 1403 mac_srs->srs_tcp_ring_count = 0; 1404 mac_srs->srs_udp_ring_count = 0; 1405 mac_srs->srs_oth_ring_count = 0; 1406 return; 1407 } 1408 1409 softring = mac_srs->srs_soft_ring_head; 1410 tcp_count = udp_count = oth_count = 0; 1411 1412 while (softring != NULL) { 1413 if (softring->s_ring_type & ST_RING_TCP) 1414 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1415 else if (softring->s_ring_type & ST_RING_UDP) 1416 mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1417 else 1418 mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1419 softring = softring->s_ring_next; 1420 } 1421 1422 ASSERT(mac_srs->srs_soft_ring_count == 1423 (tcp_count + udp_count + oth_count)); 1424 1425 mac_srs->srs_tcp_ring_count = tcp_count; 1426 mac_srs->srs_udp_ring_count = udp_count; 1427 mac_srs->srs_oth_ring_count = oth_count; 1428 } 1429 1430 void 1431 mac_srs_create_proto_softrings(int id, void *flent, uint16_t type, 1432 pri_t pri, mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1433 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1434 mac_resource_handle_t x_arg2, boolean_t set_bypass) 1435 { 1436 mac_soft_ring_t *softring; 1437 mac_rx_fifo_t mrf; 1438 1439 bzero(&mrf, sizeof (mac_rx_fifo_t)); 1440 mrf.mrf_type = MAC_RX_FIFO; 1441 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1442 mrf.mrf_intr_enable = 1443 (mac_intr_enable_t)mac_soft_ring_intr_enable; 1444 mrf.mrf_intr_disable = 1445 (mac_intr_disable_t)mac_soft_ring_intr_disable; 1446 mrf.mrf_flow_priority = pri; 1447 1448 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1449 (void *)flent, (type|ST_RING_TCP), pri, mcip, mac_srs, 1450 cpuid, rx_func, x_arg1, x_arg2); 1451 softring->s_ring_rx_arg2 = NULL; 1452 1453 /* 1454 * TCP and UDP support DLS bypass. In addition TCP 1455 * squeue can also poll their corresponding soft rings. 1456 */ 1457 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1458 mac_soft_ring_dls_bypass(softring, 1459 mcip->mci_direct_rx_fn, 1460 mcip->mci_direct_rx_arg); 1461 1462 mrf.mrf_rx_arg = softring; 1463 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1464 1465 /* 1466 * Make a call in IP to get a TCP squeue assigned to 1467 * this softring to maintain full CPU locality through 1468 * the stack and allow the squeue to be able to poll 1469 * the softring so the flow control can be pushed 1470 * all the way to H/W. 1471 */ 1472 softring->s_ring_rx_arg2 = 1473 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1474 (mac_resource_t *)&mrf); 1475 } 1476 1477 /* 1478 * Non-TCP protocols don't support squeues. Hence we 1479 * don't make any ring addition callbacks for non-TCP 1480 * rings. Now create the UDP softring and allow it to 1481 * bypass the DLS layer. 1482 */ 1483 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1484 (void *)flent, (type|ST_RING_UDP), pri, mcip, mac_srs, 1485 cpuid, rx_func, x_arg1, x_arg2); 1486 softring->s_ring_rx_arg2 = NULL; 1487 1488 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1489 mac_soft_ring_dls_bypass(softring, 1490 mcip->mci_direct_rx_fn, 1491 mcip->mci_direct_rx_arg); 1492 } 1493 1494 /* Create the Oth softrings which has to go through the DLS */ 1495 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1496 (void *)flent, (type|ST_RING_OTH), pri, mcip, mac_srs, 1497 cpuid, rx_func, x_arg1, x_arg2); 1498 softring->s_ring_rx_arg2 = NULL; 1499 } 1500 1501 /* 1502 * This routine associates a CPU or a set of CPU to process incoming 1503 * traffic from a mac client. If multiple CPUs are specified, then 1504 * so many soft rings are created with each soft ring worker thread 1505 * bound to a CPU in the set. Each soft ring in turn will be 1506 * associated with an squeue and the squeue will be moved to the 1507 * same CPU as that of the soft ring's. 1508 */ 1509 static void 1510 mac_srs_fanout_modify(mac_client_impl_t *mcip, flow_entry_t *flent, 1511 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1512 mac_resource_handle_t x_arg2, mac_soft_ring_set_t *mac_rx_srs, 1513 mac_soft_ring_set_t *mac_tx_srs) 1514 { 1515 mac_soft_ring_t *softring; 1516 uint32_t soft_ring_flag = 0; 1517 processorid_t cpuid = -1; 1518 boolean_t user_specified; 1519 int i, srings_present, new_fanout_cnt; 1520 mac_cpus_t *srs_cpu; 1521 1522 user_specified = mrp->mrp_mask & MRP_CPUS_USERSPEC; 1523 /* fanout state is REINIT. Set it back to INIT */ 1524 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1525 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1526 1527 /* how many are present right now */ 1528 srings_present = mac_rx_srs->srs_tcp_ring_count; 1529 /* new request */ 1530 srs_cpu = &mac_rx_srs->srs_cpu; 1531 new_fanout_cnt = srs_cpu->mc_fanout_cnt; 1532 1533 mutex_enter(&mac_rx_srs->srs_lock); 1534 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1535 soft_ring_flag |= ST_RING_BW_CTL; 1536 mutex_exit(&mac_rx_srs->srs_lock); 1537 1538 if (new_fanout_cnt > srings_present) { 1539 /* soft rings increased */ 1540 mutex_enter(&mac_rx_srs->srs_lock); 1541 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1542 mutex_exit(&mac_rx_srs->srs_lock); 1543 1544 for (i = mac_rx_srs->srs_tcp_ring_count; 1545 i < new_fanout_cnt; i++) { 1546 /* 1547 * Create the protocol softrings and set the 1548 * DLS bypass where possible. 1549 */ 1550 mac_srs_create_proto_softrings(i, 1551 (void *)flent, soft_ring_flag, 1552 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1553 rx_func, x_arg1, x_arg2, B_TRUE); 1554 } 1555 mac_srs_update_fanout_list(mac_rx_srs); 1556 } else if (new_fanout_cnt < srings_present) { 1557 /* soft rings decreased */ 1558 if (new_fanout_cnt == 1) { 1559 mutex_enter(&mac_rx_srs->srs_lock); 1560 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1561 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1562 mutex_exit(&mac_rx_srs->srs_lock); 1563 } 1564 /* Get rid of extra soft rings */ 1565 for (i = new_fanout_cnt; 1566 i < mac_rx_srs->srs_tcp_ring_count; i++) { 1567 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1568 if (softring->s_ring_rx_arg2 != NULL) { 1569 mcip->mci_resource_remove( 1570 (void *)mcip->mci_resource_arg, 1571 softring->s_ring_rx_arg2); 1572 } 1573 mac_soft_ring_remove(mac_rx_srs, 1574 mac_rx_srs->srs_tcp_soft_rings[i]); 1575 mac_soft_ring_remove(mac_rx_srs, 1576 mac_rx_srs->srs_udp_soft_rings[i]); 1577 mac_soft_ring_remove(mac_rx_srs, 1578 mac_rx_srs->srs_oth_soft_rings[i]); 1579 } 1580 mac_srs_update_fanout_list(mac_rx_srs); 1581 } 1582 1583 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1584 mutex_enter(&cpu_lock); 1585 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1586 cpuid = srs_cpu->mc_fanout_cpus[i]; 1587 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1588 cpuid); 1589 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1590 cpuid); 1591 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1592 cpuid); 1593 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1594 if (softring->s_ring_rx_arg2 != NULL) { 1595 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1596 softring->s_ring_rx_arg2, cpuid); 1597 } 1598 } 1599 1600 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_pollid); 1601 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_workerid); 1602 1603 /* 1604 * Bind Tx srs and soft ring threads too. Let's bind tx 1605 * srs to the last cpu in mrp list. 1606 */ 1607 if (mac_tx_srs != NULL && user_specified) { 1608 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1609 } 1610 mutex_exit(&cpu_lock); 1611 } 1612 1613 /* 1614 * Bind SRS threads and soft rings to CPUs/create fanout list. 1615 */ 1616 void 1617 mac_srs_fanout_init(mac_client_impl_t *mcip, flow_entry_t *flent, 1618 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1619 mac_resource_handle_t x_arg2, mac_soft_ring_set_t *mac_rx_srs, 1620 mac_soft_ring_set_t *mac_tx_srs) 1621 { 1622 int i; 1623 processorid_t cpuid, worker_cpuid, poll_cpuid; 1624 uint32_t soft_ring_flag = 0; 1625 int soft_ring_cnt; 1626 boolean_t user_specified = B_FALSE; 1627 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1628 1629 /* 1630 * Remove the no soft ring flag and we will adjust it 1631 * appropriately further down. 1632 */ 1633 mutex_enter(&mac_rx_srs->srs_lock); 1634 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1635 mutex_exit(&mac_rx_srs->srs_lock); 1636 1637 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1638 1639 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1640 soft_ring_flag |= ST_RING_BW_CTL; 1641 1642 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1643 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1644 user_specified = mrp->mrp_mask & MRP_CPUS_USERSPEC; 1645 /* 1646 * Ring count can be 0 if no fanout is required and no cpu 1647 * were specified. Leave the SRS worker and poll thread 1648 * unbound 1649 */ 1650 ASSERT(mrp != NULL); 1651 soft_ring_cnt = srs_cpu->mc_fanout_cnt; 1652 1653 /* Step 1: bind cpu contains cpu list where threads need to bind */ 1654 if (soft_ring_cnt > 0) { 1655 mutex_enter(&cpu_lock); 1656 for (i = 0; i < soft_ring_cnt; i++) { 1657 cpuid = srs_cpu->mc_fanout_cpus[i]; 1658 /* Create the protocol softrings */ 1659 mac_srs_create_proto_softrings(i, (void *)flent, 1660 soft_ring_flag, mac_rx_srs->srs_pri, 1661 mcip, mac_rx_srs, cpuid, rx_func, 1662 x_arg1, x_arg2, B_FALSE); 1663 } 1664 worker_cpuid = srs_cpu->mc_workerid; 1665 poll_cpuid = srs_cpu->mc_pollid; 1666 mac_srs_worker_bind(mac_rx_srs, worker_cpuid); 1667 mac_srs_poll_bind(mac_rx_srs, poll_cpuid); 1668 1669 /* 1670 * Bind Tx srs and soft ring threads too. 1671 * Let's bind tx srs to the last cpu in 1672 * mrp list. 1673 */ 1674 if (mac_tx_srs == NULL) { 1675 mutex_exit(&cpu_lock); 1676 goto alldone; 1677 } 1678 1679 if (user_specified) { 1680 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1681 } 1682 mutex_exit(&cpu_lock); 1683 } else { 1684 mutex_enter(&cpu_lock); 1685 /* 1686 * For a subflow, mrp_workerid and mrp_pollid 1687 * is not set. 1688 */ 1689 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_workerid); 1690 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_pollid); 1691 mutex_exit(&cpu_lock); 1692 goto no_softrings; 1693 } 1694 1695 alldone: 1696 if (soft_ring_cnt > 1) 1697 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1698 mac_srs_update_fanout_list(mac_rx_srs); 1699 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1700 return; 1701 1702 no_softrings: 1703 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1704 mutex_enter(&cpu_lock); 1705 cpuid = mac_next_bind_cpu(); 1706 /* Create the protocol softrings */ 1707 mac_srs_create_proto_softrings(0, (void *)flent, 1708 soft_ring_flag, mac_rx_srs->srs_pri, 1709 mcip, mac_rx_srs, cpuid, rx_func, 1710 x_arg1, x_arg2, B_FALSE); 1711 mutex_exit(&cpu_lock); 1712 } else { 1713 /* 1714 * This is the case when there is no fanout which is 1715 * true for subflows. 1716 */ 1717 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1718 } 1719 mac_srs_update_fanout_list(mac_rx_srs); 1720 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1721 } 1722 1723 /* 1724 * mac_fanout_setup: 1725 * 1726 * Calls mac_srs_fanout_init() or modify() depending upon whether 1727 * the SRS is getting initialized or re-initialized. 1728 */ 1729 void 1730 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1731 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1732 mac_resource_handle_t x_arg2) 1733 { 1734 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1735 int i, rx_srs_cnt; 1736 1737 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1738 /* 1739 * This is an aggregation port. Fanout will be setup 1740 * over the aggregation itself. 1741 */ 1742 if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) 1743 return; 1744 1745 mac_rx_srs = flent->fe_rx_srs[0]; 1746 /* 1747 * Set up the fanout on the tx side only once, with the 1748 * first rx SRS. The CPU binding, fanout, and bandwidth 1749 * criteria are common to both RX and TX, so 1750 * initializing them along side avoids redundant code. 1751 */ 1752 mac_tx_srs = flent->fe_tx_srs; 1753 rx_srs_cnt = flent->fe_rx_srs_cnt; 1754 1755 /* No fanout for subflows */ 1756 if (flent->fe_type & FLOW_USER) { 1757 mac_srs_fanout_init(mcip, flent, mrp, rx_func, 1758 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs); 1759 return; 1760 } 1761 1762 mac_flow_cpu_init(flent, mrp); 1763 1764 /* 1765 * Set up fanout for both SW (0th SRS) and HW classified 1766 * SRS (the rest of Rx SRSs in flent). 1767 */ 1768 for (i = 0; i < rx_srs_cnt; i++) { 1769 mac_rx_srs = flent->fe_rx_srs[i]; 1770 if (i != 0) 1771 mac_tx_srs = NULL; 1772 switch (mac_rx_srs->srs_fanout_state) { 1773 case SRS_FANOUT_UNINIT: 1774 mac_srs_fanout_init(mcip, flent, mrp, rx_func, 1775 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs); 1776 break; 1777 case SRS_FANOUT_INIT: 1778 break; 1779 case SRS_FANOUT_REINIT: 1780 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 1781 mac_srs_fanout_modify(mcip, flent, mrp, rx_func, 1782 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs); 1783 mac_rx_srs_restart(mac_rx_srs); 1784 break; 1785 default: 1786 VERIFY(mac_rx_srs->srs_fanout_state <= 1787 SRS_FANOUT_REINIT); 1788 break; 1789 } 1790 } 1791 } 1792 1793 /* 1794 * mac_create_soft_ring_set: 1795 * 1796 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 1797 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 1798 * processing is created. 1799 * 1800 * Details on Rx SRS: 1801 * Create a SRS and also add the necessary soft rings for TCP and 1802 * non-TCP based on fanout type and count specified. 1803 * 1804 * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 1805 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 1806 * to be heavily modified. 1807 * 1808 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 1809 */ 1810 mac_soft_ring_set_t * 1811 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 1812 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1813 mac_ring_t *ring) 1814 { 1815 mac_soft_ring_set_t *mac_srs; 1816 mac_srs_rx_t *srs_rx; 1817 mac_srs_tx_t *srs_tx; 1818 mac_bw_ctl_t *mac_bw; 1819 mac_resource_props_t *mrp; 1820 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 1821 1822 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 1823 bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 1824 srs_rx = &mac_srs->srs_rx; 1825 srs_tx = &mac_srs->srs_tx; 1826 1827 mutex_enter(&flent->fe_lock); 1828 1829 /* 1830 * Get the bandwidth control structure from the flent. Get 1831 * rid of any residual values in the control structure for 1832 * the tx bw struct and also for the rx, if the rx srs is 1833 * the 1st one being brought up (the rx bw ctl struct may 1834 * be shared by multiple SRSs) 1835 */ 1836 if (is_tx_srs) { 1837 mac_srs->srs_bw = &flent->fe_tx_bw; 1838 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 1839 flent->fe_tx_srs = mac_srs; 1840 } else { 1841 /* 1842 * The bw counter (stored in the flent) is shared 1843 * by SRS's within an rx group. 1844 */ 1845 mac_srs->srs_bw = &flent->fe_rx_bw; 1846 /* First rx SRS, clear the bw structure */ 1847 if (flent->fe_rx_srs_cnt == 0) 1848 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 1849 1850 /* 1851 * It is better to panic here rather than just assert because 1852 * on a non-debug kernel we might end up courrupting memory 1853 * and making it difficult to debug. 1854 */ 1855 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 1856 panic("Array Overrun detected due to MAC client %p " 1857 " having more rings than %d", (void *)mcip, 1858 MAX_RINGS_PER_GROUP); 1859 } 1860 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 1861 flent->fe_rx_srs_cnt++; 1862 } 1863 mac_srs->srs_flent = flent; 1864 mutex_exit(&flent->fe_lock); 1865 1866 mac_srs->srs_state = 0; 1867 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 1868 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 1869 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 1870 mac_srs_fanout_list_alloc(mac_srs); 1871 1872 /* 1873 * For a flow we use the underlying MAC client's priority range with 1874 * the priority value to find an absolute priority value. For a MAC 1875 * client we use the MAC client's maximum priority as the value. 1876 */ 1877 mrp = &flent->fe_effective_props; 1878 if ((mac_srs->srs_type & SRST_FLOW) != 0) { 1879 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 1880 mcip->mci_max_pri, mrp->mrp_priority); 1881 } else { 1882 mac_srs->srs_pri = mcip->mci_max_pri; 1883 } 1884 mac_srs->srs_mcip = mcip; 1885 /* 1886 * We need to insert the SRS in the global list before 1887 * binding the SRS and SR threads. Otherwise there is a 1888 * is a small window where the cpu reconfig callbacks 1889 * may miss the SRS in the list walk and DR could fail 1890 * as there are bound threads. 1891 */ 1892 mac_srs_add_glist(mac_srs); 1893 1894 /* Initialize bw limit */ 1895 if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 1896 mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 1897 1898 mac_bw = mac_srs->srs_bw; 1899 mutex_enter(&mac_bw->mac_bw_lock); 1900 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1901 1902 /* 1903 * Give twice the queuing capability before 1904 * dropping packets. The unit is bytes/tick. 1905 */ 1906 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 1907 mutex_exit(&mac_bw->mac_bw_lock); 1908 mac_srs->srs_type |= SRST_BW_CONTROL; 1909 } else { 1910 mac_srs->srs_drain_func = mac_rx_srs_drain; 1911 } 1912 1913 /* 1914 * We use the following policy to control Receive 1915 * Side Dynamic Polling: 1916 * 1) We switch to poll mode anytime the processing thread causes 1917 * a backlog to build up in SRS and its associated Soft Rings 1918 * (sr_poll_pkt_cnt > 0). 1919 * 2) As long as the backlog stays under the low water mark 1920 * (sr_lowat), we poll the H/W for more packets. 1921 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 1922 * stay in poll mode but don't poll the H/W for more packets. 1923 * 4) Anytime in polling mode, if we poll the H/W for packets and 1924 * find nothing plus we have an existing backlog 1925 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 1926 * the H/W for packets anymore (let the polling thread go to sleep). 1927 * 5) Once the backlog is relived (packets are processed) we reenable 1928 * polling (by signalling the poll thread) only when the backlog 1929 * dips below sr_poll_thres. 1930 * 6) sr_hiwat is used exclusively when we are not polling capable 1931 * and is used to decide when to drop packets so the SRS queue 1932 * length doesn't grow infinitely. 1933 */ 1934 if (!is_tx_srs) { 1935 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 1936 /* Low water mark needs to be less than high water mark */ 1937 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 1938 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 1939 (mac_soft_ring_max_q_cnt >> 2); 1940 /* Poll threshold need to be half of low water mark or less */ 1941 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 1942 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 1943 (srs_rx->sr_lowat >> 1); 1944 if (mac_latency_optimize) 1945 mac_srs->srs_state |= SRS_LATENCY_OPT; 1946 else 1947 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 1948 } 1949 1950 mac_srs->srs_worker = thread_create(NULL, 0, 1951 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 1952 1953 if (is_tx_srs) { 1954 /* Handle everything about Tx SRS and return */ 1955 mac_srs->srs_drain_func = mac_tx_srs_drain; 1956 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 1957 srs_tx->st_hiwat = 1958 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 1959 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 1960 srs_tx->st_arg1 = x_arg1; 1961 srs_tx->st_arg2 = x_arg2; 1962 return (mac_srs); 1963 } 1964 1965 if ((srs_type & SRST_FLOW) != 0 || 1966 FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 1967 srs_rx->sr_lower_proc = mac_rx_srs_process; 1968 else 1969 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 1970 1971 srs_rx->sr_func = rx_func; 1972 srs_rx->sr_arg1 = x_arg1; 1973 srs_rx->sr_arg2 = x_arg2; 1974 1975 if (ring != NULL) { 1976 /* Is the mac_srs created over the RX default group? */ 1977 if (ring->mr_gh == (mac_group_handle_t) 1978 (&mcip->mci_mip->mi_rx_groups[0])) 1979 mac_srs->srs_type |= SRST_DEFAULT_GRP; 1980 1981 mac_srs->srs_ring = ring; 1982 ring->mr_srs = mac_srs; 1983 ring->mr_classify_type = MAC_HW_CLASSIFIER; 1984 ring->mr_flag |= MR_INCIPIENT; 1985 1986 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 1987 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 1988 mac_srs->srs_state |= SRS_POLLING_CAPAB; 1989 1990 mac_srs->srs_poll_thr = thread_create(NULL, 0, 1991 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 1992 mac_srs->srs_pri); 1993 /* 1994 * Some drivers require serialization and don't send 1995 * packet chains in interrupt context. For such 1996 * drivers, we should always queue in soft ring 1997 * so that we get a chance to switch into a polling 1998 * mode under backlog. 1999 */ 2000 if (mcip->mci_mip->mi_v12n_level & MAC_VIRT_SERIALIZE) 2001 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2002 } 2003 return (mac_srs); 2004 } 2005 2006 /* 2007 * Figure out the number of soft rings required. Its dependant on 2008 * if protocol fanout is required (for LINKs), global settings 2009 * require us to do fanout for performance (based on mac_soft_ring_enable), 2010 * or user has specifically requested fanout. 2011 */ 2012 static uint32_t 2013 mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2014 { 2015 uint32_t fanout_type; 2016 mac_resource_props_t *mrp = &flent->fe_effective_props; 2017 2018 /* no fanout for subflows */ 2019 switch (link_type) { 2020 case SRST_FLOW: 2021 fanout_type = SRST_NO_SOFT_RINGS; 2022 break; 2023 case SRST_LINK: 2024 fanout_type = SRST_FANOUT_PROTO; 2025 break; 2026 } 2027 2028 /* A primary NIC/link is being plumbed */ 2029 if (flent->fe_type & FLOW_PRIMARY_MAC) { 2030 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2031 fanout_type |= SRST_FANOUT_SRC_IP; 2032 } 2033 } else if (flent->fe_type & FLOW_VNIC) { 2034 /* A VNIC is being created */ 2035 if (mrp != NULL && mrp->mrp_ncpus > 0) { 2036 fanout_type |= SRST_FANOUT_SRC_IP; 2037 } 2038 } 2039 2040 return (fanout_type); 2041 } 2042 2043 /* 2044 * Change a group from h/w to s/w classification. 2045 */ 2046 static void 2047 mac_rx_switch_grp_to_sw(mac_group_t *group) 2048 { 2049 mac_ring_t *ring; 2050 mac_soft_ring_set_t *mac_srs; 2051 2052 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2053 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2054 /* 2055 * Remove the SRS associated with the HW ring. 2056 * As a result, polling will be disabled. 2057 */ 2058 mac_srs = ring->mr_srs; 2059 ASSERT(mac_srs != NULL); 2060 mac_rx_srs_remove(mac_srs); 2061 ring->mr_srs = NULL; 2062 } 2063 2064 if (ring->mr_state != MR_INUSE) 2065 (void) mac_start_ring(ring); 2066 /* 2067 * We need to perform SW classification 2068 * for packets landing in these rings 2069 */ 2070 ring->mr_state = MR_INUSE; 2071 ring->mr_flag = 0; 2072 ring->mr_classify_type = MAC_SW_CLASSIFIER; 2073 } 2074 } 2075 2076 /* 2077 * Create the Rx SRS for S/W classifier and for each ring in the 2078 * group (if exclusive group). Also create the Tx SRS. 2079 */ 2080 void 2081 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2082 mac_group_t *group, uint32_t link_type) 2083 { 2084 mac_impl_t *mip = mcip->mci_mip; 2085 mac_soft_ring_set_t *mac_srs; 2086 mac_soft_ring_set_t *tx_srs = NULL; 2087 mac_ring_t *ring; 2088 uint32_t fanout_type; 2089 boolean_t created_srs = B_FALSE; 2090 2091 fanout_type = mac_find_fanout(flent, link_type); 2092 2093 /* Create the SRS for S/W classification if none exists */ 2094 if (flent->fe_rx_srs[0] == NULL) { 2095 ASSERT(flent->fe_rx_srs_cnt == 0); 2096 /* Setup the Rx SRS */ 2097 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2098 mac_rx_deliver, mcip, NULL, NULL); 2099 2100 mutex_enter(&flent->fe_lock); 2101 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2102 flent->fe_cb_arg1 = (void *)mip; 2103 flent->fe_cb_arg2 = (void *)mac_srs; 2104 mutex_exit(&flent->fe_lock); 2105 2106 /* Setup the Tx SRS as well */ 2107 ASSERT(flent->fe_tx_srs == NULL); 2108 tx_srs = mac_srs_create(mcip, flent, SRST_TX | link_type, 2109 NULL, mcip, NULL, NULL); 2110 2111 if (mcip->mci_share != NULL) { 2112 mac_srs_tx_t *tx = &tx_srs->srs_tx; 2113 ASSERT((mcip->mci_state_flags & MCIS_NO_HWRINGS) == 0); 2114 /* 2115 * A share requires a dedicated TX group. 2116 * mac_reserve_tx_group() does the work needed to 2117 * allocate a new group and populate that group 2118 * with rings according to the driver requirements 2119 * and limitations. 2120 */ 2121 tx->st_group = 2122 mac_reserve_tx_group(mip, mcip->mci_share); 2123 ASSERT(tx->st_group != NULL); 2124 tx->st_group->mrg_tx_client = mcip; 2125 } 2126 mac_tx_srs_setup(mcip, flent, link_type); 2127 created_srs = B_TRUE; 2128 } 2129 2130 if (group == NULL) { 2131 if (created_srs) { 2132 mac_fanout_setup(mcip, flent, 2133 MCIP_RESOURCE_PROPS(mcip), mac_rx_deliver, 2134 mcip, NULL); 2135 } 2136 return; 2137 } 2138 2139 /* 2140 * fanout for default SRS is done when default SRS are created 2141 * above. As each ring is added to the group, we setup the 2142 * SRS and fanout to it. 2143 */ 2144 switch (group->mrg_state) { 2145 case MAC_GROUP_STATE_RESERVED: 2146 /* 2147 * The group is exclusively ours. Create a SRS 2148 * for each ring in the group and allow the 2149 * individual SRS to dynamically poll their 2150 * Rx ring. Do this only if the client is not 2151 * a VLAN MAC client since for VLAN we do 2152 * s/w classification for the VID check. 2153 */ 2154 if (i_mac_flow_vid(mcip->mci_flent) != VLAN_ID_NONE) 2155 break; 2156 for (ring = group->mrg_rings; ring != NULL; 2157 ring = ring->mr_next) { 2158 switch (ring->mr_state) { 2159 case MR_INUSE: 2160 case MR_FREE: 2161 if (ring->mr_srs != NULL) 2162 break; 2163 if (ring->mr_state != MR_INUSE) 2164 (void) mac_start_ring(ring); 2165 2166 ring->mr_state = MR_INUSE; 2167 2168 mac_srs = mac_srs_create(mcip, flent, 2169 fanout_type | link_type, 2170 mac_rx_deliver, mcip, NULL, ring); 2171 break; 2172 default: 2173 cmn_err(CE_PANIC, "srs_setup: mcip = %p " 2174 "trying to add UNKNOWN ring = %p\n", 2175 (void *)mcip, (void *)ring); 2176 break; 2177 } 2178 } 2179 break; 2180 case MAC_GROUP_STATE_SHARED: 2181 /* 2182 * Set all rings of this group to software classified. 2183 * 2184 * If the group is current RESERVED, the existing mac client 2185 * (the only client on this group) is using this group 2186 * exclusively. In that case we need to disable polling on 2187 * the rings of the group (if it was enabled), and free the 2188 * SRS associated with the rings. 2189 */ 2190 mac_rx_switch_grp_to_sw(group); 2191 break; 2192 default: 2193 ASSERT(B_FALSE); 2194 break; 2195 } 2196 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2197 mac_rx_deliver, mcip, NULL); 2198 } 2199 2200 void 2201 mac_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2202 uint32_t link_type) 2203 { 2204 mac_soft_ring_set_t *mac_srs; 2205 mac_soft_ring_set_t *tx_srs; 2206 mac_srs_tx_t *tx; 2207 int i; 2208 2209 for (i = 0; i < flent->fe_rx_srs_cnt; i++) { 2210 mac_srs = flent->fe_rx_srs[i]; 2211 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2212 /* 2213 * Deal with all fanout tear down etc. 2214 */ 2215 mac_srs_free(mac_srs); 2216 flent->fe_rx_srs[i] = NULL; 2217 } 2218 flent->fe_rx_srs_cnt = 0; 2219 2220 tx_srs = flent->fe_tx_srs; 2221 tx = &tx_srs->srs_tx; 2222 switch (link_type) { 2223 case SRST_FLOW: 2224 /* 2225 * For flows, we need to work with passed 2226 * flent to find the Rx/Tx SRS. 2227 */ 2228 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2229 break; 2230 case SRST_LINK: 2231 mac_tx_client_quiesce(mcip, SRS_CONDEMNED); 2232 /* 2233 * Release the TX resources. First the TX group, if any 2234 * was assigned to the MAC client, which will cause the 2235 * TX rings to be moved back to the pool. Then free the 2236 * rings themselves. 2237 */ 2238 if (tx->st_group != NULL) { 2239 mac_release_tx_group(tx_srs->srs_mcip->mci_mip, 2240 tx->st_group); 2241 tx->st_group = NULL; 2242 } 2243 if (tx->st_ring_count != 0) { 2244 kmem_free(tx->st_rings, 2245 sizeof (mac_ring_handle_t) * tx->st_ring_count); 2246 } 2247 if (tx->st_arg2 != NULL) { 2248 ASSERT(tx_srs->srs_type & SRST_TX); 2249 mac_release_tx_ring(tx->st_arg2); 2250 } 2251 break; 2252 default: 2253 ASSERT(B_FALSE); 2254 break; 2255 } 2256 mac_srs_free(tx_srs); 2257 flent->fe_tx_srs = NULL; 2258 } 2259 2260 /* 2261 * This is the group state machine. The state of an Rx group is given by 2262 * the following table. The default group and its rings are started in 2263 * mac_start itself and the default group stays in SHARED state until 2264 * mac_stop at which time the group and rings are stopped and and it 2265 * reverts to the Registered state. 2266 * 2267 * Typically this function is called on a group after adding or removing a 2268 * client from it, to find out what should be the new state of the group. 2269 * If the new state is RESERVED, then the client that owns this group 2270 * exclusively is also returned. Note that adding or removing a client from 2271 * a group could also impact the default group and the caller needs to 2272 * evaluate the effect on the default group. 2273 * 2274 * Group type # of clients mi_nactiveclients Group State 2275 * in the group 2276 * 2277 * Non-default 0 N.A. REGISTERED 2278 * Non-default 1 N.A. RESERVED 2279 * Non-default > 1 N.A. SHARED 2280 * 2281 * Default 0 N.A. SHARED 2282 * Default 1 1 RESERVED 2283 * Default 1 > 1 SHARED 2284 * Default > 1 N.A. SHARED 2285 */ 2286 mac_group_state_t 2287 mac_rx_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip) 2288 { 2289 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2290 2291 *group_only_mcip = NULL; 2292 2293 /* Non-default group */ 2294 2295 if (grp != mip->mi_rx_groups) { 2296 if (MAC_RX_GROUP_NO_CLIENT(grp)) 2297 return (MAC_GROUP_STATE_REGISTERED); 2298 2299 *group_only_mcip = MAC_RX_GROUP_ONLY_CLIENT(grp); 2300 if (*group_only_mcip != NULL) 2301 return (MAC_GROUP_STATE_RESERVED); 2302 2303 return (MAC_GROUP_STATE_SHARED); 2304 } 2305 2306 /* Default group */ 2307 2308 if (MAC_RX_GROUP_NO_CLIENT(grp) || mip->mi_nactiveclients != 1) 2309 return (MAC_GROUP_STATE_SHARED); 2310 2311 *group_only_mcip = MAC_RX_GROUP_ONLY_CLIENT(grp); 2312 ASSERT(*group_only_mcip != NULL); 2313 return (MAC_GROUP_STATE_RESERVED); 2314 } 2315 2316 /* 2317 * OVERVIEW NOTES FOR DATAPATH 2318 * =========================== 2319 * 2320 * Create an SRS and setup the corresponding flow function and args. 2321 * Add a classification rule for the flow specified by 'flent' and program 2322 * the hardware classifier when applicable. 2323 * 2324 * Rx ring assignment, SRS, polling and B/W enforcement 2325 * ---------------------------------------------------- 2326 * 2327 * We try to use H/W classification on NIC and assign traffic to a 2328 * MAC address to a particular Rx ring. There is a 1-1 mapping 2329 * between a SRS and a Rx ring. The SRS (short for soft ring set) 2330 * dynamically switches the underlying Rx ring between interrupt 2331 * and polling mode and enforces any specified B/W control. 2332 * 2333 * There is always a SRS created and tied to each H/W and S/W rule. 2334 * Whenever we create a H/W rule, we always add the the same rule to 2335 * S/W classifier and tie a SRS to it. 2336 * 2337 * In case a B/W control is specified, its broken into bytes 2338 * per ticks and as soon as the quota for a tick is exhausted, 2339 * the underlying Rx ring is forced into poll mode for remianing 2340 * tick. The SRS poll thread only polls for bytes that are 2341 * allowed to come in the SRS. We typically let 4x the configured 2342 * B/W worth of packets to come in the SRS (to prevent unnecessary 2343 * drops due to bursts) but only process the specified amount. 2344 * 2345 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2346 * Rx rings (and corresponding SRSs) assigned to it. The SRS 2347 * in turn can have softrings to do protocol level fanout or 2348 * softrings to do S/W based fanout or both. In case the NIC 2349 * has no Rx rings, we do S/W classification to respective SRS. 2350 * The S/W classification rule is always setup and ready. This 2351 * allows the MAC layer to reassign Rx rings whenever needed 2352 * but packets still continue to flow via the default path and 2353 * getting S/W classified to correct SRS. 2354 * 2355 * In other cases where a NIC or VNIC is plumbed, our goal is use 2356 * H/W classifier and get two Rx ring assigned for the Link. One 2357 * for TCP and one for UDP|SCTP. The respective SRS still do the 2358 * polling on the Rx ring. For Link that is plumbed for IP, there 2359 * is a TCP squeue which also does polling and can control the 2360 * the Rx ring directly (where SRS is just pass through). For 2361 * the following cases, the SRS does the polling underneath. 2362 * 1) non IP based Links (Links which are not plumbed via ifconfig) 2363 * and paths which have no IP squeues (UDP & SCTP) 2364 * 2) If B/W control is specified on the Link 2365 * 3) If S/W fanout is secified 2366 * 2367 * Note1: As of current implementation, we try to assign only 1 Rx 2368 * ring per Link and more than 1 Rx ring for primary Link for 2369 * H/W based fanout. We always create following softrings per SRS: 2370 * 1) TCP softring which is polled by TCP squeue where possible 2371 * (and also bypasses DLS) 2372 * 2) UDP/SCTP based which bypasses DLS 2373 * 3) OTH softring which goes via DLS (currently deal with IPv6 2374 * and non TCP/UDP/SCTP for IPv4 packets). 2375 * 2376 * It is necessary to create 3 softrings since SRS has to poll 2377 * the single Rx ring underneath and enforce any link level B/W 2378 * control (we can't switch the Rx ring in poll mode just based 2379 * on TCP squeue if the same Rx ring is sharing UDP and other 2380 * traffic as well). Once polling is done and any Link level B/W 2381 * control is specified, the packets are assigned to respective 2382 * softring based on protocol. Since TCP has IP based squeue 2383 * which benefits by polling, we separate TCP packets into 2384 * its own softring which can be polled by IP squeue. We need 2385 * to separate out UDP/SCTP to UDP softring since it can bypass 2386 * the DLS layer which has heavy performance advanatges and we 2387 * need a softring (OTH) for rest. 2388 * 2389 * ToDo: The 3 softrings for protocol are needed only till we can 2390 * get rid of DLS from datapath, make IPv4 and IPv6 paths 2391 * symmetric (deal with mac_header_info for v6 and polling for 2392 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2393 * are generic), and bring SAP based classification to MAC layer 2394 * 2395 * H/W and S/W based fanout and multiple Rx rings per Link 2396 * ------------------------------------------------------- 2397 * 2398 * In case, fanout is requested (or determined automatically based 2399 * on Link speed and processor speed), we try to assign multiple 2400 * Rx rings per Link with their respective SRS. In this case 2401 * the NIC should be capable of fanning out incoming packets between 2402 * the assigned Rx rings (H/W based fanout). All the SRS 2403 * individually switch their Rx ring between interrupt and polling 2404 * mode but share a common B/W control counter in case of Link 2405 * level B/W is specified. 2406 * 2407 * If S/W based fanout is specified in lieu of H/W based fanout, 2408 * the Link SRS creates the specified number of softrings for 2409 * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2410 * out to the correct softring based on their protocol and 2411 * protocol specific hash function. 2412 * 2413 * Primary and non primary MAC clients 2414 * ----------------------------------- 2415 * 2416 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2417 * and are a Layer 2 construct. 2418 * 2419 * Primary NIC: 2420 * The Link that owns the primary MAC address and typically 2421 * is used as the data NIC in non virtualized cases. As such 2422 * H/W resources are preferntially given to primary NIC. As 2423 * far as code is concerned, there is no difference in the 2424 * primary NIC vs VNICs. They are all treated as Links. 2425 * At the very first call to mac_unicast_add() we program the S/W 2426 * classifier for the primary MAC address, get a soft ring set 2427 * (and soft rings based on 'ip_soft_ring_cnt') 2428 * and a Rx ring assigned for polling to get enabled. 2429 * When IP get plumbed and negotiates polling, we can 2430 * let squeue do the polling on TCP softring. 2431 * 2432 * VNICs: 2433 * Same as any other Link. As long as the H/W resource assignments 2434 * are equal, the data path and setup for all Links is same. 2435 * 2436 * Flows: 2437 * Can be configured on Links. They have their own SRS and the 2438 * S/W classifier is programmed appropriately based on the flow. 2439 * The flows typically deal with layer 3 and above and 2440 * creates a soft ring set specific to the flow. The receive 2441 * side function is switched from mac_rx_srs_process to 2442 * mac_rx_srs_subflow_process which first tries to assign the 2443 * packet to appropriate flow SRS and failing which assigns it 2444 * to link SRS. This allows us to avoid the layered approach 2445 * which gets complex. 2446 * 2447 * By the time mac_datapath_setup() completes, we already have the 2448 * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2449 * and S/W classifiers programmed. IP is not plumbed yet (and might 2450 * never be for Virtual Machines guest OS path). When IP is plumbed 2451 * (for both NIC and VNIC), we do a capability negotiation for polling 2452 * and upcall functions etc. 2453 * 2454 * Rx ring Assignement NOTES 2455 * ------------------------- 2456 * 2457 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2458 * as NIC with a single default ring), we assign the only ring to 2459 * primary Link as MAC_RX_HW_DEFAULT_RING. The primary Link SRS can do 2460 * polling on it as long as it is the only link in use and we compare 2461 * the MAC address for unicast packets before accepting an incoming 2462 * packet (there is no need for S/W classification in this case). We 2463 * disable polling on the only ring the moment 2nd link gets created 2464 * (the polling remains enabled even though there are broadcast and 2465 * multicast flows created). 2466 * 2467 * If the NIC has more than 1 Rx ring, we assign the default ring (the 2468 * 1st ring) to deal with broadcast, multicast and traffic for other 2469 * NICs which needs S/W classification. We assign the primary mac 2470 * addresses to another ring by specifiying a classification rule for 2471 * primary unicast MAC address to the selected ring. The primary Link 2472 * (and its SRS) can continue to poll the assigned Rx ring at all times 2473 * independantly. 2474 * 2475 * Right now we just assign MAC_RX_HW_DEFAULT_RING to note that it is 2476 * primary NIC and later we will check to see how many Rx rings we 2477 * have and can we get a non default Rx ring for the primary MAC. 2478 * 2479 * Note: In future, if no fanout is specified, we try to assign 2 Rx 2480 * rings for the primary Link with the primary MAC address + TCP going 2481 * to one ring and primary MAC address + UDP|SCTP going to other ring. 2482 * Any remaining traffic for primary MAC address can go to the default 2483 * Rx ring and get S/W classified. This way the respective SRSs don't 2484 * need to do proto fanout and don't need to have softrings at all and 2485 * can poll their respective Rx rings. 2486 * 2487 * As an optimization, when a new NIC or VNIC is created, we can get 2488 * only one Rx ring and make it a TCP specific Rx ring and use the 2489 * H/W default Rx ring for the rest (this Rx ring is never polled). 2490 */ 2491 int 2492 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2493 uint32_t link_type) 2494 { 2495 mac_impl_t *mip = mcip->mci_mip; 2496 mac_group_t *group = NULL; 2497 mac_group_t *default_group; 2498 int err; 2499 uint8_t *mac_addr; 2500 mac_rx_group_reserve_type_t rtype = MAC_RX_RESERVE_NONDEFAULT; 2501 mac_group_state_t next_state; 2502 mac_client_impl_t *group_only_mcip; 2503 2504 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2505 2506 switch (link_type) { 2507 case SRST_FLOW: 2508 mac_srs_group_setup(mcip, flent, NULL, link_type); 2509 return (0); 2510 2511 case SRST_LINK: 2512 mac_addr = flent->fe_flow_desc.fd_dst_mac; 2513 2514 /* Check if we need to reserve the default group */ 2515 if (flent->fe_type & FLOW_PRIMARY_MAC) 2516 rtype = MAC_RX_RESERVE_DEFAULT; 2517 2518 if ((mcip->mci_state_flags & MCIS_NO_HWRINGS) == 0) { 2519 /* 2520 * Check to see if we can get an exclusive group for 2521 * this mac address or if there already exists a 2522 * group that has this mac address (case of VLANs). 2523 * If no groups are available, use the default group. 2524 */ 2525 group = mac_reserve_rx_group(mcip, mac_addr, rtype); 2526 } 2527 2528 if (group == NULL) { 2529 if ((mcip->mci_state_flags & MCIS_REQ_HWRINGS) != 0) 2530 return (ENOSPC); 2531 group = &mip->mi_rx_groups[0]; 2532 } 2533 2534 /* 2535 * Some NICs don't support any Rx rings, so there may not 2536 * even be a default group. 2537 */ 2538 if (group != NULL) { 2539 flent->fe_rx_ring_group = group; 2540 /* 2541 * Add the client to the group. This could cause 2542 * either this group to move to the shared state or 2543 * cause the default group to move to the shared state. 2544 * The actions on this group are done here, while the 2545 * actions on the default group are postponed to 2546 * the end of this function. 2547 */ 2548 mac_rx_group_add_client(group, mcip); 2549 next_state = mac_rx_group_next_state(group, 2550 &group_only_mcip); 2551 2552 ASSERT((next_state == MAC_GROUP_STATE_RESERVED && 2553 mcip == group_only_mcip) || 2554 (next_state == MAC_GROUP_STATE_SHARED && 2555 group_only_mcip == NULL)); 2556 2557 mac_set_rx_group_state(group, next_state); 2558 } 2559 2560 /* 2561 * Setup the Rx and Tx SRSes. If we got a pristine group 2562 * exclusively above, mac_srs_group_setup would simply create 2563 * the required SRSes. If we ended up sharing a previously 2564 * reserved group, mac_srs_group_setup would also dismantle the 2565 * SRSes of the previously exclusive group 2566 */ 2567 mac_srs_group_setup(mcip, flent, group, link_type); 2568 2569 /* Program the S/W Classifer */ 2570 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 2571 goto setup_failed; 2572 2573 /* Program the H/W Classifier */ 2574 if ((err = mac_add_macaddr(mip, group, mac_addr, 2575 (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0)) != 0) 2576 goto setup_failed; 2577 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 2578 ASSERT(mcip->mci_unicast != NULL); 2579 break; 2580 2581 default: 2582 ASSERT(B_FALSE); 2583 break; 2584 } 2585 2586 /* 2587 * All broadcast and multicast traffic is received only on the default 2588 * group. If we have setup the datapath for a non-default group above 2589 * then move the default group to shared state to allow distribution of 2590 * incoming broadcast traffic to the other groups and dismantle the 2591 * SRSes over the default group. 2592 */ 2593 if (group != NULL) { 2594 if (group != mip->mi_rx_groups) { 2595 default_group = mip->mi_rx_groups; 2596 if (default_group->mrg_state == 2597 MAC_GROUP_STATE_RESERVED) { 2598 group_only_mcip = MAC_RX_GROUP_ONLY_CLIENT( 2599 default_group); 2600 ASSERT(group_only_mcip != NULL && 2601 mip->mi_nactiveclients > 1); 2602 2603 mac_set_rx_group_state(default_group, 2604 MAC_GROUP_STATE_SHARED); 2605 mac_srs_group_setup(group_only_mcip, 2606 group_only_mcip->mci_flent, 2607 default_group, SRST_LINK); 2608 } 2609 ASSERT(default_group->mrg_state == 2610 MAC_GROUP_STATE_SHARED); 2611 } 2612 /* 2613 * If we get an exclusive group for a VLAN MAC client we 2614 * need to take the s/w path to make the additional check for 2615 * the vid. Disable polling and set it to s/w classification. 2616 */ 2617 if (group->mrg_state == MAC_GROUP_STATE_RESERVED && 2618 i_mac_flow_vid(mcip->mci_flent) != VLAN_ID_NONE) { 2619 mac_rx_switch_grp_to_sw(group); 2620 } 2621 } 2622 return (0); 2623 2624 setup_failed: 2625 mac_datapath_teardown(mcip, flent, link_type); 2626 return (err); 2627 } 2628 2629 void 2630 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2631 uint32_t link_type) 2632 { 2633 mac_impl_t *mip = mcip->mci_mip; 2634 mac_group_t *group = NULL; 2635 mac_client_impl_t *grp_only_mcip; 2636 flow_entry_t *group_only_flent; 2637 mac_group_t *default_group; 2638 boolean_t check_default_group = B_FALSE; 2639 mac_group_state_t next_state; 2640 2641 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2642 2643 switch (link_type) { 2644 case SRST_FLOW: 2645 mac_srs_group_teardown(mcip, flent, SRST_FLOW); 2646 return; 2647 2648 case SRST_LINK: 2649 /* Stop sending packets */ 2650 mac_tx_client_block(mcip); 2651 2652 /* Stop the packets coming from the H/W */ 2653 if (mcip->mci_unicast != NULL) { 2654 int err; 2655 err = mac_remove_macaddr(mcip->mci_unicast); 2656 if (err != 0) { 2657 cmn_err(CE_WARN, "%s: failed to remove a MAC" 2658 " address because of error 0x%x", 2659 mip->mi_name, err); 2660 } 2661 mcip->mci_unicast = NULL; 2662 } 2663 2664 /* Stop the packets coming from the S/W classifier */ 2665 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 2666 mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 2667 2668 /* Now quiesce and destroy all SRS and soft rings */ 2669 mac_srs_group_teardown(mcip, flent, SRST_LINK); 2670 ASSERT((mcip->mci_flent == flent) && 2671 (flent->fe_next == NULL)); 2672 2673 /* 2674 * Release our hold on the group as well. We need 2675 * to check if the shared group has only one client 2676 * left who can use it exclusively. Also, if we 2677 * were the last client, release the group. 2678 */ 2679 group = flent->fe_rx_ring_group; 2680 if (group != NULL) { 2681 mac_rx_group_remove_client(group, mcip); 2682 next_state = mac_rx_group_next_state(group, 2683 &grp_only_mcip); 2684 if (next_state == MAC_GROUP_STATE_RESERVED) { 2685 /* 2686 * Only one client left on this RX group. 2687 */ 2688 ASSERT(grp_only_mcip != NULL); 2689 mac_set_rx_group_state(group, 2690 MAC_GROUP_STATE_RESERVED); 2691 group_only_flent = grp_only_mcip->mci_flent; 2692 2693 /* 2694 * The only remaining client has exclusive 2695 * access on the group. Allow it to 2696 * dynamically poll the H/W rings etc. 2697 */ 2698 mac_srs_group_setup(grp_only_mcip, 2699 group_only_flent, group, SRST_LINK); 2700 mac_rx_group_unmark(group, MR_INCIPIENT); 2701 } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 2702 /* 2703 * This is a non-default group being freed up. 2704 * We need to reevaluate the default group 2705 * to see if the primary client can get 2706 * exclusive access to the default group. 2707 */ 2708 ASSERT(group != mip->mi_rx_groups); 2709 mac_release_rx_group(mcip, group); 2710 mac_set_rx_group_state(group, 2711 MAC_GROUP_STATE_REGISTERED); 2712 check_default_group = B_TRUE; 2713 } else { 2714 ASSERT(next_state == MAC_GROUP_STATE_SHARED); 2715 mac_set_rx_group_state(group, 2716 MAC_GROUP_STATE_SHARED); 2717 mac_rx_group_unmark(group, MR_CONDEMNED); 2718 } 2719 flent->fe_rx_ring_group = NULL; 2720 } 2721 break; 2722 default: 2723 ASSERT(B_FALSE); 2724 break; 2725 } 2726 2727 /* 2728 * The mac client using the default group gets exclusive access to the 2729 * default group if and only if it is the sole client on the entire 2730 * mip. If so set the group state to reserved, and set up the SRSes 2731 * over the default group. 2732 */ 2733 if (check_default_group) { 2734 default_group = mip->mi_rx_groups; 2735 ASSERT(default_group->mrg_state == MAC_GROUP_STATE_SHARED); 2736 next_state = mac_rx_group_next_state(default_group, 2737 &grp_only_mcip); 2738 if (next_state == MAC_GROUP_STATE_RESERVED) { 2739 ASSERT(grp_only_mcip != NULL && 2740 mip->mi_nactiveclients == 1); 2741 mac_set_rx_group_state(default_group, 2742 MAC_GROUP_STATE_RESERVED); 2743 mac_srs_group_setup(grp_only_mcip, 2744 grp_only_mcip->mci_flent, 2745 default_group, SRST_LINK); 2746 mac_rx_group_unmark(default_group, MR_INCIPIENT); 2747 } 2748 } 2749 } 2750 2751 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 2752 2753 static void 2754 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 2755 { 2756 ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 2757 kmem_free(mac_srs->srs_tcp_soft_rings, 2758 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 2759 mac_srs->srs_tcp_soft_rings = NULL; 2760 ASSERT(mac_srs->srs_udp_soft_rings != NULL); 2761 kmem_free(mac_srs->srs_udp_soft_rings, 2762 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 2763 mac_srs->srs_udp_soft_rings = NULL; 2764 ASSERT(mac_srs->srs_oth_soft_rings != NULL); 2765 kmem_free(mac_srs->srs_oth_soft_rings, 2766 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 2767 mac_srs->srs_oth_soft_rings = NULL; 2768 } 2769 2770 /* 2771 * An RX SRS is attached to at most one mac_ring. 2772 * A TX SRS has no rings. 2773 */ 2774 static void 2775 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 2776 { 2777 mac_client_impl_t *mcip; 2778 mac_ring_t *ring; 2779 flow_entry_t *flent; 2780 2781 ring = mac_srs->srs_ring; 2782 if (mac_srs->srs_type & SRST_TX) { 2783 ASSERT(ring == NULL); 2784 return; 2785 } 2786 2787 if (ring == NULL) 2788 return; 2789 2790 /* 2791 * Broadcast flows don't have a client impl association, but they 2792 * use only soft rings. 2793 */ 2794 flent = mac_srs->srs_flent; 2795 mcip = flent->fe_mcip; 2796 ASSERT(mcip != NULL); 2797 2798 ring->mr_classify_type = MAC_NO_CLASSIFIER; 2799 ring->mr_srs = NULL; 2800 } 2801 2802 /* 2803 * Physical unlink and free of the data structures happen below. This is 2804 * driven from mac_flow_destroy(), on the last refrele of a flow. 2805 * 2806 * Assumes Rx srs is 1-1 mapped with an ring. 2807 */ 2808 void 2809 mac_srs_free(mac_soft_ring_set_t *mac_srs) 2810 { 2811 ASSERT(mac_srs->srs_mcip == NULL || 2812 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 2813 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 2814 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 2815 2816 mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE); 2817 mac_srs_ring_free(mac_srs); 2818 mac_srs_soft_rings_free(mac_srs, B_TRUE); 2819 mac_srs_fanout_list_free(mac_srs); 2820 2821 mac_srs->srs_bw = NULL; 2822 kmem_cache_free(mac_srs_cache, mac_srs); 2823 } 2824 2825 static void 2826 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 2827 { 2828 mac_soft_ring_t *softring; 2829 2830 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 2831 2832 mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 2833 if (s_ring_flag == S_RING_CONDEMNED) { 2834 while (mac_srs->srs_soft_ring_condemned_count != 2835 mac_srs->srs_soft_ring_count) 2836 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2837 } else { 2838 while (mac_srs->srs_soft_ring_quiesced_count != 2839 mac_srs->srs_soft_ring_count) 2840 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2841 } 2842 mutex_exit(&mac_srs->srs_lock); 2843 2844 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 2845 softring = softring->s_ring_next) 2846 (void) untimeout(softring->s_ring_tid); 2847 2848 (void) untimeout(mac_srs->srs_tid); 2849 2850 mutex_enter(&mac_srs->srs_lock); 2851 } 2852 2853 /* 2854 * The block comment above mac_rx_classify_flow_state_change explains the 2855 * background. At this point upcalls from the driver (both hardware classified 2856 * and software classified) have been cut off. We now need to quiesce the 2857 * SRS worker, poll, and softring threads. The SRS worker thread serves as 2858 * the master controller. The steps involved are described below in the function 2859 */ 2860 void 2861 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 2862 { 2863 uint_t s_ring_flag; 2864 uint_t srs_poll_wait_flag; 2865 2866 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 2867 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 2868 2869 if (mac_srs->srs_state & SRS_CONDEMNED) { 2870 s_ring_flag = S_RING_CONDEMNED; 2871 srs_poll_wait_flag = SRS_POLL_THR_EXITED; 2872 } else { 2873 s_ring_flag = S_RING_QUIESCE; 2874 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 2875 } 2876 2877 /* 2878 * In the case of Rx SRS wait till the poll thread is done. 2879 */ 2880 if ((mac_srs->srs_type & SRST_TX) == 0 && 2881 mac_srs->srs_poll_thr != NULL) { 2882 while (!(mac_srs->srs_state & srs_poll_wait_flag)) 2883 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2884 2885 /* 2886 * Turn off polling as part of the quiesce operation. 2887 */ 2888 MAC_SRS_POLLING_OFF(mac_srs); 2889 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 2890 } 2891 2892 /* 2893 * Then signal the soft ring worker threads to quiesce or quit 2894 * as needed and then wait till that happens. 2895 */ 2896 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 2897 2898 if (mac_srs->srs_state & SRS_CONDEMNED) 2899 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 2900 else 2901 mac_srs->srs_state |= SRS_QUIESCE_DONE; 2902 cv_signal(&mac_srs->srs_quiesce_done_cv); 2903 } 2904 2905 /* 2906 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 2907 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 2908 * or poll thread. This function is internal to the quiescing logic and is 2909 * called internally from the SRS quiesce or flow quiesce or client quiesce 2910 * higher level functions. 2911 */ 2912 void 2913 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 2914 { 2915 mac_ring_t *ring; 2916 2917 ring = mac_srs->srs_ring; 2918 ASSERT(ring == NULL || ring->mr_refcnt == 0); 2919 2920 if (srs_flag == SRS_CONDEMNED) { 2921 /* 2922 * The SRS is going away. We need to unbind the SRS and SR 2923 * threads before removing from the global SRS list. Otherwise 2924 * there is a small window where the cpu reconfig callbacks 2925 * may miss the SRS in the list walk and DR could fail since 2926 * there are still bound threads. 2927 */ 2928 mac_srs_threads_unbind(mac_srs); 2929 mac_srs_remove_glist(mac_srs); 2930 } 2931 /* 2932 * Wakeup the SRS worker and poll threads. 2933 */ 2934 mutex_enter(&mac_srs->srs_lock); 2935 mac_srs->srs_state |= srs_flag; 2936 cv_signal(&mac_srs->srs_async); 2937 cv_signal(&mac_srs->srs_cv); 2938 mutex_exit(&mac_srs->srs_lock); 2939 } 2940 2941 /* 2942 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 2943 * from the driver are done, then the Rx SRS is quiesced and only then can 2944 * we signal the soft rings. Thus this function can't be called arbitrarily 2945 * without satisfying the prerequisites. On the Tx side, the threads from 2946 * top need to quiesced, then the Tx SRS and only then can we signal the 2947 * Tx soft rings. 2948 */ 2949 static void 2950 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 2951 { 2952 mac_soft_ring_t *softring; 2953 2954 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 2955 softring = softring->s_ring_next) 2956 mac_soft_ring_signal(softring, sr_flag); 2957 } 2958 2959 /* 2960 * The block comment above mac_rx_classify_flow_state_change explains the 2961 * background. At this point the SRS is quiesced and we need to restart the 2962 * SRS worker, poll, and softring threads. The SRS worker thread serves as 2963 * the master controller. The steps involved are described below in the function 2964 */ 2965 void 2966 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 2967 { 2968 boolean_t iam_rx_srs; 2969 mac_soft_ring_t *softring; 2970 2971 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 2972 if ((mac_srs->srs_type & SRST_TX) != 0) { 2973 iam_rx_srs = B_FALSE; 2974 ASSERT((mac_srs->srs_state & 2975 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 2976 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 2977 } else { 2978 iam_rx_srs = B_TRUE; 2979 ASSERT((mac_srs->srs_state & 2980 (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 2981 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 2982 if (mac_srs->srs_poll_thr != NULL) { 2983 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 2984 SRS_POLL_THR_QUIESCED); 2985 } 2986 } 2987 2988 /* 2989 * Signal any quiesced soft ring workers to restart and wait for the 2990 * soft ring down count to come down to zero. 2991 */ 2992 if (mac_srs->srs_soft_ring_quiesced_count != 0) { 2993 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 2994 softring = softring->s_ring_next) { 2995 if (!(softring->s_ring_state & S_RING_QUIESCE)) 2996 continue; 2997 mac_soft_ring_signal(softring, S_RING_RESTART); 2998 } 2999 while (mac_srs->srs_soft_ring_quiesced_count != 0) 3000 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3001 } 3002 3003 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3004 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3005 /* 3006 * Signal the poll thread and ask it to restart. Wait till it 3007 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3008 * cleared. 3009 */ 3010 mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3011 cv_signal(&mac_srs->srs_cv); 3012 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3013 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3014 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3015 } 3016 /* Wake up any waiter waiting for the restart to complete */ 3017 mac_srs->srs_state |= SRS_RESTART_DONE; 3018 cv_signal(&mac_srs->srs_quiesce_done_cv); 3019 } 3020 3021 static void 3022 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3023 { 3024 mutex_enter(&mac_srs->srs_lock); 3025 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3026 ASSERT(mac_srs->srs_worker_cpuid == -1); 3027 mutex_exit(&mac_srs->srs_lock); 3028 return; 3029 } 3030 3031 mac_srs->srs_worker_cpuid = -1; 3032 mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3033 thread_affinity_clear(mac_srs->srs_worker); 3034 mutex_exit(&mac_srs->srs_lock); 3035 } 3036 3037 static void 3038 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3039 { 3040 mutex_enter(&mac_srs->srs_lock); 3041 if (mac_srs->srs_poll_thr == NULL || 3042 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3043 ASSERT(mac_srs->srs_poll_cpuid == -1); 3044 mutex_exit(&mac_srs->srs_lock); 3045 return; 3046 } 3047 3048 mac_srs->srs_poll_cpuid = -1; 3049 mac_srs->srs_state &= ~SRS_POLL_BOUND; 3050 thread_affinity_clear(mac_srs->srs_poll_thr); 3051 mutex_exit(&mac_srs->srs_lock); 3052 } 3053 3054 static void 3055 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3056 { 3057 mac_soft_ring_t *soft_ring; 3058 3059 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3060 3061 mutex_enter(&cpu_lock); 3062 mac_srs_worker_unbind(mac_srs); 3063 if (!(mac_srs->srs_type & SRST_TX)) 3064 mac_srs_poll_unbind(mac_srs); 3065 3066 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3067 soft_ring = soft_ring->s_ring_next) { 3068 mac_soft_ring_unbind(soft_ring); 3069 } 3070 mutex_exit(&cpu_lock); 3071 } 3072 3073 /* 3074 * When a CPU is going away, unbind all MAC threads which are bound 3075 * to that CPU. The affinity of the thread to the CPU is saved to allow 3076 * the thread to be rebound to the CPU if it comes back online. 3077 */ 3078 static void 3079 mac_walk_srs_and_unbind(int cpuid) 3080 { 3081 mac_soft_ring_set_t *mac_srs; 3082 mac_soft_ring_t *soft_ring; 3083 3084 rw_enter(&mac_srs_g_lock, RW_READER); 3085 3086 if ((mac_srs = mac_srs_g_list) == NULL) 3087 goto done; 3088 3089 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3090 if (mac_srs->srs_worker_cpuid == cpuid) { 3091 mac_srs->srs_worker_cpuid_save = cpuid; 3092 mac_srs_worker_unbind(mac_srs); 3093 } 3094 3095 if (!(mac_srs->srs_type & SRST_TX)) { 3096 if (mac_srs->srs_poll_cpuid == cpuid) { 3097 mac_srs->srs_poll_cpuid_save = cpuid; 3098 mac_srs_poll_unbind(mac_srs); 3099 } 3100 } 3101 3102 /* Next tackle the soft rings associated with the srs */ 3103 mutex_enter(&mac_srs->srs_lock); 3104 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3105 soft_ring = soft_ring->s_ring_next) { 3106 if (soft_ring->s_ring_cpuid == cpuid) { 3107 soft_ring->s_ring_cpuid_save = cpuid; 3108 mac_soft_ring_unbind(soft_ring); 3109 } 3110 } 3111 mutex_exit(&mac_srs->srs_lock); 3112 } 3113 done: 3114 rw_exit(&mac_srs_g_lock); 3115 } 3116 3117 /* TX SETUP and TEARDOWN ROUTINES */ 3118 3119 /* 3120 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3121 * handle the case where the number of rings is one. I.e. there is 3122 * a ring pointed to by mac_srs->srs_tx_arg2. 3123 */ 3124 void 3125 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3126 { 3127 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3128 mac_soft_ring_t *soft_ring; 3129 int count = mac_srs->srs_oth_ring_count; 3130 3131 ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3132 soft_ring = mac_soft_ring_create(count, 0, NULL, 3133 (ST_RING_OTH | ST_RING_TX), maxclsyspri, mcip, mac_srs, -1, 3134 NULL, mcip, (mac_resource_handle_t)tx_ring); 3135 mac_srs->srs_oth_ring_count++; 3136 /* 3137 * put this soft ring in quiesce mode too so when we restart 3138 * all soft rings in the srs are in the same state. 3139 */ 3140 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3141 } 3142 3143 static void 3144 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3145 { 3146 int sringcnt; 3147 3148 mutex_enter(&mac_srs->srs_lock); 3149 sringcnt = mac_srs->srs_soft_ring_count; 3150 ASSERT(sringcnt > 0); 3151 mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3152 3153 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3154 while (mac_srs->srs_soft_ring_condemned_count != 1) 3155 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3156 3157 if (softring == mac_srs->srs_soft_ring_head) { 3158 mac_srs->srs_soft_ring_head = softring->s_ring_next; 3159 if (mac_srs->srs_soft_ring_head != NULL) { 3160 mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3161 } else { 3162 mac_srs->srs_soft_ring_tail = NULL; 3163 } 3164 } else { 3165 softring->s_ring_prev->s_ring_next = 3166 softring->s_ring_next; 3167 if (softring->s_ring_next != NULL) { 3168 softring->s_ring_next->s_ring_prev = 3169 softring->s_ring_prev; 3170 } else { 3171 mac_srs->srs_soft_ring_tail = 3172 softring->s_ring_prev; 3173 } 3174 } 3175 mac_srs->srs_soft_ring_count--; 3176 3177 mac_srs->srs_soft_ring_condemned_count--; 3178 mutex_exit(&mac_srs->srs_lock); 3179 3180 mac_soft_ring_free(softring, B_FALSE); 3181 } 3182 3183 void 3184 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3185 { 3186 int i; 3187 mac_soft_ring_t *soft_ring, *remove_sring; 3188 3189 mutex_enter(&mac_srs->srs_lock); 3190 for (i = 0; i < mac_srs->srs_oth_ring_count; i++) { 3191 soft_ring = mac_srs->srs_oth_soft_rings[i]; 3192 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3193 break; 3194 } 3195 mutex_exit(&mac_srs->srs_lock); 3196 ASSERT(i < mac_srs->srs_oth_ring_count); 3197 remove_sring = soft_ring; 3198 mac_soft_ring_remove(mac_srs, remove_sring); 3199 mac_srs_update_fanout_list(mac_srs); 3200 } 3201 3202 /* 3203 * mac_tx_srs_setup(): 3204 * 3205 * Used to setup Tx rings. If no free Tx ring is available, then default 3206 * Tx ring is used. 3207 */ 3208 void 3209 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 3210 uint32_t srs_type) 3211 { 3212 mac_impl_t *mip = mcip->mci_mip; 3213 mac_soft_ring_set_t *tx_srs; 3214 int i, tx_ring_count = 0, tx_rings_reserved = 0; 3215 mac_ring_handle_t *tx_rings = NULL; 3216 uint32_t soft_ring_type; 3217 mac_group_t *grp = NULL; 3218 mac_ring_t *ring; 3219 mac_srs_tx_t *tx; 3220 boolean_t serialize = B_FALSE; 3221 3222 tx_srs = flent->fe_tx_srs; 3223 tx = &tx_srs->srs_tx; 3224 3225 if (tx->st_group != NULL) { 3226 grp = tx->st_group; 3227 tx_ring_count = grp->mrg_cur_count; 3228 } else { 3229 tx_ring_count = mac_tx_ring_count; 3230 } 3231 3232 if (tx_ring_count != 0) { 3233 tx_rings = kmem_zalloc(sizeof (mac_ring_handle_t) * 3234 tx_ring_count, KM_SLEEP); 3235 } 3236 3237 /* 3238 * Just use the default ring for now. We need to use 3239 * the underlying link's ring set instead of the underlying 3240 * NIC's. 3241 */ 3242 if (srs_type == SRST_FLOW || 3243 (mcip->mci_state_flags & MCIS_NO_HWRINGS) != 0) { 3244 /* use default ring */ 3245 tx_rings[0] = (void *)mip->mi_default_tx_ring; 3246 tx_rings_reserved++; 3247 goto rings_assigned; 3248 } 3249 3250 if (mcip->mci_share != NULL) 3251 ring = grp->mrg_rings; 3252 /* 3253 * An attempt is made to reserve 'tx_ring_count' number 3254 * of Tx rings. If tx_ring_count is 0, default Tx ring 3255 * is used. If it is 1, an attempt is made to reserve one 3256 * Tx ring. In both the cases, the ring information is 3257 * stored in Tx SRS. If multiple Tx rings are specified, 3258 * then each Tx ring will have a Tx-side soft ring. All 3259 * these soft rings will be hang off Tx SRS. 3260 */ 3261 for (i = 0; i < tx_ring_count; i++) { 3262 if (mcip->mci_share != NULL) { 3263 /* 3264 * The ring was already chosen and associated 3265 * with the TX group. Save it in the new 3266 * array to keep as much of the code below common 3267 * between the share and non-share cases. 3268 */ 3269 ASSERT(ring != NULL); 3270 tx_rings[i] = (mac_ring_handle_t)ring; 3271 ring = ring->mr_next; 3272 } else { 3273 tx_rings[i] = 3274 (mac_ring_handle_t)mac_reserve_tx_ring(mip, NULL); 3275 if (tx_rings[i] == NULL) { 3276 /* 3277 * We have run out of Tx rings. So 3278 * give the default ring too. 3279 */ 3280 tx_rings[i] = (void *)mip->mi_default_tx_ring; 3281 tx_rings_reserved++; 3282 break; 3283 } 3284 } 3285 tx_rings_reserved++; 3286 } 3287 3288 rings_assigned: 3289 if (mac_tx_serialize || (mip->mi_v12n_level & MAC_VIRT_SERIALIZE)) 3290 serialize = B_TRUE; 3291 /* 3292 * Did we get the requested number of tx rings? 3293 * There are 2 actions we can take depending upon the number 3294 * of tx_rings we got. 3295 * 1) If we got one, then get the tx_ring from the soft ring, 3296 * save it in SRS and free up the soft ring. 3297 * 2) If we got more than 1, then do the tx fanout among the 3298 * rings we obtained. 3299 */ 3300 ASSERT(tx_rings_reserved != 0); 3301 if (tx_rings_reserved == 1) { 3302 tx->st_arg2 = (void *)tx_rings[0]; 3303 /* For ring_count of 0 or 1, set the tx_mode and return */ 3304 if (tx_srs->srs_type & SRST_BW_CONTROL) 3305 tx->st_mode = SRS_TX_BW; 3306 else if (serialize) 3307 tx->st_mode = SRS_TX_SERIALIZE; 3308 else 3309 tx->st_mode = SRS_TX_DEFAULT; 3310 } else { 3311 /* 3312 * We got multiple Tx rings for Tx fanout. 3313 */ 3314 soft_ring_type = ST_RING_OTH | ST_RING_TX; 3315 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3316 tx->st_mode = SRS_TX_BW_FANOUT; 3317 } else { 3318 tx->st_mode = SRS_TX_FANOUT; 3319 if (serialize) 3320 soft_ring_type |= ST_RING_WORKER_ONLY; 3321 } 3322 for (i = 0; i < tx_rings_reserved; i++) { 3323 (void) mac_soft_ring_create(i, 0, NULL, soft_ring_type, 3324 maxclsyspri, mcip, tx_srs, -1, NULL, mcip, 3325 (mac_resource_handle_t)tx_rings[i]); 3326 } 3327 mac_srs_update_fanout_list(tx_srs); 3328 } 3329 tx->st_func = mac_tx_get_func(tx->st_mode); 3330 3331 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 3332 int, tx->st_mode, int, tx_srs->srs_oth_ring_count); 3333 3334 if (tx_ring_count != 0) { 3335 tx->st_ring_count = tx_rings_reserved; 3336 tx->st_rings = kmem_zalloc(sizeof (mac_ring_handle_t) * 3337 tx_rings_reserved, KM_SLEEP); 3338 for (i = 0; i < tx->st_ring_count; i++) 3339 tx->st_rings[i] = tx_rings[i]; 3340 kmem_free(tx_rings, sizeof (mac_ring_handle_t) * tx_ring_count); 3341 } 3342 } 3343 3344 /* 3345 * Update the fanout of a client if its recorded link speed doesn't match 3346 * its current link speed. 3347 */ 3348 void 3349 mac_fanout_recompute_client(mac_client_impl_t *mcip) 3350 { 3351 uint64_t link_speed; 3352 mac_resource_props_t *mcip_mrp; 3353 3354 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 3355 3356 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 3357 MAC_STAT_IFSPEED); 3358 3359 if ((link_speed != 0) && 3360 (link_speed != mcip->mci_flent->fe_nic_speed)) { 3361 mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 3362 mac_fanout_setup(mcip, mcip->mci_flent, 3363 mcip_mrp, mac_rx_deliver, mcip, NULL); 3364 } 3365 } 3366 3367 /* 3368 * Walk through the list of mac clients for the MAC. 3369 * For each active mac client, recompute the number of soft rings 3370 * associated with every client, only if current speed is different 3371 * from the speed that was previously used for soft ring computation. 3372 * If the cable is disconnected whlie the NIC is started, we would get 3373 * notification with speed set to 0. We do not recompute in that case. 3374 */ 3375 void 3376 mac_fanout_recompute(mac_impl_t *mip) 3377 { 3378 mac_client_impl_t *mcip; 3379 3380 i_mac_perim_enter(mip); 3381 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 3382 mip->mi_linkstate != LINK_STATE_UP) { 3383 i_mac_perim_exit(mip); 3384 return; 3385 } 3386 3387 for (mcip = mip->mi_clients_list; mcip != NULL; 3388 mcip = mcip->mci_client_next) { 3389 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 3390 !MCIP_DATAPATH_SETUP(mcip)) 3391 continue; 3392 mac_fanout_recompute_client(mcip); 3393 } 3394 i_mac_perim_exit(mip); 3395 } 3396 3397 /* 3398 * Given a MAC, change the polling state for all its MAC clients. 'enable' is 3399 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 3400 * default. 3401 */ 3402 void 3403 mac_poll_state_change(mac_handle_t mh, boolean_t enable) 3404 { 3405 mac_impl_t *mip = (mac_impl_t *)mh; 3406 mac_client_impl_t *mcip; 3407 3408 i_mac_perim_enter(mip); 3409 if (enable) 3410 mip->mi_state_flags &= ~MIS_POLL_DISABLE; 3411 else 3412 mip->mi_state_flags |= MIS_POLL_DISABLE; 3413 for (mcip = mip->mi_clients_list; mcip != NULL; 3414 mcip = mcip->mci_client_next) 3415 mac_client_update_classifier(mcip, B_TRUE); 3416 i_mac_perim_exit(mip); 3417 } 3418