1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2018 Joyent, Inc. 24 * Copyright 2020 RackTop Systems. 25 * Copyright 2025 Oxide Computer Company 26 */ 27 28 #include <sys/types.h> 29 #include <sys/callb.h> 30 #include <sys/cpupart.h> 31 #include <sys/pool.h> 32 #include <sys/pool_pset.h> 33 #include <sys/sdt.h> 34 #include <sys/strsubr.h> 35 #include <sys/strsun.h> 36 #include <sys/vlan.h> 37 #include <inet/ipsec_impl.h> 38 #include <inet/ip_impl.h> 39 #include <inet/sadb.h> 40 #include <inet/ipsecesp.h> 41 #include <inet/ipsecah.h> 42 43 #include <sys/mac_impl.h> 44 #include <sys/mac_client_impl.h> 45 #include <sys/mac_client_priv.h> 46 #include <sys/mac_soft_ring.h> 47 #include <sys/mac_flow_impl.h> 48 #include <sys/mac_stat.h> 49 50 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 51 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 52 static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 53 static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 54 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 55 56 static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 57 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 58 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 59 static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 60 static void mac_srs_add_glist(mac_soft_ring_set_t *); 61 static void mac_srs_remove_glist(mac_soft_ring_set_t *); 62 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 63 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 64 65 static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 66 static void mac_walk_srs_and_bind(int); 67 static void mac_walk_srs_and_unbind(int); 68 69 extern boolean_t mac_latency_optimize; 70 71 static kmem_cache_t *mac_srs_cache; 72 kmem_cache_t *mac_soft_ring_cache; 73 74 /* 75 * The duration in msec we wait before signalling the soft ring 76 * worker thread in case packets get queued. 77 */ 78 uint32_t mac_soft_ring_worker_wait = 0; 79 80 /* 81 * A global tunable for turning polling on/off. By default, dynamic 82 * polling is always on and is always very beneficial. It should be 83 * turned off with absolute care and for the rare workload (very 84 * low latency sensitive traffic). 85 */ 86 int mac_poll_enable = B_TRUE; 87 88 /* 89 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 90 * Large values could end up in consuming lot of system memory and cause 91 * system hang. 92 */ 93 int mac_soft_ring_max_q_cnt = 1024; 94 int mac_soft_ring_min_q_cnt = 256; 95 int mac_soft_ring_poll_thres = 16; 96 97 boolean_t mac_tx_serialize = B_FALSE; 98 99 /* 100 * mac_tx_srs_hiwat is the queue depth threshold at which callers of 101 * mac_tx() will be notified of flow control condition. 102 * 103 * TCP does not honour flow control condition sent up by mac_tx(). 104 * Thus provision is made for TCP to allow more packets to be queued 105 * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 106 * 107 * Note that mac_tx_srs_hiwat is always be lesser than 108 * mac_tx_srs_max_q_cnt. 109 */ 110 uint32_t mac_tx_srs_max_q_cnt = 100000; 111 uint32_t mac_tx_srs_hiwat = 1000; 112 113 /* 114 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 115 * 116 * Global tunables that determines the number of soft rings to be used for 117 * fanning out incoming traffic on a link. These count will be used only 118 * when no explicit set of CPUs was assigned to the data-links. 119 * 120 * mac_rx_soft_ring_count tunable will come into effect only if 121 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 122 * default only for sun4v platforms. 123 * 124 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 125 * 10Gbps link and is not dependent upon mac_soft_ring_enable. 126 * 127 * The number of soft rings for fanout for a link or a flow is determined 128 * by mac_compute_soft_ring_count() routine. This routine will take into 129 * account mac_soft_ring_enable, mac_rx_soft_ring_count and 130 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 131 * 132 * If a bandwidth is specified, the determination of the number of soft 133 * rings is based on specified bandwidth, CPU speed and number of CPUs in 134 * the system. 135 */ 136 uint_t mac_rx_soft_ring_count = 8; 137 uint_t mac_rx_soft_ring_10gig_count = 8; 138 139 /* 140 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 141 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 142 * list is used to walk the list of all MAC threads when a CPU is 143 * coming online or going offline. 144 */ 145 static mac_soft_ring_set_t *mac_srs_g_list = NULL; 146 static krwlock_t mac_srs_g_lock; 147 148 /* 149 * Whether the SRS threads should be bound, or not. 150 */ 151 boolean_t mac_srs_thread_bind = B_TRUE; 152 153 /* 154 * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 155 * dladm command would override this. 156 */ 157 boolean_t mac_tx_intr_retarget = B_FALSE; 158 boolean_t mac_rx_intr_retarget = B_FALSE; 159 160 /* 161 * If cpu bindings are specified by user, then Tx SRS and its soft 162 * rings should also be bound to the CPUs specified by user. The 163 * CPUs for Tx bindings are at the end of the cpu list provided by 164 * the user. If enough CPUs are not available (for Tx and Rx 165 * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 166 */ 167 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 168 processorid_t cpuid; \ 169 int i; \ 170 mac_soft_ring_t *softring; \ 171 mac_cpus_t *srs_cpu; \ 172 \ 173 srs_cpu = &mac_tx_srs->srs_cpu; \ 174 cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 175 mac_srs_worker_bind(mac_tx_srs, cpuid); \ 176 if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 177 for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 178 cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 179 softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 180 if (cpuid != -1) { \ 181 (void) mac_soft_ring_bind(softring, \ 182 cpuid); \ 183 } \ 184 } \ 185 } \ 186 } 187 188 /* 189 * Re-targeting is allowed only for exclusive group or for primary. 190 */ 191 #define RETARGETABLE_CLIENT(group, mcip) \ 192 ((((group) != NULL) && \ 193 ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 194 mac_is_primary_client(mcip)) 195 196 #define MAC_RING_RETARGETABLE(ring) \ 197 (((ring) != NULL) && \ 198 ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 199 !((ring)->mr_info.mri_intr.mi_ddi_shared)) 200 201 202 /* INIT and FINI ROUTINES */ 203 204 void 205 mac_soft_ring_init(void) 206 { 207 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 208 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 209 210 mac_srs_cache = kmem_cache_create("mac_srs_cache", 211 sizeof (mac_soft_ring_set_t), 212 64, NULL, NULL, NULL, NULL, NULL, 0); 213 214 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 215 mutex_enter(&cpu_lock); 216 register_cpu_setup_func(mac_srs_cpu_setup, NULL); 217 mutex_exit(&cpu_lock); 218 } 219 220 void 221 mac_soft_ring_finish(void) 222 { 223 mutex_enter(&cpu_lock); 224 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 225 mutex_exit(&cpu_lock); 226 rw_destroy(&mac_srs_g_lock); 227 kmem_cache_destroy(mac_soft_ring_cache); 228 kmem_cache_destroy(mac_srs_cache); 229 } 230 231 static void 232 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 233 { 234 mac_soft_ring_t *softring, *next, *head; 235 236 /* 237 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 238 * DR. The callbacks from DR are called with cpu_lock held, and hence 239 * can't wait to grab the mac perimeter. The soft ring list is hence 240 * protected for read access by srs_lock. Changing the soft ring list 241 * needs the mac perimeter and the srs_lock. 242 */ 243 mutex_enter(&mac_srs->srs_lock); 244 245 head = mac_srs->srs_soft_ring_head; 246 mac_srs->srs_soft_ring_head = NULL; 247 mac_srs->srs_soft_ring_tail = NULL; 248 mac_srs->srs_soft_ring_count = 0; 249 250 mutex_exit(&mac_srs->srs_lock); 251 252 for (softring = head; softring != NULL; softring = next) { 253 next = softring->s_ring_next; 254 mac_soft_ring_free(softring); 255 } 256 } 257 258 static void 259 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 260 { 261 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 262 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 263 264 rw_enter(&mac_srs_g_lock, RW_WRITER); 265 mutex_enter(&mac_srs->srs_lock); 266 267 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 268 269 if (mac_srs_g_list == NULL) { 270 mac_srs_g_list = mac_srs; 271 } else { 272 mac_srs->srs_next = mac_srs_g_list; 273 mac_srs_g_list->srs_prev = mac_srs; 274 mac_srs->srs_prev = NULL; 275 mac_srs_g_list = mac_srs; 276 } 277 mac_srs->srs_state |= SRS_IN_GLIST; 278 279 mutex_exit(&mac_srs->srs_lock); 280 rw_exit(&mac_srs_g_lock); 281 } 282 283 static void 284 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 285 { 286 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 287 288 rw_enter(&mac_srs_g_lock, RW_WRITER); 289 mutex_enter(&mac_srs->srs_lock); 290 291 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 292 293 if (mac_srs == mac_srs_g_list) { 294 mac_srs_g_list = mac_srs->srs_next; 295 if (mac_srs_g_list != NULL) 296 mac_srs_g_list->srs_prev = NULL; 297 } else { 298 mac_srs->srs_prev->srs_next = mac_srs->srs_next; 299 if (mac_srs->srs_next != NULL) 300 mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 301 } 302 mac_srs->srs_state &= ~SRS_IN_GLIST; 303 304 mutex_exit(&mac_srs->srs_lock); 305 rw_exit(&mac_srs_g_lock); 306 } 307 308 /* POLLING SETUP AND TEAR DOWN ROUTINES */ 309 310 /* 311 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 312 * 313 * These routines are used to call back into the upper layer 314 * (primarily TCP squeue) to stop polling the soft rings or 315 * restart polling. 316 */ 317 void 318 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 319 mac_soft_ring_set_t *mac_srs) 320 { 321 mac_soft_ring_t *softring; 322 323 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 324 325 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 326 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 327 return; 328 } 329 330 for (softring = mac_srs->srs_soft_ring_head; 331 softring != NULL; softring = softring->s_ring_next) { 332 if ((softring->s_ring_type & ST_RING_TCP) && 333 (softring->s_ring_rx_arg2 != NULL)) { 334 mcip->mci_resource_quiesce(mcip->mci_resource_arg, 335 softring->s_ring_rx_arg2); 336 } 337 } 338 } 339 340 void 341 mac_srs_client_poll_restart(mac_client_impl_t *mcip, 342 mac_soft_ring_set_t *mac_srs) 343 { 344 mac_soft_ring_t *softring; 345 346 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 347 348 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 349 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 350 return; 351 } 352 353 for (softring = mac_srs->srs_soft_ring_head; 354 softring != NULL; softring = softring->s_ring_next) { 355 if ((softring->s_ring_type & ST_RING_TCP) && 356 (softring->s_ring_rx_arg2 != NULL)) { 357 mcip->mci_resource_restart(mcip->mci_resource_arg, 358 softring->s_ring_rx_arg2); 359 } 360 } 361 } 362 363 /* 364 * Register the given SRS and associated soft rings with the consumer and 365 * enable the polling interface used by the consumer.(i.e IP) over this 366 * SRS and associated soft rings. 367 */ 368 void 369 mac_srs_client_poll_enable(mac_client_impl_t *mcip, 370 mac_soft_ring_set_t *mac_srs) 371 { 372 mac_rx_fifo_t mrf; 373 mac_soft_ring_t *softring; 374 375 ASSERT(mac_srs->srs_mcip == mcip); 376 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 377 378 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 379 return; 380 381 bzero(&mrf, sizeof (mac_rx_fifo_t)); 382 mrf.mrf_type = MAC_RX_FIFO; 383 384 /* 385 * A SRS is capable of acting as a soft ring for cases 386 * where no fanout is needed. This is the case for userland 387 * flows. 388 */ 389 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 390 return; 391 392 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 393 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 394 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 395 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 396 397 softring = mac_srs->srs_soft_ring_head; 398 while (softring != NULL) { 399 if (mcip->mci_direct_rx.mdrx_v4 != NULL && 400 softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 401 /* 402 * TCP and UDP support DLS bypass. Squeue polling 403 * support implies DLS bypass since the squeue poll 404 * path does not have DLS processing. 405 */ 406 mac_soft_ring_dls_bypass(softring, 407 mcip->mci_direct_rx.mdrx_v4, 408 mcip->mci_direct_rx.mdrx_arg_v4); 409 } 410 if (mcip->mci_direct_rx.mdrx_v6 != NULL && 411 softring->s_ring_type & (ST_RING_TCP6 | ST_RING_UDP6)) { 412 /* 413 * TCP and UDP support DLS bypass. Squeue polling 414 * support implies DLS bypass since the squeue poll 415 * path does not have DLS processing. 416 */ 417 mac_soft_ring_dls_bypass(softring, 418 mcip->mci_direct_rx.mdrx_v6, 419 mcip->mci_direct_rx.mdrx_arg_v6); 420 } 421 422 /* 423 * Non-TCP protocols don't support squeues. Hence we don't 424 * make any ring addition callbacks for non-TCP rings 425 */ 426 if (!(softring->s_ring_type & ST_RING_TCP)) { 427 softring->s_ring_rx_arg2 = NULL; 428 softring = softring->s_ring_next; 429 continue; 430 } 431 mrf.mrf_rx_arg = softring; 432 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 433 mrf.mrf_cpu_id = softring->s_ring_cpuid; 434 mrf.mrf_flow_priority = mac_srs->srs_pri; 435 436 softring->s_ring_rx_arg2 = mcip->mci_resource_add( 437 mcip->mci_resource_arg, (mac_resource_t *)&mrf); 438 439 softring = softring->s_ring_next; 440 } 441 } 442 443 /* 444 * Unregister the given SRS and associated soft rings with the consumer and 445 * disable the polling interface used by the consumer.(i.e IP) over this 446 * SRS and associated soft rings. 447 */ 448 void 449 mac_srs_client_poll_disable(mac_client_impl_t *mcip, 450 mac_soft_ring_set_t *mac_srs) 451 { 452 mac_soft_ring_t *softring; 453 454 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 455 456 /* 457 * A SRS is capable of acting as a soft ring for cases 458 * where no protocol fanout is needed. This is the case 459 * for userland flows. Nothing to do here. 460 */ 461 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 462 return; 463 464 mutex_enter(&mac_srs->srs_lock); 465 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 466 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 467 mutex_exit(&mac_srs->srs_lock); 468 return; 469 } 470 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 471 mutex_exit(&mac_srs->srs_lock); 472 473 /* 474 * DLS bypass is now disabled in the case of both TCP and UDP. 475 * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 476 * callback. In addition, in the case of TCP, invoke IP's callback 477 * for ring removal. 478 */ 479 for (softring = mac_srs->srs_soft_ring_head; 480 softring != NULL; softring = softring->s_ring_next) { 481 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 482 continue; 483 484 if ((softring->s_ring_type & ST_RING_TCP) && 485 softring->s_ring_rx_arg2 != NULL) { 486 mcip->mci_resource_remove(mcip->mci_resource_arg, 487 softring->s_ring_rx_arg2); 488 } 489 490 mutex_enter(&softring->s_ring_lock); 491 while (softring->s_ring_state & S_RING_PROC) { 492 softring->s_ring_state |= S_RING_CLIENT_WAIT; 493 cv_wait(&softring->s_ring_client_cv, 494 &softring->s_ring_lock); 495 } 496 softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 497 softring->s_ring_rx_arg2 = NULL; 498 softring->s_ring_rx_func = mac_rx_deliver; 499 softring->s_ring_rx_arg1 = mcip; 500 mutex_exit(&softring->s_ring_lock); 501 } 502 } 503 504 /* 505 * Enable or disable poll capability of the SRS on the underlying Rx ring. 506 * 507 * There is a need to enable or disable the poll capability of an SRS over an 508 * Rx ring depending on the number of mac clients sharing the ring and also 509 * whether user flows are configured on it. However the poll state is actively 510 * manipulated by the SRS worker and poll threads and uncoordinated changes by 511 * yet another thread to the underlying capability can surprise them leading 512 * to assert failures. Instead we quiesce the SRS, make the changes and then 513 * restart the SRS. 514 */ 515 static void 516 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 517 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 518 { 519 boolean_t need_restart = B_FALSE; 520 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 521 mac_ring_t *ring; 522 523 if (!SRS_QUIESCED(mac_srs)) { 524 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 525 need_restart = B_TRUE; 526 } 527 528 ring = mac_srs->srs_ring; 529 if ((ring != NULL) && 530 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 531 if (turn_off_poll_capab) 532 mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 533 else if (mac_poll_enable) 534 mac_srs->srs_state |= SRS_POLLING_CAPAB; 535 } 536 srs_rx->sr_lower_proc = rx_func; 537 538 if (need_restart) 539 mac_rx_srs_restart(mac_srs); 540 } 541 542 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 543 544 /* 545 * Return the next CPU to be used to bind a MAC kernel thread. 546 * If a cpupart is specified, the cpu chosen must be from that 547 * cpu partition. 548 */ 549 static processorid_t 550 mac_next_bind_cpu(cpupart_t *cpupart) 551 { 552 static cpu_t *cp = NULL; 553 cpu_t *cp_start; 554 555 ASSERT(MUTEX_HELD(&cpu_lock)); 556 557 if (cp == NULL) 558 cp = cpu_list; 559 560 cp = cp->cpu_next_onln; 561 cp_start = cp; 562 563 do { 564 if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 565 return (cp->cpu_id); 566 567 } while ((cp = cp->cpu_next_onln) != cp_start); 568 569 return (-1); /* No matching CPU found online */ 570 } 571 572 /* ARGSUSED */ 573 static int 574 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 575 { 576 ASSERT(MUTEX_HELD(&cpu_lock)); 577 switch (what) { 578 case CPU_CONFIG: 579 case CPU_ON: 580 case CPU_CPUPART_IN: 581 mac_walk_srs_and_bind(id); 582 break; 583 584 case CPU_UNCONFIG: 585 case CPU_OFF: 586 case CPU_CPUPART_OUT: 587 mac_walk_srs_and_unbind(id); 588 break; 589 590 default: 591 break; 592 } 593 return (0); 594 } 595 596 /* 597 * mac_compute_soft_ring_count(): 598 * 599 * This routine computes the number of soft rings needed to handle incoming 600 * load given a flow_entry. 601 * 602 * The routine does the following: 603 * 1) soft rings will be created if mac_soft_ring_enable is set. 604 * 2) If the underlying link is a 10Gbps link, then soft rings will be 605 * created even if mac_soft_ring_enable is not set. The number of soft 606 * rings, so created, will equal mac_rx_soft_ring_10gig_count. 607 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 608 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 609 * 10Gbps link. 610 * 611 * If a bandwidth limit is specified, the number that gets computed is 612 * dependent upon CPU speed, the number of Rx rings configured, and 613 * the bandwidth limit. 614 * If more Rx rings are available, less number of soft rings is needed. 615 * 616 * mac_use_bw_heuristic is another "hidden" variable that can be used to 617 * override the default use of soft ring count computation. Depending upon 618 * the usefulness of it, mac_use_bw_heuristic can later be made into a 619 * data-link property or removed altogether. 620 * 621 * TODO: Cleanup and tighten some of the assumptions. 622 */ 623 boolean_t mac_check_overlay = B_TRUE; 624 boolean_t mac_use_bw_heuristic = B_TRUE; 625 static int 626 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 627 { 628 uint64_t cpu_speed, bw = 0; 629 int srings = 0; 630 boolean_t bw_enabled = B_FALSE; 631 mac_client_impl_t *mcip = flent->fe_mcip; 632 633 ASSERT(!(flent->fe_type & FLOW_USER)); 634 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 635 mac_use_bw_heuristic) { 636 /* bandwidth enabled */ 637 bw_enabled = B_TRUE; 638 bw = flent->fe_resource_props.mrp_maxbw; 639 } 640 if (!bw_enabled) { 641 /* No bandwidth enabled */ 642 if (mac_soft_ring_enable) 643 srings = mac_rx_soft_ring_count; 644 645 /* Is this a 10Gig link? */ 646 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 647 MAC_STAT_IFSPEED); 648 /* convert to Mbps */ 649 if (((flent->fe_nic_speed)/1000000) > 1000 && 650 mac_rx_soft_ring_10gig_count > 0) { 651 /* This is a 10Gig link */ 652 srings = mac_rx_soft_ring_10gig_count; 653 /* 654 * Use 2 times mac_rx_soft_ring_10gig_count for 655 * sun4v systems. 656 */ 657 if (mac_soft_ring_enable) 658 srings = srings * 2; 659 } else if (mac_check_overlay == B_TRUE && 660 (mcip->mci_state_flags & MCIS_IS_VNIC) != 0) { 661 /* Is this a VNIC on an overlay? */ 662 mac_handle_t mh = (mac_handle_t)mcip->mci_mip; 663 if (mac_is_overlay(mh) == B_TRUE) { 664 srings = mac_rx_soft_ring_10gig_count; 665 } 666 } 667 668 669 } else { 670 /* 671 * Soft ring computation using CPU speed and specified 672 * bandwidth limit. 673 */ 674 /* Assumption: all CPUs have the same frequency */ 675 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 676 677 /* cpu_speed is in MHz; make bw in units of Mbps. */ 678 bw = bw/1000000; 679 680 if (bw >= 1000) { 681 /* 682 * bw is greater than or equal to 1Gbps. 683 * The number of soft rings required is a function 684 * of bandwidth and CPU speed. To keep this simple, 685 * let's use this rule: 1GHz CPU can handle 1Gbps. 686 * If bw is less than 1 Gbps, then there is no need 687 * for soft rings. Assumption is that CPU speeds 688 * (on modern systems) are at least 1GHz. 689 */ 690 srings = bw/cpu_speed; 691 if (srings <= 1 && mac_soft_ring_enable) { 692 /* 693 * Give at least 2 soft rings 694 * for sun4v systems 695 */ 696 srings = 2; 697 } 698 } 699 } 700 /* 701 * If the flent has multiple Rx SRSs, then each SRS need not 702 * have that many soft rings on top of it. The number of 703 * soft rings for each Rx SRS is found by dividing srings by 704 * rx_srs_cnt. 705 */ 706 if (rx_srs_cnt > 1) { 707 int remainder; 708 709 remainder = srings%rx_srs_cnt; 710 srings = srings/rx_srs_cnt; 711 if (remainder != 0) 712 srings++; 713 /* 714 * Fanning out to 1 soft ring is not very useful. 715 * Set it as well to 0 and mac_srs_fanout_init() 716 * will take care of creating a single soft ring 717 * for proto fanout. 718 */ 719 if (srings == 1) 720 srings = 0; 721 } 722 /* Do some more massaging */ 723 srings = min(srings, maxcpus); 724 srings = min(srings, MAX_SR_FANOUT); 725 return (srings); 726 } 727 728 /* 729 * mac_tx_cpu_init: 730 * set up CPUs for Tx interrupt re-targeting and Tx worker 731 * thread binding 732 */ 733 static void 734 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 735 cpupart_t *cpupart) 736 { 737 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 738 mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 739 mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 740 mac_soft_ring_t *sringp; 741 mac_ring_t *ring; 742 processorid_t worker_cpuid; 743 boolean_t retargetable_client = B_FALSE; 744 int i, j; 745 746 if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 747 flent->fe_mcip)) { 748 retargetable_client = B_TRUE; 749 } 750 751 if (MAC_TX_SOFT_RINGS(tx_srs)) { 752 if (mrp != NULL) 753 j = mrp->mrp_ncpus - 1; 754 for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 755 if (mrp != NULL) { 756 if (j < 0) 757 j = mrp->mrp_ncpus - 1; 758 worker_cpuid = mrp->mrp_cpu[j]; 759 } else { 760 /* 761 * Bind interrupt to the next CPU available 762 * and leave the worker unbound. 763 */ 764 worker_cpuid = -1; 765 } 766 sringp = tx_srs->srs_tx_soft_rings[i]; 767 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 768 srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 769 if (MAC_RING_RETARGETABLE(ring) && 770 retargetable_client) { 771 mutex_enter(&cpu_lock); 772 srs_cpu->mc_tx_intr_cpu[i] = 773 (mrp != NULL) ? mrp->mrp_cpu[j] : 774 (mac_tx_intr_retarget ? 775 mac_next_bind_cpu(cpupart) : -1); 776 mutex_exit(&cpu_lock); 777 } else { 778 srs_cpu->mc_tx_intr_cpu[i] = -1; 779 } 780 if (mrp != NULL) 781 j--; 782 } 783 } else { 784 /* Tx mac_ring_handle_t is stored in st_arg2 */ 785 srs_cpu->mc_tx_fanout_cpus[0] = 786 (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 787 ring = (mac_ring_t *)srs_tx->st_arg2; 788 if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 789 mutex_enter(&cpu_lock); 790 srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 791 mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 792 (mac_tx_intr_retarget ? 793 mac_next_bind_cpu(cpupart) : -1); 794 mutex_exit(&cpu_lock); 795 } else { 796 srs_cpu->mc_tx_intr_cpu[0] = -1; 797 } 798 } 799 } 800 801 /* 802 * Assignment of user specified CPUs to a link. 803 * 804 * Minimum CPUs required to get an optimal assignmet: 805 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 806 * flag is set -- one for polling, one for fanout soft ring. 807 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 808 * for polling, one for SRS worker thread and one for fanout soft ring. 809 * 810 * The CPUs needed for Tx side is equal to the number of Tx rings 811 * the link is using. 812 * 813 * mac_flow_user_cpu_init() categorizes the CPU assignment depending 814 * upon the number of CPUs in 3 different buckets. 815 * 816 * In the first bucket, the most optimal case is handled. The user has 817 * passed enough number of CPUs and every thread gets its own CPU. 818 * 819 * The second and third are the sub-optimal cases. Enough CPUs are not 820 * available. 821 * 822 * The second bucket handles the case where atleast one distinct CPU is 823 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 824 * SRS or soft rings). 825 * 826 * In the third case (worst case scenario), specified CPU count is less 827 * than the Rx rings configured for the link. In this case, we round 828 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 829 */ 830 static void 831 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 832 { 833 mac_soft_ring_set_t *rx_srs, *tx_srs; 834 int i, srs_cnt; 835 mac_cpus_t *srs_cpu; 836 int no_of_cpus, cpu_cnt; 837 int rx_srs_cnt, reqd_rx_cpu_cnt; 838 int fanout_cpu_cnt, reqd_tx_cpu_cnt; 839 int reqd_poll_worker_cnt, fanout_cnt_per_srs; 840 mac_resource_props_t *emrp = &flent->fe_effective_props; 841 842 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 843 /* 844 * The check for nbc_ncpus to be within limits for 845 * the user specified case was done earlier and if 846 * not within limits, an error would have been 847 * returned to the user. 848 */ 849 ASSERT(mrp->mrp_ncpus > 0); 850 851 no_of_cpus = mrp->mrp_ncpus; 852 853 if (mrp->mrp_rx_intr_cpu != -1) { 854 /* 855 * interrupt has been re-targetted. Poll 856 * thread needs to be bound to interrupt 857 * CPU. 858 * 859 * Find where in the list is the intr 860 * CPU and swap it with the first one. 861 * We will be using the first CPU in the 862 * list for poll. 863 */ 864 for (i = 0; i < no_of_cpus; i++) { 865 if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 866 break; 867 } 868 mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 869 mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 870 } 871 872 /* 873 * Requirements: 874 * The number of CPUs that each Rx ring needs is dependent 875 * upon mac_latency_optimize flag. 876 * 1) If set, atleast 2 CPUs are needed -- one for 877 * polling, one for fanout soft ring. 878 * 2) If not set, then atleast 3 CPUs are needed -- one 879 * for polling, one for srs worker thread, and one for 880 * fanout soft ring. 881 */ 882 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 883 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 884 reqd_rx_cpu_cnt = mac_latency_optimize ? 885 (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 886 887 /* How many CPUs are needed for Tx side? */ 888 tx_srs = flent->fe_tx_srs; 889 reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 890 tx_srs->srs_tx_ring_count : 1; 891 892 /* CPUs needed for Rx SRSes poll and worker threads */ 893 reqd_poll_worker_cnt = mac_latency_optimize ? 894 rx_srs_cnt : rx_srs_cnt * 2; 895 896 /* Has the user provided enough CPUs? */ 897 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 898 /* 899 * Best case scenario. There is enough CPUs. All 900 * Rx rings will get their own set of CPUs plus 901 * Tx soft rings will get their own. 902 */ 903 /* 904 * fanout_cpu_cnt is the number of CPUs available 905 * for Rx side fanout soft rings. 906 */ 907 fanout_cpu_cnt = no_of_cpus - 908 reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 909 910 /* 911 * Divide fanout_cpu_cnt by rx_srs_cnt to find 912 * out how many fanout soft rings each Rx SRS 913 * can have. 914 */ 915 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 916 917 /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 918 fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 919 920 /* Do the assignment for the default Rx ring */ 921 cpu_cnt = 0; 922 rx_srs = flent->fe_rx_srs[0]; 923 ASSERT(rx_srs->srs_ring == NULL); 924 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 925 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 926 srs_cpu = &rx_srs->srs_cpu; 927 srs_cpu->mc_ncpus = no_of_cpus; 928 bcopy(mrp->mrp_cpu, 929 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 930 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 931 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 932 /* Retarget the interrupt to the same CPU as the poll */ 933 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 934 srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 935 srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 936 for (i = 0; i < fanout_cnt_per_srs; i++) 937 srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 938 939 /* Do the assignment for h/w Rx SRSes */ 940 if (flent->fe_rx_srs_cnt > 1) { 941 cpu_cnt = 0; 942 for (srs_cnt = 1; 943 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 944 rx_srs = flent->fe_rx_srs[srs_cnt]; 945 ASSERT(rx_srs->srs_ring != NULL); 946 if (rx_srs->srs_fanout_state == 947 SRS_FANOUT_INIT) { 948 rx_srs->srs_fanout_state = 949 SRS_FANOUT_REINIT; 950 } 951 srs_cpu = &rx_srs->srs_cpu; 952 srs_cpu->mc_ncpus = no_of_cpus; 953 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 954 sizeof (srs_cpu->mc_cpus)); 955 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 956 /* The first CPU in the list is the intr CPU */ 957 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 958 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 959 srs_cpu->mc_rx_workerid = 960 (mac_latency_optimize ? 961 srs_cpu->mc_rx_pollid : 962 mrp->mrp_cpu[cpu_cnt++]); 963 for (i = 0; i < fanout_cnt_per_srs; i++) { 964 srs_cpu->mc_rx_fanout_cpus[i] = 965 mrp->mrp_cpu[cpu_cnt++]; 966 } 967 ASSERT(cpu_cnt <= no_of_cpus); 968 } 969 } 970 goto tx_cpu_init; 971 } 972 973 /* 974 * Sub-optimal case. 975 * We have the following information: 976 * no_of_cpus - no. of cpus that user passed. 977 * rx_srs_cnt - no. of rx rings. 978 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 979 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 980 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 981 */ 982 /* 983 * If we bind the Rx fanout soft rings to the same CPUs 984 * as poll/worker, would that be enough? 985 */ 986 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 987 boolean_t worker_assign = B_FALSE; 988 989 /* 990 * If mac_latency_optimize is not set, are there 991 * enough CPUs to assign a CPU for worker also? 992 */ 993 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 994 worker_assign = B_TRUE; 995 /* 996 * Zero'th Rx SRS is the default Rx ring. It is not 997 * associated with h/w Rx ring. 998 */ 999 rx_srs = flent->fe_rx_srs[0]; 1000 ASSERT(rx_srs->srs_ring == NULL); 1001 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1002 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1003 cpu_cnt = 0; 1004 srs_cpu = &rx_srs->srs_cpu; 1005 srs_cpu->mc_ncpus = no_of_cpus; 1006 bcopy(mrp->mrp_cpu, 1007 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 1008 srs_cpu->mc_rx_fanout_cnt = 1; 1009 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 1010 /* Retarget the interrupt to the same CPU as the poll */ 1011 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1012 srs_cpu->mc_rx_workerid = 1013 ((!mac_latency_optimize && worker_assign) ? 1014 mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 1015 1016 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1017 1018 /* Do CPU bindings for SRSes having h/w Rx rings */ 1019 if (flent->fe_rx_srs_cnt > 1) { 1020 cpu_cnt = 0; 1021 for (srs_cnt = 1; 1022 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1023 rx_srs = flent->fe_rx_srs[srs_cnt]; 1024 ASSERT(rx_srs->srs_ring != NULL); 1025 if (rx_srs->srs_fanout_state == 1026 SRS_FANOUT_INIT) { 1027 rx_srs->srs_fanout_state = 1028 SRS_FANOUT_REINIT; 1029 } 1030 srs_cpu = &rx_srs->srs_cpu; 1031 srs_cpu->mc_ncpus = no_of_cpus; 1032 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1033 sizeof (srs_cpu->mc_cpus)); 1034 srs_cpu->mc_rx_pollid = 1035 mrp->mrp_cpu[cpu_cnt]; 1036 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1037 srs_cpu->mc_rx_workerid = 1038 ((!mac_latency_optimize && worker_assign) ? 1039 mrp->mrp_cpu[++cpu_cnt] : 1040 srs_cpu->mc_rx_pollid); 1041 srs_cpu->mc_rx_fanout_cnt = 1; 1042 srs_cpu->mc_rx_fanout_cpus[0] = 1043 mrp->mrp_cpu[cpu_cnt]; 1044 cpu_cnt++; 1045 ASSERT(cpu_cnt <= no_of_cpus); 1046 } 1047 } 1048 goto tx_cpu_init; 1049 } 1050 1051 /* 1052 * Real sub-optimal case. Not enough CPUs for poll and 1053 * Tx soft rings. Do a round robin assignment where 1054 * each Rx SRS will get the same CPU for poll, worker 1055 * and fanout soft ring. 1056 */ 1057 cpu_cnt = 0; 1058 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1059 rx_srs = flent->fe_rx_srs[srs_cnt]; 1060 srs_cpu = &rx_srs->srs_cpu; 1061 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1062 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1063 srs_cpu->mc_ncpus = no_of_cpus; 1064 bcopy(mrp->mrp_cpu, 1065 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 1066 srs_cpu->mc_rx_fanout_cnt = 1; 1067 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 1068 /* Retarget the interrupt to the same CPU as the poll */ 1069 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1070 srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 1071 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1072 if (++cpu_cnt >= no_of_cpus) 1073 cpu_cnt = 0; 1074 } 1075 1076 tx_cpu_init: 1077 mac_tx_cpu_init(flent, mrp, NULL); 1078 1079 /* 1080 * Copy the user specified CPUs to the effective CPUs 1081 */ 1082 for (i = 0; i < mrp->mrp_ncpus; i++) { 1083 emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 1084 } 1085 emrp->mrp_ncpus = mrp->mrp_ncpus; 1086 emrp->mrp_mask = mrp->mrp_mask; 1087 bzero(emrp->mrp_pool, MAXPATHLEN); 1088 } 1089 1090 /* 1091 * mac_flow_cpu_init(): 1092 * 1093 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1094 * the CPU binding information in srs_cpu for all Rx SRSes associated 1095 * with a flent. 1096 */ 1097 static void 1098 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1099 { 1100 mac_soft_ring_set_t *rx_srs; 1101 processorid_t cpuid; 1102 int i, j, k, srs_cnt, maxcpus, soft_ring_cnt = 0; 1103 mac_cpus_t *srs_cpu; 1104 mac_resource_props_t *emrp = &flent->fe_effective_props; 1105 1106 /* 1107 * The maximum number of CPUs available can either be 1108 * the number of CPUs in the pool or the number of CPUs 1109 * in the system. 1110 */ 1111 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 1112 /* 1113 * We cannot exceed the hard limit imposed by data structures. 1114 * Leave space for polling CPU and the SRS worker thread when 1115 * "mac_latency_optimize" is not set. 1116 */ 1117 maxcpus = MIN(maxcpus, MRP_NCPUS - 2); 1118 1119 /* 1120 * Compute the number of soft rings needed on top for each Rx 1121 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1122 * associated with h/w Rx rings. Soft ring count needed for 1123 * each h/w Rx SRS is computed and the same is applied to 1124 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1125 * is the software classified Rx SRS. 1126 */ 1127 soft_ring_cnt = mac_compute_soft_ring_count(flent, 1128 flent->fe_rx_srs_cnt - 1, maxcpus); 1129 if (soft_ring_cnt == 0) { 1130 /* 1131 * Even when soft_ring_cnt is 0, we still need 1132 * to create a soft ring for TCP, UDP and 1133 * OTHER. So set it to 1. 1134 */ 1135 soft_ring_cnt = 1; 1136 } 1137 1138 emrp->mrp_ncpus = 0; 1139 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt && 1140 emrp->mrp_ncpus < MRP_NCPUS; srs_cnt++) { 1141 rx_srs = flent->fe_rx_srs[srs_cnt]; 1142 srs_cpu = &rx_srs->srs_cpu; 1143 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1144 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1145 srs_cpu->mc_ncpus = soft_ring_cnt; 1146 srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1147 mutex_enter(&cpu_lock); 1148 for (j = 0; j < soft_ring_cnt; j++) { 1149 cpuid = mac_next_bind_cpu(cpupart); 1150 srs_cpu->mc_cpus[j] = cpuid; 1151 srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1152 } 1153 cpuid = mac_next_bind_cpu(cpupart); 1154 srs_cpu->mc_rx_pollid = cpuid; 1155 srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 1156 srs_cpu->mc_rx_pollid : -1); 1157 /* increment ncpus to account for polling cpu */ 1158 srs_cpu->mc_ncpus++; 1159 srs_cpu->mc_cpus[j++] = cpuid; 1160 if (!mac_latency_optimize) { 1161 cpuid = mac_next_bind_cpu(cpupart); 1162 srs_cpu->mc_ncpus++; 1163 srs_cpu->mc_cpus[j++] = cpuid; 1164 } 1165 srs_cpu->mc_rx_workerid = cpuid; 1166 mutex_exit(&cpu_lock); 1167 1168 /* 1169 * Copy fanout CPUs to fe_effective_props without duplicates. 1170 */ 1171 for (i = 0; i < srs_cpu->mc_ncpus && 1172 emrp->mrp_ncpus < MRP_NCPUS; i++) { 1173 for (j = 0; j < emrp->mrp_ncpus; j++) { 1174 if (emrp->mrp_cpu[j] == srs_cpu->mc_cpus[i]) 1175 break; 1176 } 1177 if (j == emrp->mrp_ncpus) { 1178 emrp->mrp_cpu[emrp->mrp_ncpus++] = 1179 srs_cpu->mc_cpus[i]; 1180 } 1181 } 1182 } 1183 1184 mac_tx_cpu_init(flent, NULL, cpupart); 1185 } 1186 1187 /* 1188 * DATAPATH SETUP ROUTINES 1189 * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1190 */ 1191 1192 /* 1193 * mac_srs_fanout_list_alloc: 1194 * 1195 * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 1196 * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 1197 * array space is needed to store Tx soft rings. Thus we allocate so 1198 * much array space for srs_tx_soft_rings. 1199 * 1200 * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 1201 * of space to st_soft_rings. This array is used for quick access to 1202 * soft ring associated with a pseudo Tx ring based on the pseudo 1203 * ring's index (mr_index). 1204 */ 1205 static void 1206 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1207 { 1208 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1209 1210 if (mac_srs->srs_type & SRST_TX) { 1211 mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 1212 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1213 MAX_RINGS_PER_GROUP, KM_SLEEP); 1214 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 1215 mac_srs_tx_t *tx = &mac_srs->srs_tx; 1216 1217 tx->st_soft_rings = (mac_soft_ring_t **) 1218 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1219 MAX_RINGS_PER_GROUP, KM_SLEEP); 1220 } 1221 } else { 1222 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1223 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1224 KM_SLEEP); 1225 mac_srs->srs_tcp6_soft_rings = (mac_soft_ring_t **) 1226 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1227 KM_SLEEP); 1228 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1229 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1230 KM_SLEEP); 1231 mac_srs->srs_udp6_soft_rings = (mac_soft_ring_t **) 1232 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1233 KM_SLEEP); 1234 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1235 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1236 KM_SLEEP); 1237 } 1238 } 1239 1240 static void 1241 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1242 { 1243 cpu_t *cp; 1244 boolean_t clear = B_FALSE; 1245 1246 ASSERT(MUTEX_HELD(&cpu_lock)); 1247 1248 if (!mac_srs_thread_bind) 1249 return; 1250 1251 cp = cpu_get(cpuid); 1252 if (cp == NULL || !cpu_is_online(cp)) 1253 return; 1254 1255 mutex_enter(&mac_srs->srs_lock); 1256 mac_srs->srs_state |= SRS_WORKER_BOUND; 1257 if (mac_srs->srs_worker_cpuid != -1) 1258 clear = B_TRUE; 1259 mac_srs->srs_worker_cpuid = cpuid; 1260 mutex_exit(&mac_srs->srs_lock); 1261 1262 if (clear) 1263 thread_affinity_clear(mac_srs->srs_worker); 1264 1265 thread_affinity_set(mac_srs->srs_worker, cpuid); 1266 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1267 } 1268 1269 static void 1270 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1271 { 1272 cpu_t *cp; 1273 boolean_t clear = B_FALSE; 1274 1275 ASSERT(MUTEX_HELD(&cpu_lock)); 1276 1277 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1278 return; 1279 1280 cp = cpu_get(cpuid); 1281 if (cp == NULL || !cpu_is_online(cp)) 1282 return; 1283 1284 mutex_enter(&mac_srs->srs_lock); 1285 mac_srs->srs_state |= SRS_POLL_BOUND; 1286 if (mac_srs->srs_poll_cpuid != -1) 1287 clear = B_TRUE; 1288 mac_srs->srs_poll_cpuid = cpuid; 1289 mutex_exit(&mac_srs->srs_lock); 1290 1291 if (clear) 1292 thread_affinity_clear(mac_srs->srs_poll_thr); 1293 1294 thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1295 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1296 } 1297 1298 /* 1299 * Re-target interrupt to the passed CPU. If re-target is successful, 1300 * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 1301 */ 1302 void 1303 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1304 { 1305 cpu_t *cp; 1306 mac_ring_t *ring = mac_srs->srs_ring; 1307 mac_intr_t *mintr = &ring->mr_info.mri_intr; 1308 flow_entry_t *flent = mac_srs->srs_flent; 1309 boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 1310 1311 ASSERT(MUTEX_HELD(&cpu_lock)); 1312 1313 /* 1314 * Don't re-target the interrupt for these cases: 1315 * 1) ring is NULL 1316 * 2) the interrupt is shared (mi_ddi_shared) 1317 * 3) ddi_handle is NULL and !primary 1318 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 1319 * Case 3 & 4 are because of mac_client_intr_cpu() routine. 1320 * This routine will re-target fixed interrupt for primary 1321 * mac client if the client has only one ring. In that 1322 * case, mc_rx_intr_cpu will already have the correct value. 1323 */ 1324 if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 1325 (mintr->mi_ddi_handle == NULL && !primary) || (primary && 1326 mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 1327 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1328 return; 1329 } 1330 1331 if (mintr->mi_ddi_handle == NULL) 1332 return; 1333 1334 cp = cpu_get(cpuid); 1335 if (cp == NULL || !cpu_is_online(cp)) 1336 return; 1337 1338 /* Drop the cpu_lock as set_intr_affinity() holds it */ 1339 mutex_exit(&cpu_lock); 1340 if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 1341 mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 1342 else 1343 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1344 mutex_enter(&cpu_lock); 1345 } 1346 1347 /* 1348 * Re-target Tx interrupts 1349 */ 1350 void 1351 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 1352 { 1353 cpu_t *cp; 1354 mac_ring_t *ring; 1355 mac_intr_t *mintr; 1356 mac_soft_ring_t *sringp; 1357 mac_srs_tx_t *srs_tx; 1358 mac_cpus_t *srs_cpu; 1359 processorid_t cpuid; 1360 int i; 1361 1362 ASSERT(MUTEX_HELD(&cpu_lock)); 1363 1364 srs_cpu = &mac_srs->srs_cpu; 1365 if (MAC_TX_SOFT_RINGS(mac_srs)) { 1366 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 1367 sringp = mac_srs->srs_tx_soft_rings[i]; 1368 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 1369 cpuid = srs_cpu->mc_tx_intr_cpu[i]; 1370 cp = cpu_get(cpuid); 1371 if (cp == NULL || !cpu_is_online(cp) || 1372 !MAC_RING_RETARGETABLE(ring)) { 1373 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1374 continue; 1375 } 1376 mintr = &ring->mr_info.mri_intr; 1377 /* 1378 * Drop the cpu_lock as set_intr_affinity() 1379 * holds it 1380 */ 1381 mutex_exit(&cpu_lock); 1382 if (set_intr_affinity(mintr->mi_ddi_handle, 1383 cpuid) == DDI_SUCCESS) { 1384 srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 1385 } else { 1386 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1387 } 1388 mutex_enter(&cpu_lock); 1389 } 1390 } else { 1391 cpuid = srs_cpu->mc_tx_intr_cpu[0]; 1392 cp = cpu_get(cpuid); 1393 if (cp == NULL || !cpu_is_online(cp)) { 1394 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1395 return; 1396 } 1397 srs_tx = &mac_srs->srs_tx; 1398 ring = (mac_ring_t *)srs_tx->st_arg2; 1399 if (MAC_RING_RETARGETABLE(ring)) { 1400 mintr = &ring->mr_info.mri_intr; 1401 mutex_exit(&cpu_lock); 1402 if ((set_intr_affinity(mintr->mi_ddi_handle, 1403 cpuid) == DDI_SUCCESS)) { 1404 srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 1405 } else { 1406 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1407 } 1408 mutex_enter(&cpu_lock); 1409 } 1410 } 1411 } 1412 1413 /* 1414 * When a CPU comes back online, bind the MAC kernel threads which 1415 * were previously bound to that CPU, and had to be unbound because 1416 * the CPU was going away. 1417 * 1418 * These functions are called with cpu_lock held and hence we can't 1419 * cv_wait to grab the mac perimeter. Since these functions walk the soft 1420 * ring list of an SRS without being in the perimeter, the list itself 1421 * is protected by the SRS lock. 1422 */ 1423 static void 1424 mac_walk_srs_and_bind(int cpuid) 1425 { 1426 mac_soft_ring_set_t *mac_srs; 1427 mac_soft_ring_t *soft_ring; 1428 1429 rw_enter(&mac_srs_g_lock, RW_READER); 1430 1431 if ((mac_srs = mac_srs_g_list) == NULL) 1432 goto done; 1433 1434 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1435 if (mac_srs->srs_worker_cpuid == -1 && 1436 mac_srs->srs_worker_cpuid_save == cpuid) { 1437 mac_srs->srs_worker_cpuid_save = -1; 1438 mac_srs_worker_bind(mac_srs, cpuid); 1439 } 1440 1441 if (!(mac_srs->srs_type & SRST_TX)) { 1442 if (mac_srs->srs_poll_cpuid == -1 && 1443 mac_srs->srs_poll_cpuid_save == cpuid) { 1444 mac_srs->srs_poll_cpuid_save = -1; 1445 mac_srs_poll_bind(mac_srs, cpuid); 1446 } 1447 } 1448 1449 /* Next tackle the soft rings associated with the srs */ 1450 mutex_enter(&mac_srs->srs_lock); 1451 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1452 soft_ring = soft_ring->s_ring_next) { 1453 if (soft_ring->s_ring_cpuid == -1 && 1454 soft_ring->s_ring_cpuid_save == cpuid) { 1455 soft_ring->s_ring_cpuid_save = -1; 1456 (void) mac_soft_ring_bind(soft_ring, cpuid); 1457 } 1458 } 1459 mutex_exit(&mac_srs->srs_lock); 1460 } 1461 done: 1462 rw_exit(&mac_srs_g_lock); 1463 } 1464 1465 /* 1466 * Change the priority of the SRS's poll and worker thread. Additionally, 1467 * update the priority of the worker threads for the SRS's soft rings. 1468 * Need to modify any associated squeue threads. 1469 */ 1470 void 1471 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1472 { 1473 mac_soft_ring_t *ringp; 1474 1475 mac_srs->srs_pri = prival; 1476 thread_lock(mac_srs->srs_worker); 1477 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1478 thread_unlock(mac_srs->srs_worker); 1479 if (mac_srs->srs_poll_thr != NULL) { 1480 thread_lock(mac_srs->srs_poll_thr); 1481 (void) thread_change_pri(mac_srs->srs_poll_thr, 1482 mac_srs->srs_pri, 0); 1483 thread_unlock(mac_srs->srs_poll_thr); 1484 } 1485 if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1486 return; 1487 while (ringp != mac_srs->srs_soft_ring_tail) { 1488 thread_lock(ringp->s_ring_worker); 1489 (void) thread_change_pri(ringp->s_ring_worker, 1490 mac_srs->srs_pri, 0); 1491 thread_unlock(ringp->s_ring_worker); 1492 ringp = ringp->s_ring_next; 1493 } 1494 ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1495 thread_lock(ringp->s_ring_worker); 1496 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1497 thread_unlock(ringp->s_ring_worker); 1498 } 1499 1500 /* 1501 * Change the receive bandwidth limit. 1502 */ 1503 static void 1504 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1505 { 1506 mac_soft_ring_t *softring; 1507 1508 mutex_enter(&srs->srs_lock); 1509 mutex_enter(&srs->srs_bw->mac_bw_lock); 1510 1511 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1512 /* Reset bandwidth limit */ 1513 if (srs->srs_type & SRST_BW_CONTROL) { 1514 softring = srs->srs_soft_ring_head; 1515 while (softring != NULL) { 1516 softring->s_ring_type &= ~ST_RING_BW_CTL; 1517 softring = softring->s_ring_next; 1518 } 1519 srs->srs_type &= ~SRST_BW_CONTROL; 1520 srs->srs_drain_func = mac_rx_srs_drain; 1521 } 1522 } else { 1523 /* Set/Modify bandwidth limit */ 1524 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1525 /* 1526 * Give twice the queuing capability before 1527 * dropping packets. The unit is bytes/tick. 1528 */ 1529 srs->srs_bw->mac_bw_drop_threshold = 1530 srs->srs_bw->mac_bw_limit << 1; 1531 if (!(srs->srs_type & SRST_BW_CONTROL)) { 1532 softring = srs->srs_soft_ring_head; 1533 while (softring != NULL) { 1534 softring->s_ring_type |= ST_RING_BW_CTL; 1535 softring = softring->s_ring_next; 1536 } 1537 srs->srs_type |= SRST_BW_CONTROL; 1538 srs->srs_drain_func = mac_rx_srs_drain_bw; 1539 } 1540 } 1541 done: 1542 mutex_exit(&srs->srs_bw->mac_bw_lock); 1543 mutex_exit(&srs->srs_lock); 1544 } 1545 1546 /* Change the transmit bandwidth limit */ 1547 static void 1548 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1549 { 1550 uint32_t tx_mode, ring_info = 0; 1551 mac_srs_tx_t *srs_tx = &srs->srs_tx; 1552 mac_client_impl_t *mcip = srs->srs_mcip; 1553 1554 /* 1555 * We need to quiesce/restart the client here because mac_tx() and 1556 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1557 * st_mode and related fields, which are modified by the code below. 1558 */ 1559 mac_tx_client_quiesce((mac_client_handle_t)mcip); 1560 1561 mutex_enter(&srs->srs_lock); 1562 mutex_enter(&srs->srs_bw->mac_bw_lock); 1563 1564 tx_mode = srs_tx->st_mode; 1565 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1566 /* Reset bandwidth limit */ 1567 if (tx_mode == SRS_TX_BW) { 1568 if (srs_tx->st_arg2 != NULL) 1569 ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1570 if (mac_tx_serialize || 1571 (ring_info & MAC_RING_TX_SERIALIZE)) { 1572 srs_tx->st_mode = SRS_TX_SERIALIZE; 1573 } else { 1574 srs_tx->st_mode = SRS_TX_DEFAULT; 1575 } 1576 } else if (tx_mode == SRS_TX_BW_FANOUT) { 1577 srs_tx->st_mode = SRS_TX_FANOUT; 1578 } else if (tx_mode == SRS_TX_BW_AGGR) { 1579 srs_tx->st_mode = SRS_TX_AGGR; 1580 } 1581 srs->srs_type &= ~SRST_BW_CONTROL; 1582 } else { 1583 /* Set/Modify bandwidth limit */ 1584 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1585 /* 1586 * Give twice the queuing capability before 1587 * dropping packets. The unit is bytes/tick. 1588 */ 1589 srs->srs_bw->mac_bw_drop_threshold = 1590 srs->srs_bw->mac_bw_limit << 1; 1591 srs->srs_type |= SRST_BW_CONTROL; 1592 if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 1593 tx_mode != SRS_TX_BW_AGGR) { 1594 if (tx_mode == SRS_TX_SERIALIZE || 1595 tx_mode == SRS_TX_DEFAULT) { 1596 srs_tx->st_mode = SRS_TX_BW; 1597 } else if (tx_mode == SRS_TX_FANOUT) { 1598 srs_tx->st_mode = SRS_TX_BW_FANOUT; 1599 } else if (tx_mode == SRS_TX_AGGR) { 1600 srs_tx->st_mode = SRS_TX_BW_AGGR; 1601 } else { 1602 ASSERT(0); 1603 } 1604 } 1605 } 1606 done: 1607 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1608 mutex_exit(&srs->srs_bw->mac_bw_lock); 1609 mutex_exit(&srs->srs_lock); 1610 1611 mac_tx_client_restart((mac_client_handle_t)mcip); 1612 } 1613 1614 /* 1615 * The uber function that deals with any update to bandwidth limits. 1616 */ 1617 void 1618 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1619 { 1620 int count; 1621 1622 for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1623 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1624 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1625 } 1626 1627 /* 1628 * When the first sub-flow is added to a link, we disable polling on the 1629 * link and also modify the entry point to mac_rx_srs_subflow_process(). 1630 * (polling is disabled because with the subflow added, accounting 1631 * for polling needs additional logic, it is assumed that when a subflow is 1632 * added, we can take some hit as a result of disabling polling rather than 1633 * adding more complexity - if this becomes a perf. issue we need to 1634 * re-rvaluate this logic). When the last subflow is removed, we turn back 1635 * polling and also reset the entry point to mac_rx_srs_process(). 1636 * 1637 * In the future if there are multiple SRS, we can simply 1638 * take one and give it to the flow rather than disabling polling and 1639 * resetting the entry point. 1640 */ 1641 void 1642 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1643 { 1644 flow_entry_t *flent = mcip->mci_flent; 1645 int i; 1646 mac_impl_t *mip = mcip->mci_mip; 1647 mac_rx_func_t rx_func; 1648 uint_t rx_srs_cnt; 1649 boolean_t enable_classifier; 1650 1651 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1652 1653 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1654 1655 rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1656 mac_rx_srs_process; 1657 1658 /* Tell mac_srs_poll_state_change to disable polling if necessary */ 1659 if (mip->mi_state_flags & MIS_POLL_DISABLE) 1660 enable_classifier = B_TRUE; 1661 1662 /* 1663 * If receive function has already been configured correctly for 1664 * current subflow configuration, do nothing. 1665 */ 1666 if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1667 return; 1668 1669 rx_srs_cnt = flent->fe_rx_srs_cnt; 1670 for (i = 0; i < rx_srs_cnt; i++) { 1671 ASSERT(flent->fe_rx_srs[i] != NULL); 1672 mac_srs_poll_state_change(flent->fe_rx_srs[i], 1673 enable_classifier, rx_func); 1674 } 1675 1676 /* 1677 * Change the S/W classifier so that we can land in the 1678 * correct processing function with correct argument. 1679 * If all subflows have been removed we can revert to 1680 * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process(). 1681 */ 1682 mutex_enter(&flent->fe_lock); 1683 flent->fe_cb_fn = (flow_fn_t)rx_func; 1684 flent->fe_cb_arg1 = (void *)mip; 1685 flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1686 mutex_exit(&flent->fe_lock); 1687 } 1688 1689 static void 1690 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1691 { 1692 int tcp_count = 0, tcp6_count = 0, udp_count = 0, udp6_count = 0, 1693 oth_count = 0, tx_count = 0; 1694 1695 mac_soft_ring_t *softring; 1696 1697 softring = mac_srs->srs_soft_ring_head; 1698 if (softring == NULL) { 1699 ASSERT(mac_srs->srs_soft_ring_count == 0); 1700 mac_srs->srs_tcp_ring_count = 0; 1701 mac_srs->srs_udp_ring_count = 0; 1702 mac_srs->srs_tcp6_ring_count = 0; 1703 mac_srs->srs_udp6_ring_count = 0; 1704 mac_srs->srs_oth_ring_count = 0; 1705 mac_srs->srs_tx_ring_count = 0; 1706 return; 1707 } 1708 1709 while (softring != NULL) { 1710 if (softring->s_ring_type & ST_RING_TCP) { 1711 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1712 } else if (softring->s_ring_type & ST_RING_TCP6) { 1713 mac_srs->srs_tcp6_soft_rings[tcp6_count++] = softring; 1714 } else if (softring->s_ring_type & ST_RING_UDP) { 1715 mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1716 } else if (softring->s_ring_type & ST_RING_UDP6) { 1717 mac_srs->srs_udp6_soft_rings[udp6_count++] = softring; 1718 } else if (softring->s_ring_type & ST_RING_OTH) { 1719 mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1720 } else { 1721 ASSERT(softring->s_ring_type & ST_RING_TX); 1722 mac_srs->srs_tx_soft_rings[tx_count++] = softring; 1723 } 1724 softring = softring->s_ring_next; 1725 } 1726 1727 ASSERT(mac_srs->srs_soft_ring_count == (tcp_count + tcp6_count + 1728 udp_count + udp6_count + oth_count + tx_count)); 1729 mac_srs->srs_tcp_ring_count = tcp_count; 1730 mac_srs->srs_tcp6_ring_count = tcp6_count; 1731 mac_srs->srs_udp_ring_count = udp_count; 1732 mac_srs->srs_udp6_ring_count = udp6_count; 1733 mac_srs->srs_oth_ring_count = oth_count; 1734 mac_srs->srs_tx_ring_count = tx_count; 1735 } 1736 1737 void 1738 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 1739 mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1740 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1741 mac_resource_handle_t x_arg2, boolean_t set_bypass) 1742 { 1743 mac_soft_ring_t *softring; 1744 mac_rx_fifo_t mrf; 1745 1746 bzero(&mrf, sizeof (mac_rx_fifo_t)); 1747 mrf.mrf_type = MAC_RX_FIFO; 1748 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1749 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 1750 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1751 mrf.mrf_flow_priority = pri; 1752 1753 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1754 (type|ST_RING_TCP), pri, mcip, mac_srs, 1755 cpuid, rx_func, x_arg1, x_arg2); 1756 softring->s_ring_rx_arg2 = NULL; 1757 1758 /* 1759 * TCP and UDP support DLS bypass. In addition TCP 1760 * squeue can also poll their corresponding soft rings. 1761 */ 1762 if (set_bypass && mcip->mci_direct_rx.mdrx_v4 != NULL && 1763 (mcip->mci_resource_arg != NULL)) { 1764 mac_soft_ring_dls_bypass(softring, 1765 mcip->mci_direct_rx.mdrx_v4, 1766 mcip->mci_direct_rx.mdrx_arg_v4); 1767 1768 mrf.mrf_rx_arg = softring; 1769 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1770 1771 /* 1772 * Make a call in IP to get a TCP squeue assigned to 1773 * this softring to maintain full CPU locality through 1774 * the stack and allow the squeue to be able to poll 1775 * the softring so the flow control can be pushed 1776 * all the way to H/W. 1777 */ 1778 softring->s_ring_rx_arg2 = 1779 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1780 (mac_resource_t *)&mrf); 1781 } 1782 1783 /* 1784 * Non-TCP protocols don't support squeues. Hence we 1785 * don't make any ring addition callbacks for non-TCP 1786 * rings. Now create the UDP softring and allow it to 1787 * bypass the DLS layer. 1788 */ 1789 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1790 (type|ST_RING_UDP), pri, mcip, mac_srs, 1791 cpuid, rx_func, x_arg1, x_arg2); 1792 softring->s_ring_rx_arg2 = NULL; 1793 1794 if (set_bypass && mcip->mci_direct_rx.mdrx_v4 != NULL && 1795 (mcip->mci_resource_arg != NULL)) { 1796 mac_soft_ring_dls_bypass(softring, 1797 mcip->mci_direct_rx.mdrx_v4, 1798 mcip->mci_direct_rx.mdrx_arg_v4); 1799 } 1800 1801 /* TCP for IPv6. */ 1802 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1803 (type|ST_RING_TCP6), pri, mcip, mac_srs, 1804 cpuid, rx_func, x_arg1, x_arg2); 1805 softring->s_ring_rx_arg2 = NULL; 1806 1807 if (set_bypass && mcip->mci_direct_rx.mdrx_v6 != NULL && 1808 (mcip->mci_resource_arg != NULL)) { 1809 mac_soft_ring_dls_bypass(softring, 1810 mcip->mci_direct_rx.mdrx_v6, 1811 mcip->mci_direct_rx.mdrx_arg_v6); 1812 1813 mrf.mrf_rx_arg = softring; 1814 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1815 1816 softring->s_ring_rx_arg2 = 1817 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1818 (mac_resource_t *)&mrf); 1819 } 1820 1821 /* UDP for IPv6. */ 1822 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1823 (type|ST_RING_UDP6), pri, mcip, mac_srs, 1824 cpuid, rx_func, x_arg1, x_arg2); 1825 softring->s_ring_rx_arg2 = NULL; 1826 1827 if (set_bypass && mcip->mci_direct_rx.mdrx_v6 != NULL && 1828 (mcip->mci_resource_arg != NULL)) { 1829 mac_soft_ring_dls_bypass(softring, 1830 mcip->mci_direct_rx.mdrx_v6, 1831 mcip->mci_direct_rx.mdrx_arg_v6); 1832 } 1833 1834 /* Create the Oth softrings which has to go through the DLS. */ 1835 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1836 (type|ST_RING_OTH), pri, mcip, mac_srs, 1837 cpuid, rx_func, x_arg1, x_arg2); 1838 softring->s_ring_rx_arg2 = NULL; 1839 } 1840 1841 /* 1842 * This routine associates a CPU or a set of CPU to process incoming 1843 * traffic from a mac client. If multiple CPUs are specified, then 1844 * so many soft rings are created with each soft ring worker thread 1845 * bound to a CPU in the set. Each soft ring in turn will be 1846 * associated with an squeue and the squeue will be moved to the 1847 * same CPU as that of the soft ring's. 1848 */ 1849 static void 1850 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 1851 void *x_arg1, mac_resource_handle_t x_arg2, 1852 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1853 { 1854 mac_soft_ring_t *softring; 1855 uint32_t soft_ring_flag = 0; 1856 processorid_t cpuid = -1; 1857 int i, srings_present, new_fanout_cnt; 1858 mac_cpus_t *srs_cpu; 1859 1860 /* fanout state is REINIT. Set it back to INIT */ 1861 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1862 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1863 1864 /* how many are present right now */ 1865 srings_present = mac_rx_srs->srs_tcp_ring_count; 1866 /* new request */ 1867 srs_cpu = &mac_rx_srs->srs_cpu; 1868 new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1869 1870 mutex_enter(&mac_rx_srs->srs_lock); 1871 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1872 soft_ring_flag |= ST_RING_BW_CTL; 1873 mutex_exit(&mac_rx_srs->srs_lock); 1874 1875 if (new_fanout_cnt > srings_present) { 1876 /* soft rings increased */ 1877 mutex_enter(&mac_rx_srs->srs_lock); 1878 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1879 mutex_exit(&mac_rx_srs->srs_lock); 1880 1881 for (i = mac_rx_srs->srs_tcp_ring_count; 1882 i < new_fanout_cnt; i++) { 1883 /* 1884 * Create the protocol softrings and set the 1885 * DLS bypass where possible. 1886 */ 1887 mac_srs_create_proto_softrings(i, soft_ring_flag, 1888 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1889 rx_func, x_arg1, x_arg2, B_TRUE); 1890 } 1891 mac_srs_update_fanout_list(mac_rx_srs); 1892 } else if (new_fanout_cnt < srings_present) { 1893 /* soft rings decreased */ 1894 if (new_fanout_cnt == 1) { 1895 mutex_enter(&mac_rx_srs->srs_lock); 1896 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1897 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1898 mutex_exit(&mac_rx_srs->srs_lock); 1899 } 1900 /* Get rid of extra soft rings */ 1901 for (i = new_fanout_cnt; 1902 i < mac_rx_srs->srs_tcp_ring_count; i++) { 1903 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1904 if (softring->s_ring_rx_arg2 != NULL) { 1905 mcip->mci_resource_remove( 1906 (void *)mcip->mci_resource_arg, 1907 softring->s_ring_rx_arg2); 1908 } 1909 softring = mac_rx_srs->srs_tcp6_soft_rings[i]; 1910 if (softring->s_ring_rx_arg2 != NULL) { 1911 mcip->mci_resource_remove( 1912 (void *)mcip->mci_resource_arg, 1913 softring->s_ring_rx_arg2); 1914 } 1915 mac_soft_ring_remove(mac_rx_srs, 1916 mac_rx_srs->srs_tcp_soft_rings[i]); 1917 mac_soft_ring_remove(mac_rx_srs, 1918 mac_rx_srs->srs_tcp6_soft_rings[i]); 1919 mac_soft_ring_remove(mac_rx_srs, 1920 mac_rx_srs->srs_udp_soft_rings[i]); 1921 mac_soft_ring_remove(mac_rx_srs, 1922 mac_rx_srs->srs_udp6_soft_rings[i]); 1923 mac_soft_ring_remove(mac_rx_srs, 1924 mac_rx_srs->srs_oth_soft_rings[i]); 1925 } 1926 mac_srs_update_fanout_list(mac_rx_srs); 1927 } 1928 1929 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1930 mutex_enter(&cpu_lock); 1931 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1932 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1933 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1934 cpuid); 1935 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp6_soft_rings[i], 1936 cpuid); 1937 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1938 cpuid); 1939 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1940 cpuid); 1941 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp6_soft_rings[i], 1942 cpuid); 1943 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1944 if (softring->s_ring_rx_arg2 != NULL) { 1945 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1946 softring->s_ring_rx_arg2, cpuid); 1947 } 1948 softring = mac_rx_srs->srs_tcp6_soft_rings[i]; 1949 if (softring->s_ring_rx_arg2 != NULL) { 1950 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1951 softring->s_ring_rx_arg2, cpuid); 1952 } 1953 } 1954 1955 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1956 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1957 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1958 /* 1959 * Bind Tx srs and soft ring threads too. Let's bind tx 1960 * srs to the last cpu in mrp list. 1961 */ 1962 if (mac_tx_srs != NULL) { 1963 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1964 mac_tx_srs_retarget_intr(mac_tx_srs); 1965 } 1966 mutex_exit(&cpu_lock); 1967 } 1968 1969 /* 1970 * Bind SRS threads and soft rings to CPUs/create fanout list. 1971 */ 1972 void 1973 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 1974 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1975 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 1976 cpupart_t *cpupart) 1977 { 1978 int i; 1979 processorid_t cpuid; 1980 uint32_t soft_ring_flag = 0; 1981 int soft_ring_cnt; 1982 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1983 1984 /* 1985 * Remove the no soft ring flag and we will adjust it 1986 * appropriately further down. 1987 */ 1988 mutex_enter(&mac_rx_srs->srs_lock); 1989 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1990 mutex_exit(&mac_rx_srs->srs_lock); 1991 1992 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1993 1994 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1995 soft_ring_flag |= ST_RING_BW_CTL; 1996 1997 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1998 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1999 /* 2000 * Ring count can be 0 if no fanout is required and no cpu 2001 * were specified. Leave the SRS worker and poll thread 2002 * unbound 2003 */ 2004 ASSERT(mrp != NULL); 2005 soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 2006 2007 /* Step 1: bind cpu contains cpu list where threads need to bind */ 2008 if (soft_ring_cnt > 0) { 2009 mutex_enter(&cpu_lock); 2010 for (i = 0; i < soft_ring_cnt; i++) { 2011 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 2012 /* Create the protocol softrings */ 2013 mac_srs_create_proto_softrings(i, soft_ring_flag, 2014 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 2015 rx_func, x_arg1, x_arg2, B_FALSE); 2016 } 2017 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 2018 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 2019 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 2020 /* 2021 * Bind Tx srs and soft ring threads too. 2022 * Let's bind tx srs to the last cpu in 2023 * mrp list. 2024 */ 2025 if (mac_tx_srs == NULL) { 2026 mutex_exit(&cpu_lock); 2027 goto alldone; 2028 } 2029 2030 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 2031 mac_tx_srs_retarget_intr(mac_tx_srs); 2032 mutex_exit(&cpu_lock); 2033 } else { 2034 mutex_enter(&cpu_lock); 2035 /* 2036 * For a subflow, mrp_workerid and mrp_pollid 2037 * is not set. 2038 */ 2039 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 2040 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 2041 mutex_exit(&cpu_lock); 2042 goto no_softrings; 2043 } 2044 2045 alldone: 2046 if (soft_ring_cnt > 1) 2047 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 2048 mac_srs_update_fanout_list(mac_rx_srs); 2049 mac_srs_client_poll_enable(mcip, mac_rx_srs); 2050 return; 2051 2052 no_softrings: 2053 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 2054 mutex_enter(&cpu_lock); 2055 cpuid = mac_next_bind_cpu(cpupart); 2056 /* Create the protocol softrings */ 2057 mac_srs_create_proto_softrings(0, soft_ring_flag, 2058 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 2059 rx_func, x_arg1, x_arg2, B_FALSE); 2060 mutex_exit(&cpu_lock); 2061 } else { 2062 /* 2063 * This is the case when there is no fanout which is 2064 * true for subflows. 2065 */ 2066 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 2067 } 2068 mac_srs_update_fanout_list(mac_rx_srs); 2069 mac_srs_client_poll_enable(mcip, mac_rx_srs); 2070 } 2071 2072 /* 2073 * Calls mac_srs_fanout_init() or modify() depending upon whether 2074 * the SRS is getting initialized or re-initialized. 2075 */ 2076 void 2077 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2078 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 2079 mac_resource_handle_t x_arg2, cpupart_t *cpupart) 2080 { 2081 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 2082 int i, rx_srs_cnt; 2083 2084 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 2085 2086 /* 2087 * Aggr ports do not have SRSes. This function should never be 2088 * called on an aggr port. 2089 */ 2090 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2091 mac_rx_srs = flent->fe_rx_srs[0]; 2092 2093 /* 2094 * Set up the fanout on the tx side only once, with the 2095 * first rx SRS. The CPU binding, fanout, and bandwidth 2096 * criteria are common to both RX and TX, so 2097 * initializing them along side avoids redundant code. 2098 */ 2099 mac_tx_srs = flent->fe_tx_srs; 2100 rx_srs_cnt = flent->fe_rx_srs_cnt; 2101 2102 /* No fanout for subflows */ 2103 if (flent->fe_type & FLOW_USER) { 2104 mac_srs_fanout_init(mcip, mrp, rx_func, 2105 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2106 cpupart); 2107 return; 2108 } 2109 2110 if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 2111 mac_flow_user_cpu_init(flent, mrp); 2112 else 2113 mac_flow_cpu_init(flent, cpupart); 2114 2115 mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2116 2117 /* 2118 * Set up fanout for both SW (0th SRS) and HW classified 2119 * SRS (the rest of Rx SRSs in flent). 2120 */ 2121 for (i = 0; i < rx_srs_cnt; i++) { 2122 mac_rx_srs = flent->fe_rx_srs[i]; 2123 if (i != 0) 2124 mac_tx_srs = NULL; 2125 switch (mac_rx_srs->srs_fanout_state) { 2126 case SRS_FANOUT_UNINIT: 2127 mac_srs_fanout_init(mcip, mrp, rx_func, 2128 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2129 cpupart); 2130 break; 2131 case SRS_FANOUT_INIT: 2132 break; 2133 case SRS_FANOUT_REINIT: 2134 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 2135 mac_srs_fanout_modify(mcip, rx_func, x_arg1, 2136 x_arg2, mac_rx_srs, mac_tx_srs); 2137 mac_rx_srs_restart(mac_rx_srs); 2138 break; 2139 default: 2140 VERIFY(mac_rx_srs->srs_fanout_state <= 2141 SRS_FANOUT_REINIT); 2142 break; 2143 } 2144 } 2145 } 2146 2147 /* 2148 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2149 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2150 * processing is created. 2151 * 2152 * Details on Rx SRS: 2153 * Create a SRS and also add the necessary soft rings for TCP and 2154 * non-TCP based on fanout type and count specified. 2155 * 2156 * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2157 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2158 * to be heavily modified. 2159 * 2160 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2161 */ 2162 mac_soft_ring_set_t * 2163 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2164 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2165 mac_ring_t *ring) 2166 { 2167 mac_soft_ring_set_t *mac_srs; 2168 mac_srs_rx_t *srs_rx; 2169 mac_srs_tx_t *srs_tx; 2170 mac_bw_ctl_t *mac_bw; 2171 mac_resource_props_t *mrp; 2172 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2173 2174 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2175 bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2176 srs_rx = &mac_srs->srs_rx; 2177 srs_tx = &mac_srs->srs_tx; 2178 2179 mutex_enter(&flent->fe_lock); 2180 2181 /* 2182 * Get the bandwidth control structure from the flent. Get 2183 * rid of any residual values in the control structure for 2184 * the tx bw struct and also for the rx, if the rx srs is 2185 * the 1st one being brought up (the rx bw ctl struct may 2186 * be shared by multiple SRSs) 2187 */ 2188 if (is_tx_srs) { 2189 mac_srs->srs_bw = &flent->fe_tx_bw; 2190 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2191 flent->fe_tx_srs = mac_srs; 2192 } else { 2193 /* 2194 * The bw counter (stored in the flent) is shared 2195 * by SRS's within an rx group. 2196 */ 2197 mac_srs->srs_bw = &flent->fe_rx_bw; 2198 /* First rx SRS, clear the bw structure */ 2199 if (flent->fe_rx_srs_cnt == 0) 2200 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2201 2202 /* 2203 * It is better to panic here rather than just assert because 2204 * on a non-debug kernel we might end up courrupting memory 2205 * and making it difficult to debug. 2206 */ 2207 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 2208 panic("Array Overrun detected due to MAC client %p " 2209 " having more rings than %d", (void *)mcip, 2210 MAX_RINGS_PER_GROUP); 2211 } 2212 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2213 flent->fe_rx_srs_cnt++; 2214 } 2215 mac_srs->srs_flent = flent; 2216 mutex_exit(&flent->fe_lock); 2217 2218 mac_srs->srs_state = 0; 2219 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2220 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2221 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 2222 mac_srs->srs_mcip = mcip; 2223 mac_srs_fanout_list_alloc(mac_srs); 2224 2225 /* 2226 * For a flow we use the underlying MAC client's priority range with 2227 * the priority value to find an absolute priority value. For a MAC 2228 * client we use the MAC client's maximum priority as the value. 2229 */ 2230 mrp = &flent->fe_effective_props; 2231 if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2232 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2233 mcip->mci_max_pri, mrp->mrp_priority); 2234 } else { 2235 mac_srs->srs_pri = mcip->mci_max_pri; 2236 } 2237 /* 2238 * We need to insert the SRS in the global list before 2239 * binding the SRS and SR threads. Otherwise there is a 2240 * is a small window where the cpu reconfig callbacks 2241 * may miss the SRS in the list walk and DR could fail 2242 * as there are bound threads. 2243 */ 2244 mac_srs_add_glist(mac_srs); 2245 2246 /* Initialize bw limit */ 2247 if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2248 mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2249 2250 mac_bw = mac_srs->srs_bw; 2251 mutex_enter(&mac_bw->mac_bw_lock); 2252 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2253 2254 /* 2255 * Give twice the queuing capability before 2256 * dropping packets. The unit is bytes/tick. 2257 */ 2258 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2259 mutex_exit(&mac_bw->mac_bw_lock); 2260 mac_srs->srs_type |= SRST_BW_CONTROL; 2261 } else { 2262 mac_srs->srs_drain_func = mac_rx_srs_drain; 2263 } 2264 2265 /* 2266 * We use the following policy to control Receive 2267 * Side Dynamic Polling: 2268 * 1) We switch to poll mode anytime the processing thread causes 2269 * a backlog to build up in SRS and its associated Soft Rings 2270 * (sr_poll_pkt_cnt > 0). 2271 * 2) As long as the backlog stays under the low water mark 2272 * (sr_lowat), we poll the H/W for more packets. 2273 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2274 * stay in poll mode but don't poll the H/W for more packets. 2275 * 4) Anytime in polling mode, if we poll the H/W for packets and 2276 * find nothing plus we have an existing backlog 2277 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2278 * the H/W for packets anymore (let the polling thread go to sleep). 2279 * 5) Once the backlog is relieved (packets are processed) we reenable 2280 * polling (by signalling the poll thread) only when the backlog 2281 * dips below sr_poll_thres. 2282 * 6) sr_hiwat is used exclusively when we are not polling capable 2283 * and is used to decide when to drop packets so the SRS queue 2284 * length doesn't grow infinitely. 2285 */ 2286 if (!is_tx_srs) { 2287 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2288 /* Low water mark needs to be less than high water mark */ 2289 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2290 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2291 (mac_soft_ring_max_q_cnt >> 2); 2292 /* Poll threshold need to be half of low water mark or less */ 2293 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2294 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2295 (srs_rx->sr_lowat >> 1); 2296 if (mac_latency_optimize) 2297 mac_srs->srs_state |= SRS_LATENCY_OPT; 2298 else 2299 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2300 } 2301 2302 /* 2303 * Create the srs_worker with twice the stack of a normal kernel thread 2304 * to reduce the likelihood of stack overflows in receive-side 2305 * processing. (The larger stacks are not the only precaution taken 2306 * against stack overflows; see the use of mac_rx_srs_stack_needed 2307 * in mac_sched.c). 2308 */ 2309 mac_srs->srs_worker = thread_create(NULL, default_stksize << 1, 2310 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2311 2312 if (is_tx_srs) { 2313 /* Handle everything about Tx SRS and return */ 2314 mac_srs->srs_drain_func = mac_tx_srs_drain; 2315 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2316 srs_tx->st_hiwat = 2317 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2318 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2319 srs_tx->st_arg1 = x_arg1; 2320 srs_tx->st_arg2 = x_arg2; 2321 goto done; 2322 } 2323 2324 if ((srs_type & SRST_FLOW) != 0 || 2325 FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2326 srs_rx->sr_lower_proc = mac_rx_srs_process; 2327 else 2328 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2329 2330 srs_rx->sr_func = rx_func; 2331 srs_rx->sr_arg1 = x_arg1; 2332 srs_rx->sr_arg2 = x_arg2; 2333 2334 if (ring != NULL) { 2335 uint_t ring_info; 2336 2337 /* Is the mac_srs created over the RX default group? */ 2338 if (ring->mr_gh == (mac_group_handle_t) 2339 MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2340 mac_srs->srs_type |= SRST_DEFAULT_GRP; 2341 } 2342 mac_srs->srs_ring = ring; 2343 ring->mr_srs = mac_srs; 2344 ring->mr_classify_type = MAC_HW_CLASSIFIER; 2345 ring->mr_flag |= MR_INCIPIENT; 2346 2347 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 2348 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2349 mac_srs->srs_state |= SRS_POLLING_CAPAB; 2350 2351 mac_srs->srs_poll_thr = thread_create(NULL, 0, 2352 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2353 mac_srs->srs_pri); 2354 /* 2355 * Some drivers require serialization and don't send 2356 * packet chains in interrupt context. For such 2357 * drivers, we should always queue in the soft ring 2358 * so that we get a chance to switch into polling 2359 * mode under backlog. 2360 */ 2361 ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 2362 if (ring_info & MAC_RING_RX_ENQUEUE) 2363 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2364 } 2365 done: 2366 mac_srs_stat_create(mac_srs); 2367 return (mac_srs); 2368 } 2369 2370 /* 2371 * Figure out the number of soft rings required. Its dependant on 2372 * if protocol fanout is required (for LINKs), global settings 2373 * require us to do fanout for performance (based on mac_soft_ring_enable), 2374 * or user has specifically requested fanout. 2375 */ 2376 static uint32_t 2377 mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2378 { 2379 uint32_t fanout_type; 2380 mac_resource_props_t *mrp = &flent->fe_effective_props; 2381 2382 /* no fanout for subflows */ 2383 switch (link_type) { 2384 case SRST_FLOW: 2385 fanout_type = SRST_NO_SOFT_RINGS; 2386 break; 2387 case SRST_LINK: 2388 fanout_type = SRST_FANOUT_PROTO; 2389 break; 2390 } 2391 2392 /* A primary NIC/link is being plumbed */ 2393 if (flent->fe_type & FLOW_PRIMARY_MAC) { 2394 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2395 fanout_type |= SRST_FANOUT_SRC_IP; 2396 } 2397 } else if (flent->fe_type & FLOW_VNIC) { 2398 /* A VNIC is being created */ 2399 if (mrp != NULL && mrp->mrp_ncpus > 0) { 2400 fanout_type |= SRST_FANOUT_SRC_IP; 2401 } 2402 } 2403 2404 return (fanout_type); 2405 } 2406 2407 /* 2408 * Change a group from h/w to s/w classification. 2409 */ 2410 void 2411 mac_rx_switch_grp_to_sw(mac_group_t *group) 2412 { 2413 mac_ring_t *ring; 2414 mac_soft_ring_set_t *mac_srs; 2415 2416 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2417 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2418 /* 2419 * Remove the SRS associated with the HW ring. 2420 * As a result, polling will be disabled. 2421 */ 2422 mac_srs = ring->mr_srs; 2423 ASSERT(mac_srs != NULL); 2424 mac_rx_srs_remove(mac_srs); 2425 ring->mr_srs = NULL; 2426 } 2427 2428 if (ring->mr_state != MR_INUSE) 2429 (void) mac_start_ring(ring); 2430 2431 /* 2432 * We need to perform SW classification 2433 * for packets landing in these rings 2434 */ 2435 ring->mr_flag = 0; 2436 ring->mr_classify_type = MAC_SW_CLASSIFIER; 2437 } 2438 } 2439 2440 /* 2441 * Create the Rx SRS for S/W classifier and for each ring in the 2442 * group (if exclusive group). Also create the Tx SRS. 2443 */ 2444 void 2445 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2446 uint32_t link_type) 2447 { 2448 cpupart_t *cpupart; 2449 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2450 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2451 boolean_t use_default = B_FALSE; 2452 2453 mac_rx_srs_group_setup(mcip, flent, link_type); 2454 mac_tx_srs_group_setup(mcip, flent, link_type); 2455 2456 /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */ 2457 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 2458 return; 2459 2460 pool_lock(); 2461 cpupart = mac_pset_find(mrp, &use_default); 2462 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2463 mac_rx_deliver, mcip, NULL, cpupart); 2464 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 2465 pool_unlock(); 2466 } 2467 2468 /* 2469 * Set up the Rx SRSes. If there is no group associated with the 2470 * client, then only setup SW classification. If the client has 2471 * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an 2472 * SRS for each HW ring. If the client is sharing a group, then make 2473 * sure to teardown the HW SRSes. 2474 */ 2475 void 2476 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2477 uint32_t link_type) 2478 { 2479 mac_impl_t *mip = mcip->mci_mip; 2480 mac_soft_ring_set_t *mac_srs; 2481 mac_ring_t *ring; 2482 uint32_t fanout_type; 2483 mac_group_t *rx_group = flent->fe_rx_ring_group; 2484 boolean_t no_unicast; 2485 2486 /* 2487 * If this is an an aggr port, then don't setup Rx SRS and Rx 2488 * soft rings as they won't be used. However, we still need to 2489 * start the rings to receive data on them. 2490 */ 2491 if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) { 2492 if (rx_group == NULL) 2493 return; 2494 2495 for (ring = rx_group->mrg_rings; ring != NULL; 2496 ring = ring->mr_next) { 2497 if (ring->mr_state != MR_INUSE) 2498 (void) mac_start_ring(ring); 2499 } 2500 2501 return; 2502 } 2503 2504 /* 2505 * Aggr ports should never have SRSes. 2506 */ 2507 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2508 2509 fanout_type = mac_find_fanout(flent, link_type); 2510 no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0; 2511 2512 /* Create the SRS for SW classification if none exists */ 2513 if (flent->fe_rx_srs[0] == NULL) { 2514 ASSERT(flent->fe_rx_srs_cnt == 0); 2515 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2516 mac_rx_deliver, mcip, NULL, NULL); 2517 mutex_enter(&flent->fe_lock); 2518 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2519 flent->fe_cb_arg1 = (void *)mip; 2520 flent->fe_cb_arg2 = (void *)mac_srs; 2521 mutex_exit(&flent->fe_lock); 2522 } 2523 2524 if (rx_group == NULL) 2525 return; 2526 2527 /* 2528 * If the group is marked RESERVED then setup an SRS and 2529 * fanout for each HW ring. 2530 */ 2531 switch (rx_group->mrg_state) { 2532 case MAC_GROUP_STATE_RESERVED: 2533 for (ring = rx_group->mrg_rings; ring != NULL; 2534 ring = ring->mr_next) { 2535 uint16_t vid = i_mac_flow_vid(mcip->mci_flent); 2536 2537 switch (ring->mr_state) { 2538 case MR_INUSE: 2539 case MR_FREE: 2540 if (ring->mr_srs != NULL) 2541 break; 2542 if (ring->mr_state != MR_INUSE) 2543 (void) mac_start_ring(ring); 2544 2545 /* 2546 * If a client requires SW VLAN 2547 * filtering or has no unicast address 2548 * then we don't create any HW ring 2549 * SRSes. 2550 */ 2551 if ((!MAC_GROUP_HW_VLAN(rx_group) && 2552 vid != VLAN_ID_NONE) || no_unicast) 2553 break; 2554 2555 /* 2556 * When a client has exclusive use of 2557 * a group, and that group's traffic 2558 * is fully HW classified, we create 2559 * an SRS for each HW ring in order to 2560 * make use of dynamic polling of said 2561 * HW rings. 2562 */ 2563 mac_srs = mac_srs_create(mcip, flent, 2564 fanout_type | link_type, 2565 mac_rx_deliver, mcip, NULL, ring); 2566 break; 2567 default: 2568 cmn_err(CE_PANIC, 2569 "srs_setup: mcip = %p " 2570 "trying to add UNKNOWN ring = %p\n", 2571 (void *)mcip, (void *)ring); 2572 break; 2573 } 2574 } 2575 break; 2576 case MAC_GROUP_STATE_SHARED: 2577 /* 2578 * When a group is shared by multiple clients, we must 2579 * use SW classifiction to ensure packets are 2580 * delivered to the correct client. 2581 */ 2582 mac_rx_switch_grp_to_sw(rx_group); 2583 break; 2584 default: 2585 ASSERT(B_FALSE); 2586 break; 2587 } 2588 } 2589 2590 /* 2591 * Set up the TX SRS. 2592 */ 2593 void 2594 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2595 uint32_t link_type) 2596 { 2597 /* 2598 * If this is an exclusive client (e.g. an aggr port), then 2599 * don't setup Tx SRS and Tx soft rings as they won't be used. 2600 * However, we still need to start the rings to send data 2601 * across them. 2602 */ 2603 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 2604 mac_ring_t *ring; 2605 mac_group_t *grp; 2606 2607 grp = (mac_group_t *)flent->fe_tx_ring_group; 2608 2609 if (grp == NULL) 2610 return; 2611 2612 for (ring = grp->mrg_rings; ring != NULL; 2613 ring = ring->mr_next) { 2614 if (ring->mr_state != MR_INUSE) 2615 (void) mac_start_ring(ring); 2616 } 2617 2618 return; 2619 } 2620 2621 /* 2622 * Aggr ports should never have SRSes. 2623 */ 2624 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2625 2626 if (flent->fe_tx_srs == NULL) { 2627 (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 2628 NULL, mcip, NULL, NULL); 2629 } 2630 2631 mac_tx_srs_setup(mcip, flent); 2632 } 2633 2634 /* 2635 * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown 2636 * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set 2637 * when we wish to move a MAC client from one group to another. In 2638 * that case, we need to release the current HW SRSes but keep the SW 2639 * SRS for continued traffic classifiction. 2640 */ 2641 void 2642 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 2643 { 2644 mac_soft_ring_set_t *mac_srs; 2645 int i; 2646 int count = flent->fe_rx_srs_cnt; 2647 2648 for (i = 0; i < count; i++) { 2649 if (i == 0 && hwonly) 2650 continue; 2651 mac_srs = flent->fe_rx_srs[i]; 2652 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2653 mac_srs_free(mac_srs); 2654 flent->fe_rx_srs[i] = NULL; 2655 flent->fe_rx_srs_cnt--; 2656 } 2657 2658 /* 2659 * If we are only tearing down the HW SRSes then there must be 2660 * one SRS left for SW classification. Otherwise we are tearing 2661 * down both HW and SW and there should be no SRSes left. 2662 */ 2663 if (hwonly) 2664 VERIFY3S(flent->fe_rx_srs_cnt, ==, 1); 2665 else 2666 VERIFY3S(flent->fe_rx_srs_cnt, ==, 0); 2667 } 2668 2669 /* 2670 * Remove the TX SRS. 2671 */ 2672 void 2673 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2674 uint32_t link_type) 2675 { 2676 mac_soft_ring_set_t *tx_srs; 2677 mac_srs_tx_t *tx; 2678 2679 if ((tx_srs = flent->fe_tx_srs) == NULL) 2680 return; 2681 2682 tx = &tx_srs->srs_tx; 2683 switch (link_type) { 2684 case SRST_FLOW: 2685 /* 2686 * For flows, we need to work with passed 2687 * flent to find the Rx/Tx SRS. 2688 */ 2689 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2690 break; 2691 case SRST_LINK: 2692 mac_tx_client_condemn((mac_client_handle_t)mcip); 2693 if (tx->st_arg2 != NULL) { 2694 ASSERT(tx_srs->srs_type & SRST_TX); 2695 /* 2696 * The ring itself will be stopped when 2697 * we release the group or in the 2698 * mac_datapath_teardown (for the default 2699 * group) 2700 */ 2701 tx->st_arg2 = NULL; 2702 } 2703 break; 2704 default: 2705 ASSERT(B_FALSE); 2706 break; 2707 } 2708 mac_srs_free(tx_srs); 2709 flent->fe_tx_srs = NULL; 2710 } 2711 2712 /* 2713 * This is the group state machine. 2714 * 2715 * The state of an Rx group is given by 2716 * the following table. The default group and its rings are started in 2717 * mac_start itself and the default group stays in SHARED state until 2718 * mac_stop at which time the group and rings are stopped and and it 2719 * reverts to the Registered state. 2720 * 2721 * Typically this function is called on a group after adding or removing a 2722 * client from it, to find out what should be the new state of the group. 2723 * If the new state is RESERVED, then the client that owns this group 2724 * exclusively is also returned. Note that adding or removing a client from 2725 * a group could also impact the default group and the caller needs to 2726 * evaluate the effect on the default group. 2727 * 2728 * Group type # of clients mi_nactiveclients Group State 2729 * in the group 2730 * 2731 * Non-default 0 N.A. REGISTERED 2732 * Non-default 1 N.A. RESERVED 2733 * 2734 * Default 0 N.A. SHARED 2735 * Default 1 1 RESERVED 2736 * Default 1 > 1 SHARED 2737 * Default > 1 N.A. SHARED 2738 * 2739 * For a TX group, the following is the state table. 2740 * 2741 * Group type # of clients Group State 2742 * in the group 2743 * 2744 * Non-default 0 REGISTERED 2745 * Non-default 1 RESERVED 2746 * 2747 * Default 0 REGISTERED 2748 * Default 1 RESERVED 2749 * Default > 1 SHARED 2750 */ 2751 mac_group_state_t 2752 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 2753 mac_group_t *defgrp, boolean_t rx_group) 2754 { 2755 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2756 2757 *group_only_mcip = NULL; 2758 2759 /* Non-default group */ 2760 2761 if (grp != defgrp) { 2762 if (MAC_GROUP_NO_CLIENT(grp)) 2763 return (MAC_GROUP_STATE_REGISTERED); 2764 2765 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2766 if (*group_only_mcip != NULL) 2767 return (MAC_GROUP_STATE_RESERVED); 2768 2769 return (MAC_GROUP_STATE_SHARED); 2770 } 2771 2772 /* Default group */ 2773 2774 if (MAC_GROUP_NO_CLIENT(grp)) { 2775 if (rx_group) 2776 return (MAC_GROUP_STATE_SHARED); 2777 else 2778 return (MAC_GROUP_STATE_REGISTERED); 2779 } 2780 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2781 if (*group_only_mcip == NULL) 2782 return (MAC_GROUP_STATE_SHARED); 2783 2784 if (rx_group && mip->mi_nactiveclients != 1) 2785 return (MAC_GROUP_STATE_SHARED); 2786 2787 ASSERT(*group_only_mcip != NULL); 2788 return (MAC_GROUP_STATE_RESERVED); 2789 } 2790 2791 /* 2792 * OVERVIEW NOTES FOR DATAPATH 2793 * =========================== 2794 * 2795 * Create an SRS and setup the corresponding flow function and args. 2796 * Add a classification rule for the flow specified by 'flent' and program 2797 * the hardware classifier when applicable. 2798 * 2799 * Rx ring assignment, SRS, polling and B/W enforcement 2800 * ---------------------------------------------------- 2801 * 2802 * We try to use H/W classification on NIC and assign traffic to a 2803 * MAC address to a particular Rx ring. There is a 1-1 mapping 2804 * between a SRS and a Rx ring. The SRS (short for soft ring set) 2805 * dynamically switches the underlying Rx ring between interrupt 2806 * and polling mode and enforces any specified B/W control. 2807 * 2808 * There is always a SRS created and tied to each H/W and S/W rule. 2809 * Whenever we create a H/W rule, we always add the the same rule to 2810 * S/W classifier and tie a SRS to it. 2811 * 2812 * In case a B/W control is specified, its broken into bytes 2813 * per ticks and as soon as the quota for a tick is exhausted, 2814 * the underlying Rx ring is forced into poll mode for remianing 2815 * tick. The SRS poll thread only polls for bytes that are 2816 * allowed to come in the SRS. We typically let 4x the configured 2817 * B/W worth of packets to come in the SRS (to prevent unnecessary 2818 * drops due to bursts) but only process the specified amount. 2819 * 2820 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2821 * Rx rings (and corresponding SRSs) assigned to it. The SRS 2822 * in turn can have softrings to do protocol level fanout or 2823 * softrings to do S/W based fanout or both. In case the NIC 2824 * has no Rx rings, we do S/W classification to respective SRS. 2825 * The S/W classification rule is always setup and ready. This 2826 * allows the MAC layer to reassign Rx rings whenever needed 2827 * but packets still continue to flow via the default path and 2828 * getting S/W classified to correct SRS. 2829 * 2830 * In other cases where a NIC or VNIC is plumbed, our goal is use 2831 * H/W classifier and get two Rx ring assigned for the Link. One 2832 * for TCP and one for UDP|SCTP. The respective SRS still do the 2833 * polling on the Rx ring. For Link that is plumbed for IP, there 2834 * is a TCP squeue which also does polling and can control the 2835 * the Rx ring directly (where SRS is just pass through). For 2836 * the following cases, the SRS does the polling underneath. 2837 * 1) non IP based Links (Links which are not plumbed via ifconfig) 2838 * and paths which have no IP squeues (UDP & SCTP) 2839 * 2) If B/W control is specified on the Link 2840 * 3) If S/W fanout is secified 2841 * 2842 * Note1: As of current implementation, we try to assign only 1 Rx 2843 * ring per Link and more than 1 Rx ring for primary Link for 2844 * H/W based fanout. We always create following softrings per SRS: 2845 * 1) TCP softring which is polled by TCP squeue where possible 2846 * (and also bypasses DLS) 2847 * 2) UDP/SCTP based which bypasses DLS 2848 * 3) OTH softring which goes via DLS (currently deal with IPv6 2849 * and non TCP/UDP/SCTP for IPv4 packets). 2850 * 2851 * It is necessary to create 3 softrings since SRS has to poll 2852 * the single Rx ring underneath and enforce any link level B/W 2853 * control (we can't switch the Rx ring in poll mode just based 2854 * on TCP squeue if the same Rx ring is sharing UDP and other 2855 * traffic as well). Once polling is done and any Link level B/W 2856 * control is specified, the packets are assigned to respective 2857 * softring based on protocol. Since TCP has IP based squeue 2858 * which benefits by polling, we separate TCP packets into 2859 * its own softring which can be polled by IP squeue. We need 2860 * to separate out UDP/SCTP to UDP softring since it can bypass 2861 * the DLS layer which has heavy performance advanatges and we 2862 * need a softring (OTH) for rest. 2863 * 2864 * ToDo: The 3 softrings for protocol are needed only till we can 2865 * get rid of DLS from datapath, make IPv4 and IPv6 paths 2866 * symmetric (deal with mac_header_info for v6 and polling for 2867 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2868 * are generic), and bring SAP based classification to MAC layer 2869 * 2870 * H/W and S/W based fanout and multiple Rx rings per Link 2871 * ------------------------------------------------------- 2872 * 2873 * In case, fanout is requested (or determined automatically based 2874 * on Link speed and processor speed), we try to assign multiple 2875 * Rx rings per Link with their respective SRS. In this case 2876 * the NIC should be capable of fanning out incoming packets between 2877 * the assigned Rx rings (H/W based fanout). All the SRS 2878 * individually switch their Rx ring between interrupt and polling 2879 * mode but share a common B/W control counter in case of Link 2880 * level B/W is specified. 2881 * 2882 * If S/W based fanout is specified in lieu of H/W based fanout, 2883 * the Link SRS creates the specified number of softrings for 2884 * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2885 * out to the correct softring based on their protocol and 2886 * protocol specific hash function. 2887 * 2888 * Primary and non primary MAC clients 2889 * ----------------------------------- 2890 * 2891 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2892 * and are a Layer 2 construct. 2893 * 2894 * Primary NIC: 2895 * The Link that owns the primary MAC address and typically 2896 * is used as the data NIC in non virtualized cases. As such 2897 * H/W resources are preferntially given to primary NIC. As 2898 * far as code is concerned, there is no difference in the 2899 * primary NIC vs VNICs. They are all treated as Links. 2900 * At the very first call to mac_unicast_add() we program the S/W 2901 * classifier for the primary MAC address, get a soft ring set 2902 * (and soft rings based on 'ip_soft_ring_cnt') 2903 * and a Rx ring assigned for polling to get enabled. 2904 * When IP get plumbed and negotiates polling, we can 2905 * let squeue do the polling on TCP softring. 2906 * 2907 * VNICs: 2908 * Same as any other Link. As long as the H/W resource assignments 2909 * are equal, the data path and setup for all Links is same. 2910 * 2911 * Flows: 2912 * Can be configured on Links. They have their own SRS and the 2913 * S/W classifier is programmed appropriately based on the flow. 2914 * The flows typically deal with layer 3 and above and 2915 * creates a soft ring set specific to the flow. The receive 2916 * side function is switched from mac_rx_srs_process to 2917 * mac_rx_srs_subflow_process which first tries to assign the 2918 * packet to appropriate flow SRS and failing which assigns it 2919 * to link SRS. This allows us to avoid the layered approach 2920 * which gets complex. 2921 * 2922 * By the time mac_datapath_setup() completes, we already have the 2923 * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2924 * and S/W classifiers programmed. IP is not plumbed yet (and might 2925 * never be for Virtual Machines guest OS path). When IP is plumbed 2926 * (for both NIC and VNIC), we do a capability negotiation for polling 2927 * and upcall functions etc. 2928 * 2929 * Rx ring Assignement NOTES 2930 * ------------------------- 2931 * 2932 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2933 * as NIC with a single default ring), we assign the only ring to 2934 * primary Link. The primary Link SRS can do polling on it as long as 2935 * it is the only link in use and we compare the MAC address for unicast 2936 * packets before accepting an incoming packet (there is no need for S/W 2937 * classification in this case). We disable polling on the only ring the 2938 * moment 2nd link gets created (the polling remains enabled even though 2939 * there are broadcast and * multicast flows created). 2940 * 2941 * If the NIC has more than 1 Rx ring, we assign the default ring (the 2942 * 1st ring) to deal with broadcast, multicast and traffic for other 2943 * NICs which needs S/W classification. We assign the primary mac 2944 * addresses to another ring by specifiying a classification rule for 2945 * primary unicast MAC address to the selected ring. The primary Link 2946 * (and its SRS) can continue to poll the assigned Rx ring at all times 2947 * independantly. 2948 * 2949 * Note: In future, if no fanout is specified, we try to assign 2 Rx 2950 * rings for the primary Link with the primary MAC address + TCP going 2951 * to one ring and primary MAC address + UDP|SCTP going to other ring. 2952 * Any remaining traffic for primary MAC address can go to the default 2953 * Rx ring and get S/W classified. This way the respective SRSs don't 2954 * need to do proto fanout and don't need to have softrings at all and 2955 * can poll their respective Rx rings. 2956 * 2957 * As an optimization, when a new NIC or VNIC is created, we can get 2958 * only one Rx ring and make it a TCP specific Rx ring and use the 2959 * H/W default Rx ring for the rest (this Rx ring is never polled). 2960 * 2961 * For clients that don't have MAC address, but want to receive and 2962 * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 2963 * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 2964 * always give the default group and use software classification (i.e. 2965 * even if this is the only client in the default group, we will 2966 * leave group as shared). 2967 */ 2968 2969 int 2970 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2971 uint32_t link_type) 2972 { 2973 mac_impl_t *mip = mcip->mci_mip; 2974 mac_group_t *rgroup = NULL; 2975 mac_group_t *tgroup = NULL; 2976 mac_group_t *default_rgroup; 2977 mac_group_t *default_tgroup; 2978 int err; 2979 uint16_t vid; 2980 uint8_t *mac_addr; 2981 mac_group_state_t next_state; 2982 mac_client_impl_t *group_only_mcip; 2983 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2984 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2985 boolean_t rxhw; 2986 boolean_t txhw; 2987 boolean_t use_default = B_FALSE; 2988 cpupart_t *cpupart; 2989 boolean_t no_unicast; 2990 boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 2991 mac_client_impl_t *reloc_pmcip = NULL; 2992 boolean_t use_hw; 2993 2994 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2995 2996 switch (link_type) { 2997 case SRST_FLOW: 2998 mac_srs_group_setup(mcip, flent, link_type); 2999 return (0); 3000 3001 case SRST_LINK: 3002 no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 3003 mac_addr = flent->fe_flow_desc.fd_dst_mac; 3004 3005 /* Default RX group */ 3006 default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 3007 3008 /* Default TX group */ 3009 default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 3010 3011 if (no_unicast) { 3012 rgroup = default_rgroup; 3013 tgroup = default_tgroup; 3014 goto grp_found; 3015 } 3016 rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 3017 (mrp->mrp_nrxrings > 0 || 3018 (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 3019 txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 3020 (mrp->mrp_ntxrings > 0 || 3021 (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 3022 3023 /* 3024 * All the rings initially belong to the default group 3025 * under dynamic grouping. The primary client uses the 3026 * default group when it is the only client. The 3027 * default group is also used as the destination for 3028 * all multicast and broadcast traffic of all clients. 3029 * Therefore, the primary client loses its ability to 3030 * poll the softrings on addition of a second client. 3031 * To avoid a performance penalty, MAC will move the 3032 * primary client to a dedicated group when it can. 3033 * 3034 * When using static grouping, the primary client 3035 * begins life on a non-default group. There is 3036 * no moving needed upon addition of a second client. 3037 */ 3038 if (!isprimary && mip->mi_nactiveclients == 2 && 3039 (group_only_mcip = mac_primary_client_handle(mip)) != 3040 NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 3041 reloc_pmcip = mac_check_primary_relocation( 3042 group_only_mcip, rxhw); 3043 } 3044 3045 /* 3046 * Check to see if we can get an exclusive group for 3047 * this mac address or if there already exists a 3048 * group that has this mac address (case of VLANs). 3049 * If no groups are available, use the default group. 3050 */ 3051 rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 3052 if (rgroup == NULL && rxhw) { 3053 err = ENOSPC; 3054 goto setup_failed; 3055 } else if (rgroup == NULL) { 3056 rgroup = default_rgroup; 3057 } 3058 3059 /* 3060 * If we are adding a second client to a 3061 * non-default group then we need to move the 3062 * existing client to the default group and 3063 * add the new client to the default group as 3064 * well. 3065 */ 3066 if (rgroup != default_rgroup && 3067 rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) { 3068 group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup); 3069 err = mac_rx_switch_group(group_only_mcip, rgroup, 3070 default_rgroup); 3071 3072 if (err != 0) 3073 goto setup_failed; 3074 3075 rgroup = default_rgroup; 3076 } 3077 3078 /* 3079 * Check to see if we can get an exclusive group for 3080 * this mac client. If no groups are available, use 3081 * the default group. 3082 */ 3083 tgroup = mac_reserve_tx_group(mcip, B_FALSE); 3084 if (tgroup == NULL && txhw) { 3085 if (rgroup != NULL && rgroup != default_rgroup) 3086 mac_release_rx_group(mcip, rgroup); 3087 err = ENOSPC; 3088 goto setup_failed; 3089 } else if (tgroup == NULL) { 3090 tgroup = default_tgroup; 3091 } 3092 3093 /* 3094 * Some NICs don't support any Rx rings, so there may not 3095 * even be a default group. 3096 */ 3097 grp_found: 3098 if (rgroup != NULL) { 3099 if (rgroup != default_rgroup && 3100 MAC_GROUP_NO_CLIENT(rgroup) && 3101 (rxhw || mcip->mci_share != 0)) { 3102 MAC_RX_GRP_RESERVED(mip); 3103 if (mip->mi_rx_group_type == 3104 MAC_GROUP_TYPE_DYNAMIC) { 3105 MAC_RX_RING_RESERVED(mip, 3106 rgroup->mrg_cur_count); 3107 } 3108 } 3109 3110 flent->fe_rx_ring_group = rgroup; 3111 /* 3112 * Add the client to the group and update the 3113 * group's state. If rgroup != default_group 3114 * then the rgroup should only ever have one 3115 * client and be in the RESERVED state. But no 3116 * matter what, the default_rgroup will enter 3117 * the SHARED state since it has to receive 3118 * all broadcast and multicast traffic. This 3119 * case is handled later in the function. 3120 */ 3121 mac_group_add_client(rgroup, mcip); 3122 next_state = mac_group_next_state(rgroup, 3123 &group_only_mcip, default_rgroup, B_TRUE); 3124 mac_set_group_state(rgroup, next_state); 3125 } 3126 3127 if (tgroup != NULL) { 3128 if (tgroup != default_tgroup && 3129 MAC_GROUP_NO_CLIENT(tgroup) && 3130 (txhw || mcip->mci_share != 0)) { 3131 MAC_TX_GRP_RESERVED(mip); 3132 if (mip->mi_tx_group_type == 3133 MAC_GROUP_TYPE_DYNAMIC) { 3134 MAC_TX_RING_RESERVED(mip, 3135 tgroup->mrg_cur_count); 3136 } 3137 } 3138 flent->fe_tx_ring_group = tgroup; 3139 mac_group_add_client(tgroup, mcip); 3140 next_state = mac_group_next_state(tgroup, 3141 &group_only_mcip, default_tgroup, B_FALSE); 3142 tgroup->mrg_state = next_state; 3143 } 3144 3145 /* We are setting up minimal datapath only */ 3146 if (no_unicast) { 3147 mac_srs_group_setup(mcip, flent, link_type); 3148 break; 3149 } 3150 3151 /* Program software classification. */ 3152 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3153 goto setup_failed; 3154 3155 /* Program hardware classification. */ 3156 vid = i_mac_flow_vid(flent); 3157 use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0; 3158 err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw); 3159 3160 if (err != 0) 3161 goto setup_failed; 3162 3163 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 3164 VERIFY3P(mcip->mci_unicast, !=, NULL); 3165 3166 /* 3167 * Setup the Rx and Tx SRSes. If the client has a 3168 * reserved group, then mac_srs_group_setup() creates 3169 * the required SRSes for the HW rings. If we have a 3170 * shared group, mac_srs_group_setup() dismantles the 3171 * HW SRSes of the previously exclusive group. 3172 */ 3173 mac_srs_group_setup(mcip, flent, link_type); 3174 3175 /* (Re)init the v6 token & local addr used by link protection */ 3176 mac_protect_update_mac_token(mcip); 3177 break; 3178 3179 default: 3180 ASSERT(B_FALSE); 3181 break; 3182 } 3183 3184 /* 3185 * All broadcast and multicast traffic is received only on the default 3186 * group. If we have setup the datapath for a non-default group above 3187 * then move the default group to shared state to allow distribution of 3188 * incoming broadcast traffic to the other groups and dismantle the 3189 * SRSes over the default group. 3190 */ 3191 if (rgroup != NULL) { 3192 if (rgroup != default_rgroup) { 3193 if (default_rgroup->mrg_state == 3194 MAC_GROUP_STATE_RESERVED) { 3195 group_only_mcip = MAC_GROUP_ONLY_CLIENT( 3196 default_rgroup); 3197 ASSERT(group_only_mcip != NULL && 3198 mip->mi_nactiveclients > 1); 3199 3200 mac_set_group_state(default_rgroup, 3201 MAC_GROUP_STATE_SHARED); 3202 mac_rx_srs_group_setup(group_only_mcip, 3203 group_only_mcip->mci_flent, SRST_LINK); 3204 pool_lock(); 3205 cpupart = mac_pset_find(mrp, &use_default); 3206 mac_fanout_setup(group_only_mcip, 3207 group_only_mcip->mci_flent, 3208 MCIP_RESOURCE_PROPS(group_only_mcip), 3209 mac_rx_deliver, group_only_mcip, NULL, 3210 cpupart); 3211 mac_set_pool_effective(use_default, cpupart, 3212 mrp, emrp); 3213 pool_unlock(); 3214 } 3215 ASSERT(default_rgroup->mrg_state == 3216 MAC_GROUP_STATE_SHARED); 3217 } 3218 3219 /* 3220 * A VLAN MAC client on a reserved group still 3221 * requires SW classification if the MAC doesn't 3222 * provide VLAN HW filtering. 3223 * 3224 * Clients with no unicast address also require SW 3225 * classification. 3226 */ 3227 if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 3228 ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) || 3229 no_unicast)) { 3230 mac_rx_switch_grp_to_sw(rgroup); 3231 } 3232 3233 } 3234 3235 mac_set_rings_effective(mcip); 3236 return (0); 3237 3238 setup_failed: 3239 /* Switch the primary back to default group */ 3240 if (reloc_pmcip != NULL) { 3241 (void) mac_rx_switch_group(reloc_pmcip, 3242 reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 3243 } 3244 mac_datapath_teardown(mcip, flent, link_type); 3245 return (err); 3246 } 3247 3248 void 3249 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3250 uint32_t link_type) 3251 { 3252 mac_impl_t *mip = mcip->mci_mip; 3253 mac_group_t *group = NULL; 3254 mac_client_impl_t *grp_only_mcip; 3255 flow_entry_t *group_only_flent; 3256 mac_group_t *default_group; 3257 boolean_t check_default_group = B_FALSE; 3258 mac_group_state_t next_state; 3259 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 3260 uint16_t vid; 3261 3262 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3263 3264 switch (link_type) { 3265 case SRST_FLOW: 3266 mac_rx_srs_group_teardown(flent, B_FALSE); 3267 mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3268 return; 3269 3270 case SRST_LINK: 3271 /* Stop sending packets */ 3272 mac_tx_client_block(mcip); 3273 group = flent->fe_rx_ring_group; 3274 vid = i_mac_flow_vid(flent); 3275 3276 /* 3277 * Stop the packet flow from the hardware by disabling 3278 * any hardware filters assigned to this client. 3279 */ 3280 if (mcip->mci_unicast != NULL) { 3281 int err; 3282 3283 err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid); 3284 3285 if (err != 0) { 3286 cmn_err(CE_WARN, "%s: failed to remove a MAC HW" 3287 " filters because of error 0x%x", 3288 mip->mi_name, err); 3289 } 3290 3291 mcip->mci_unicast = NULL; 3292 } 3293 3294 /* Stop the packets coming from the S/W classifier */ 3295 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3296 mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3297 3298 /* Quiesce and destroy all the SRSes. */ 3299 mac_rx_srs_group_teardown(flent, B_FALSE); 3300 mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 3301 3302 ASSERT3P(mcip->mci_flent, ==, flent); 3303 ASSERT3P(flent->fe_next, ==, NULL); 3304 3305 /* 3306 * Release our hold on the group as well. We need 3307 * to check if the shared group has only one client 3308 * left who can use it exclusively. Also, if we 3309 * were the last client, release the group. 3310 */ 3311 default_group = MAC_DEFAULT_RX_GROUP(mip); 3312 if (group != NULL) { 3313 mac_group_remove_client(group, mcip); 3314 next_state = mac_group_next_state(group, 3315 &grp_only_mcip, default_group, B_TRUE); 3316 3317 if (next_state == MAC_GROUP_STATE_RESERVED) { 3318 /* 3319 * Only one client left on this RX group. 3320 */ 3321 VERIFY3P(grp_only_mcip, !=, NULL); 3322 mac_set_group_state(group, 3323 MAC_GROUP_STATE_RESERVED); 3324 group_only_flent = grp_only_mcip->mci_flent; 3325 3326 /* 3327 * The only remaining client has exclusive 3328 * access on the group. Allow it to 3329 * dynamically poll the H/W rings etc. 3330 */ 3331 mac_rx_srs_group_setup(grp_only_mcip, 3332 group_only_flent, SRST_LINK); 3333 mac_fanout_setup(grp_only_mcip, 3334 group_only_flent, 3335 MCIP_RESOURCE_PROPS(grp_only_mcip), 3336 mac_rx_deliver, grp_only_mcip, NULL, NULL); 3337 mac_rx_group_unmark(group, MR_INCIPIENT); 3338 mac_set_rings_effective(grp_only_mcip); 3339 } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3340 /* 3341 * This is a non-default group being freed up. 3342 * We need to reevaluate the default group 3343 * to see if the primary client can get 3344 * exclusive access to the default group. 3345 */ 3346 VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip)); 3347 if (mrp->mrp_mask & MRP_RX_RINGS) { 3348 MAC_RX_GRP_RELEASED(mip); 3349 if (mip->mi_rx_group_type == 3350 MAC_GROUP_TYPE_DYNAMIC) { 3351 MAC_RX_RING_RELEASED(mip, 3352 group->mrg_cur_count); 3353 } 3354 } 3355 mac_release_rx_group(mcip, group); 3356 mac_set_group_state(group, 3357 MAC_GROUP_STATE_REGISTERED); 3358 check_default_group = B_TRUE; 3359 } else { 3360 VERIFY3S(next_state, ==, 3361 MAC_GROUP_STATE_SHARED); 3362 mac_set_group_state(group, 3363 MAC_GROUP_STATE_SHARED); 3364 mac_rx_group_unmark(group, MR_CONDEMNED); 3365 } 3366 flent->fe_rx_ring_group = NULL; 3367 } 3368 /* 3369 * Remove the client from the TX group. Additionally, if 3370 * this a non-default group, then we also need to release 3371 * the group. 3372 */ 3373 group = flent->fe_tx_ring_group; 3374 default_group = MAC_DEFAULT_TX_GROUP(mip); 3375 if (group != NULL) { 3376 mac_group_remove_client(group, mcip); 3377 next_state = mac_group_next_state(group, 3378 &grp_only_mcip, default_group, B_FALSE); 3379 if (next_state == MAC_GROUP_STATE_REGISTERED) { 3380 if (group != default_group) { 3381 if (mrp->mrp_mask & MRP_TX_RINGS) { 3382 MAC_TX_GRP_RELEASED(mip); 3383 if (mip->mi_tx_group_type == 3384 MAC_GROUP_TYPE_DYNAMIC) { 3385 MAC_TX_RING_RELEASED( 3386 mip, group-> 3387 mrg_cur_count); 3388 } 3389 } 3390 mac_release_tx_group(mcip, group); 3391 /* 3392 * If the default group is reserved, 3393 * then we need to set the effective 3394 * rings as we would have given 3395 * back some rings when the group 3396 * was released 3397 */ 3398 if (mip->mi_tx_group_type == 3399 MAC_GROUP_TYPE_DYNAMIC && 3400 default_group->mrg_state == 3401 MAC_GROUP_STATE_RESERVED) { 3402 grp_only_mcip = 3403 MAC_GROUP_ONLY_CLIENT 3404 (default_group); 3405 mac_set_rings_effective( 3406 grp_only_mcip); 3407 } 3408 } else { 3409 mac_ring_t *ring; 3410 int cnt; 3411 int ringcnt; 3412 3413 /* 3414 * Stop all the rings except the 3415 * default ring. 3416 */ 3417 ringcnt = group->mrg_cur_count; 3418 ring = group->mrg_rings; 3419 for (cnt = 0; cnt < ringcnt; cnt++) { 3420 if (ring->mr_state == 3421 MR_INUSE && ring != 3422 (mac_ring_t *) 3423 mip->mi_default_tx_ring) { 3424 mac_stop_ring(ring); 3425 ring->mr_flag = 0; 3426 } 3427 ring = ring->mr_next; 3428 } 3429 } 3430 } else if (next_state == MAC_GROUP_STATE_RESERVED) { 3431 mac_set_rings_effective(grp_only_mcip); 3432 } 3433 flent->fe_tx_ring_group = NULL; 3434 group->mrg_state = next_state; 3435 } 3436 break; 3437 default: 3438 ASSERT(B_FALSE); 3439 break; 3440 } 3441 3442 /* 3443 * The mac client using the default group gets exclusive access to the 3444 * default group if and only if it is the sole client on the entire 3445 * mip. If so set the group state to reserved, and set up the SRSes 3446 * over the default group. 3447 */ 3448 if (check_default_group) { 3449 default_group = MAC_DEFAULT_RX_GROUP(mip); 3450 VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED); 3451 next_state = mac_group_next_state(default_group, 3452 &grp_only_mcip, default_group, B_TRUE); 3453 if (next_state == MAC_GROUP_STATE_RESERVED) { 3454 VERIFY3P(grp_only_mcip, !=, NULL); 3455 VERIFY3U(mip->mi_nactiveclients, ==, 1); 3456 mac_set_group_state(default_group, 3457 MAC_GROUP_STATE_RESERVED); 3458 mac_rx_srs_group_setup(grp_only_mcip, 3459 grp_only_mcip->mci_flent, SRST_LINK); 3460 mac_fanout_setup(grp_only_mcip, 3461 grp_only_mcip->mci_flent, 3462 MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 3463 grp_only_mcip, NULL, NULL); 3464 mac_rx_group_unmark(default_group, MR_INCIPIENT); 3465 mac_set_rings_effective(grp_only_mcip); 3466 } 3467 } 3468 3469 /* 3470 * If the primary is the only one left and the MAC supports 3471 * dynamic grouping, we need to see if the primary needs to 3472 * be moved to the default group so that it can use all the 3473 * H/W rings. 3474 */ 3475 if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 3476 mip->mi_nactiveclients == 1 && 3477 mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 3478 default_group = MAC_DEFAULT_RX_GROUP(mip); 3479 grp_only_mcip = mac_primary_client_handle(mip); 3480 if (grp_only_mcip == NULL) 3481 return; 3482 group_only_flent = grp_only_mcip->mci_flent; 3483 mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 3484 /* 3485 * If the primary has an explicit property set, leave it 3486 * alone. 3487 */ 3488 if (mrp->mrp_mask & MRP_RX_RINGS) 3489 return; 3490 /* 3491 * Switch the primary to the default group. 3492 */ 3493 (void) mac_rx_switch_group(grp_only_mcip, 3494 group_only_flent->fe_rx_ring_group, default_group); 3495 } 3496 } 3497 3498 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3499 3500 static void 3501 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3502 { 3503 if (mac_srs->srs_type & SRST_TX) { 3504 mac_srs_tx_t *tx; 3505 3506 ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 3507 ASSERT(mac_srs->srs_udp_soft_rings == NULL); 3508 ASSERT(mac_srs->srs_tcp6_soft_rings == NULL); 3509 ASSERT(mac_srs->srs_udp6_soft_rings == NULL); 3510 ASSERT(mac_srs->srs_oth_soft_rings == NULL); 3511 ASSERT(mac_srs->srs_tx_soft_rings != NULL); 3512 kmem_free(mac_srs->srs_tx_soft_rings, 3513 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3514 mac_srs->srs_tx_soft_rings = NULL; 3515 tx = &mac_srs->srs_tx; 3516 if (tx->st_soft_rings != NULL) { 3517 kmem_free(tx->st_soft_rings, 3518 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3519 } 3520 } else { 3521 ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3522 3523 ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3524 kmem_free(mac_srs->srs_tcp_soft_rings, 3525 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3526 mac_srs->srs_tcp_soft_rings = NULL; 3527 3528 ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3529 kmem_free(mac_srs->srs_udp_soft_rings, 3530 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3531 mac_srs->srs_udp_soft_rings = NULL; 3532 3533 ASSERT(mac_srs->srs_tcp6_soft_rings != NULL); 3534 kmem_free(mac_srs->srs_tcp6_soft_rings, 3535 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3536 mac_srs->srs_tcp6_soft_rings = NULL; 3537 3538 ASSERT(mac_srs->srs_udp6_soft_rings != NULL); 3539 kmem_free(mac_srs->srs_udp6_soft_rings, 3540 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3541 mac_srs->srs_udp6_soft_rings = NULL; 3542 3543 ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3544 kmem_free(mac_srs->srs_oth_soft_rings, 3545 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3546 mac_srs->srs_oth_soft_rings = NULL; 3547 } 3548 } 3549 3550 /* 3551 * An RX SRS is attached to at most one mac_ring. 3552 * A TX SRS has no rings. 3553 */ 3554 static void 3555 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3556 { 3557 mac_client_impl_t *mcip; 3558 mac_ring_t *ring; 3559 flow_entry_t *flent; 3560 3561 ring = mac_srs->srs_ring; 3562 if (mac_srs->srs_type & SRST_TX) { 3563 ASSERT(ring == NULL); 3564 return; 3565 } 3566 3567 if (ring == NULL) 3568 return; 3569 3570 /* 3571 * Broadcast flows don't have a client impl association, but they 3572 * use only soft rings. 3573 */ 3574 flent = mac_srs->srs_flent; 3575 mcip = flent->fe_mcip; 3576 ASSERT(mcip != NULL); 3577 3578 ring->mr_classify_type = MAC_NO_CLASSIFIER; 3579 ring->mr_srs = NULL; 3580 } 3581 3582 /* 3583 * Physical unlink and free of the data structures happen below. This is 3584 * driven from mac_flow_destroy(), on the last refrele of a flow. 3585 * 3586 * Assumes Rx srs is 1-1 mapped with an ring. 3587 */ 3588 void 3589 mac_srs_free(mac_soft_ring_set_t *mac_srs) 3590 { 3591 ASSERT(mac_srs->srs_mcip == NULL || 3592 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3593 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3594 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3595 3596 mac_drop_chain(mac_srs->srs_first, "SRS free"); 3597 mac_srs_ring_free(mac_srs); 3598 mac_srs_soft_rings_free(mac_srs); 3599 mac_srs_fanout_list_free(mac_srs); 3600 3601 mac_srs->srs_bw = NULL; 3602 mac_srs_stat_delete(mac_srs); 3603 kmem_cache_free(mac_srs_cache, mac_srs); 3604 } 3605 3606 static void 3607 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3608 { 3609 mac_soft_ring_t *softring; 3610 3611 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3612 3613 mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3614 if (s_ring_flag == S_RING_CONDEMNED) { 3615 while (mac_srs->srs_soft_ring_condemned_count != 3616 mac_srs->srs_soft_ring_count) 3617 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3618 } else { 3619 while (mac_srs->srs_soft_ring_quiesced_count != 3620 mac_srs->srs_soft_ring_count) 3621 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3622 } 3623 mutex_exit(&mac_srs->srs_lock); 3624 3625 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3626 softring = softring->s_ring_next) { 3627 (void) untimeout(softring->s_ring_tid); 3628 softring->s_ring_tid = NULL; 3629 } 3630 3631 (void) untimeout(mac_srs->srs_tid); 3632 mac_srs->srs_tid = NULL; 3633 3634 mutex_enter(&mac_srs->srs_lock); 3635 } 3636 3637 /* 3638 * The block comment above mac_rx_classify_flow_state_change explains the 3639 * background. At this point upcalls from the driver (both hardware classified 3640 * and software classified) have been cut off. We now need to quiesce the 3641 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3642 * the master controller. The steps involved are described below in the function 3643 */ 3644 void 3645 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3646 { 3647 uint_t s_ring_flag; 3648 uint_t srs_poll_wait_flag; 3649 3650 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3651 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3652 3653 if (mac_srs->srs_state & SRS_CONDEMNED) { 3654 s_ring_flag = S_RING_CONDEMNED; 3655 srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3656 } else { 3657 s_ring_flag = S_RING_QUIESCE; 3658 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3659 } 3660 3661 /* 3662 * In the case of Rx SRS wait till the poll thread is done. 3663 */ 3664 if ((mac_srs->srs_type & SRST_TX) == 0 && 3665 mac_srs->srs_poll_thr != NULL) { 3666 while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3667 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3668 3669 /* 3670 * Turn off polling as part of the quiesce operation. 3671 */ 3672 MAC_SRS_POLLING_OFF(mac_srs); 3673 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3674 } 3675 3676 /* 3677 * Then signal the soft ring worker threads to quiesce or quit 3678 * as needed and then wait till that happens. 3679 */ 3680 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3681 3682 if (mac_srs->srs_state & SRS_CONDEMNED) 3683 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3684 else 3685 mac_srs->srs_state |= SRS_QUIESCE_DONE; 3686 cv_signal(&mac_srs->srs_quiesce_done_cv); 3687 } 3688 3689 /* 3690 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3691 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3692 * or poll thread. This function is internal to the quiescing logic and is 3693 * called internally from the SRS quiesce or flow quiesce or client quiesce 3694 * higher level functions. 3695 */ 3696 void 3697 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3698 { 3699 mac_ring_t *ring; 3700 3701 ring = mac_srs->srs_ring; 3702 ASSERT(ring == NULL || ring->mr_refcnt == 0); 3703 3704 if (srs_flag == SRS_CONDEMNED) { 3705 /* 3706 * The SRS is going away. We need to unbind the SRS and SR 3707 * threads before removing from the global SRS list. Otherwise 3708 * there is a small window where the cpu reconfig callbacks 3709 * may miss the SRS in the list walk and DR could fail since 3710 * there are still bound threads. 3711 */ 3712 mac_srs_threads_unbind(mac_srs); 3713 mac_srs_remove_glist(mac_srs); 3714 } 3715 /* 3716 * Wakeup the SRS worker and poll threads. 3717 */ 3718 mutex_enter(&mac_srs->srs_lock); 3719 mac_srs->srs_state |= srs_flag; 3720 cv_signal(&mac_srs->srs_async); 3721 cv_signal(&mac_srs->srs_cv); 3722 mutex_exit(&mac_srs->srs_lock); 3723 } 3724 3725 /* 3726 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3727 * from the driver are done, then the Rx SRS is quiesced and only then can 3728 * we signal the soft rings. Thus this function can't be called arbitrarily 3729 * without satisfying the prerequisites. On the Tx side, the threads from 3730 * top need to quiesced, then the Tx SRS and only then can we signal the 3731 * Tx soft rings. 3732 */ 3733 static void 3734 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3735 { 3736 mac_soft_ring_t *softring; 3737 3738 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3739 softring = softring->s_ring_next) 3740 mac_soft_ring_signal(softring, sr_flag); 3741 } 3742 3743 /* 3744 * The block comment above mac_rx_classify_flow_state_change explains the 3745 * background. At this point the SRS is quiesced and we need to restart the 3746 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3747 * the master controller. The steps involved are described below in the function 3748 */ 3749 void 3750 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3751 { 3752 boolean_t iam_rx_srs; 3753 mac_soft_ring_t *softring; 3754 3755 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3756 if ((mac_srs->srs_type & SRST_TX) != 0) { 3757 iam_rx_srs = B_FALSE; 3758 ASSERT((mac_srs->srs_state & 3759 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3760 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3761 } else { 3762 iam_rx_srs = B_TRUE; 3763 ASSERT((mac_srs->srs_state & 3764 (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3765 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3766 if (mac_srs->srs_poll_thr != NULL) { 3767 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3768 SRS_POLL_THR_QUIESCED); 3769 } 3770 } 3771 3772 /* 3773 * Signal any quiesced soft ring workers to restart and wait for the 3774 * soft ring down count to come down to zero. 3775 */ 3776 if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3777 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3778 softring = softring->s_ring_next) { 3779 if (!(softring->s_ring_state & S_RING_QUIESCE)) 3780 continue; 3781 mac_soft_ring_signal(softring, S_RING_RESTART); 3782 } 3783 while (mac_srs->srs_soft_ring_quiesced_count != 0) 3784 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3785 } 3786 3787 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3788 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3789 /* 3790 * Signal the poll thread and ask it to restart. Wait till it 3791 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3792 * cleared. 3793 */ 3794 mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3795 cv_signal(&mac_srs->srs_cv); 3796 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3797 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3798 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3799 } 3800 /* Wake up any waiter waiting for the restart to complete */ 3801 mac_srs->srs_state |= SRS_RESTART_DONE; 3802 cv_signal(&mac_srs->srs_quiesce_done_cv); 3803 } 3804 3805 static void 3806 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3807 { 3808 mutex_enter(&mac_srs->srs_lock); 3809 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3810 ASSERT(mac_srs->srs_worker_cpuid == -1); 3811 mutex_exit(&mac_srs->srs_lock); 3812 return; 3813 } 3814 3815 mac_srs->srs_worker_cpuid = -1; 3816 mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3817 thread_affinity_clear(mac_srs->srs_worker); 3818 mutex_exit(&mac_srs->srs_lock); 3819 } 3820 3821 static void 3822 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3823 { 3824 mutex_enter(&mac_srs->srs_lock); 3825 if (mac_srs->srs_poll_thr == NULL || 3826 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3827 ASSERT(mac_srs->srs_poll_cpuid == -1); 3828 mutex_exit(&mac_srs->srs_lock); 3829 return; 3830 } 3831 3832 mac_srs->srs_poll_cpuid = -1; 3833 mac_srs->srs_state &= ~SRS_POLL_BOUND; 3834 thread_affinity_clear(mac_srs->srs_poll_thr); 3835 mutex_exit(&mac_srs->srs_lock); 3836 } 3837 3838 static void 3839 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3840 { 3841 mac_soft_ring_t *soft_ring; 3842 3843 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3844 3845 mutex_enter(&cpu_lock); 3846 mac_srs_worker_unbind(mac_srs); 3847 if (!(mac_srs->srs_type & SRST_TX)) 3848 mac_srs_poll_unbind(mac_srs); 3849 3850 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3851 soft_ring = soft_ring->s_ring_next) { 3852 mac_soft_ring_unbind(soft_ring); 3853 } 3854 mutex_exit(&cpu_lock); 3855 } 3856 3857 /* 3858 * When a CPU is going away, unbind all MAC threads which are bound 3859 * to that CPU. The affinity of the thread to the CPU is saved to allow 3860 * the thread to be rebound to the CPU if it comes back online. 3861 */ 3862 static void 3863 mac_walk_srs_and_unbind(int cpuid) 3864 { 3865 mac_soft_ring_set_t *mac_srs; 3866 mac_soft_ring_t *soft_ring; 3867 3868 rw_enter(&mac_srs_g_lock, RW_READER); 3869 3870 if ((mac_srs = mac_srs_g_list) == NULL) 3871 goto done; 3872 3873 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3874 if (mac_srs->srs_worker_cpuid == cpuid) { 3875 mac_srs->srs_worker_cpuid_save = cpuid; 3876 mac_srs_worker_unbind(mac_srs); 3877 } 3878 3879 if (!(mac_srs->srs_type & SRST_TX)) { 3880 if (mac_srs->srs_poll_cpuid == cpuid) { 3881 mac_srs->srs_poll_cpuid_save = cpuid; 3882 mac_srs_poll_unbind(mac_srs); 3883 } 3884 } 3885 3886 /* Next tackle the soft rings associated with the srs */ 3887 mutex_enter(&mac_srs->srs_lock); 3888 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3889 soft_ring = soft_ring->s_ring_next) { 3890 if (soft_ring->s_ring_cpuid == cpuid) { 3891 soft_ring->s_ring_cpuid_save = cpuid; 3892 mac_soft_ring_unbind(soft_ring); 3893 } 3894 } 3895 mutex_exit(&mac_srs->srs_lock); 3896 } 3897 done: 3898 rw_exit(&mac_srs_g_lock); 3899 } 3900 3901 /* TX SETUP and TEARDOWN ROUTINES */ 3902 3903 /* 3904 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3905 * handle the case where the number of rings is one. I.e. there is 3906 * a ring pointed to by mac_srs->srs_tx_arg2. 3907 */ 3908 void 3909 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3910 { 3911 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3912 mac_soft_ring_t *soft_ring; 3913 int count = mac_srs->srs_tx_ring_count; 3914 uint32_t soft_ring_type = ST_RING_TX; 3915 uint_t ring_info; 3916 3917 ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3918 ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 3919 if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 3920 soft_ring_type |= ST_RING_WORKER_ONLY; 3921 soft_ring = mac_soft_ring_create(count, 0, 3922 soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3923 NULL, mcip, (mac_resource_handle_t)tx_ring); 3924 mac_srs->srs_tx_ring_count++; 3925 mac_srs_update_fanout_list(mac_srs); 3926 /* 3927 * put this soft ring in quiesce mode too so when we restart 3928 * all soft rings in the srs are in the same state. 3929 */ 3930 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3931 } 3932 3933 static void 3934 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3935 { 3936 int sringcnt; 3937 3938 mutex_enter(&mac_srs->srs_lock); 3939 sringcnt = mac_srs->srs_soft_ring_count; 3940 ASSERT(sringcnt > 0); 3941 mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3942 3943 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3944 while (mac_srs->srs_soft_ring_condemned_count != 1) 3945 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3946 3947 if (softring == mac_srs->srs_soft_ring_head) { 3948 mac_srs->srs_soft_ring_head = softring->s_ring_next; 3949 if (mac_srs->srs_soft_ring_head != NULL) { 3950 mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3951 } else { 3952 mac_srs->srs_soft_ring_tail = NULL; 3953 } 3954 } else { 3955 softring->s_ring_prev->s_ring_next = 3956 softring->s_ring_next; 3957 if (softring->s_ring_next != NULL) { 3958 softring->s_ring_next->s_ring_prev = 3959 softring->s_ring_prev; 3960 } else { 3961 mac_srs->srs_soft_ring_tail = 3962 softring->s_ring_prev; 3963 } 3964 } 3965 mac_srs->srs_soft_ring_count--; 3966 3967 mac_srs->srs_soft_ring_condemned_count--; 3968 mutex_exit(&mac_srs->srs_lock); 3969 3970 mac_soft_ring_free(softring); 3971 } 3972 3973 void 3974 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3975 { 3976 int i; 3977 mac_soft_ring_t *soft_ring, *remove_sring; 3978 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3979 3980 mutex_enter(&mac_srs->srs_lock); 3981 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 3982 soft_ring = mac_srs->srs_tx_soft_rings[i]; 3983 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3984 break; 3985 } 3986 mutex_exit(&mac_srs->srs_lock); 3987 ASSERT(i < mac_srs->srs_tx_ring_count); 3988 remove_sring = soft_ring; 3989 /* 3990 * In the case of aggr, the soft ring associated with a Tx ring 3991 * is also stored in st_soft_rings[] array. That entry should 3992 * be removed. 3993 */ 3994 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 3995 mac_srs_tx_t *tx = &mac_srs->srs_tx; 3996 3997 ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 3998 tx->st_soft_rings[tx_ring->mr_index] = NULL; 3999 } 4000 mac_soft_ring_remove(mac_srs, remove_sring); 4001 mac_srs_update_fanout_list(mac_srs); 4002 } 4003 4004 /* 4005 * mac_tx_srs_setup(): 4006 * Used to setup Tx rings. If no free Tx ring is available, then default 4007 * Tx ring is used. 4008 */ 4009 void 4010 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 4011 { 4012 mac_impl_t *mip = mcip->mci_mip; 4013 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 4014 int i; 4015 int tx_ring_count = 0; 4016 uint32_t soft_ring_type; 4017 mac_group_t *grp = NULL; 4018 mac_ring_t *ring; 4019 mac_srs_tx_t *tx = &tx_srs->srs_tx; 4020 boolean_t is_aggr; 4021 uint_t ring_info = 0; 4022 4023 is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0; 4024 grp = flent->fe_tx_ring_group; 4025 if (grp == NULL) { 4026 ring = (mac_ring_t *)mip->mi_default_tx_ring; 4027 goto no_group; 4028 } 4029 tx_ring_count = grp->mrg_cur_count; 4030 ring = grp->mrg_rings; 4031 /* 4032 * An attempt is made to reserve 'tx_ring_count' number 4033 * of Tx rings. If tx_ring_count is 0, default Tx ring 4034 * is used. If it is 1, an attempt is made to reserve one 4035 * Tx ring. In both the cases, the ring information is 4036 * stored in Tx SRS. If multiple Tx rings are specified, 4037 * then each Tx ring will have a Tx-side soft ring. All 4038 * these soft rings will be hang off Tx SRS. 4039 */ 4040 switch (grp->mrg_state) { 4041 case MAC_GROUP_STATE_SHARED: 4042 case MAC_GROUP_STATE_RESERVED: 4043 if (tx_ring_count <= 1 && !is_aggr) { 4044 no_group: 4045 if (ring != NULL && 4046 ring->mr_state != MR_INUSE) { 4047 (void) mac_start_ring(ring); 4048 ring_info = mac_hwring_getinfo( 4049 (mac_ring_handle_t)ring); 4050 } 4051 tx->st_arg2 = (void *)ring; 4052 mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 4053 if (tx_srs->srs_type & SRST_BW_CONTROL) { 4054 tx->st_mode = SRS_TX_BW; 4055 } else if (mac_tx_serialize || 4056 (ring_info & MAC_RING_TX_SERIALIZE)) { 4057 tx->st_mode = SRS_TX_SERIALIZE; 4058 } else { 4059 tx->st_mode = SRS_TX_DEFAULT; 4060 } 4061 break; 4062 } 4063 soft_ring_type = ST_RING_TX; 4064 if (tx_srs->srs_type & SRST_BW_CONTROL) { 4065 tx->st_mode = is_aggr ? 4066 SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 4067 } else { 4068 tx->st_mode = is_aggr ? SRS_TX_AGGR : 4069 SRS_TX_FANOUT; 4070 } 4071 for (i = 0; i < tx_ring_count; i++) { 4072 ASSERT(ring != NULL); 4073 switch (ring->mr_state) { 4074 case MR_INUSE: 4075 case MR_FREE: 4076 ASSERT(ring->mr_srs == NULL); 4077 4078 if (ring->mr_state != MR_INUSE) 4079 (void) mac_start_ring(ring); 4080 ring_info = mac_hwring_getinfo( 4081 (mac_ring_handle_t)ring); 4082 if (mac_tx_serialize || (ring_info & 4083 MAC_RING_TX_SERIALIZE)) { 4084 soft_ring_type |= 4085 ST_RING_WORKER_ONLY; 4086 } 4087 (void) mac_soft_ring_create(i, 0, 4088 soft_ring_type, maxclsyspri, 4089 mcip, tx_srs, -1, NULL, mcip, 4090 (mac_resource_handle_t)ring); 4091 break; 4092 default: 4093 cmn_err(CE_PANIC, 4094 "srs_setup: mcip = %p " 4095 "trying to add UNKNOWN ring = %p\n", 4096 (void *)mcip, (void *)ring); 4097 break; 4098 } 4099 ring = ring->mr_next; 4100 } 4101 mac_srs_update_fanout_list(tx_srs); 4102 break; 4103 default: 4104 ASSERT(B_FALSE); 4105 break; 4106 } 4107 tx->st_func = mac_tx_get_func(tx->st_mode); 4108 if (is_aggr) { 4109 VERIFY(i_mac_capab_get((mac_handle_t)mip, 4110 MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 4111 } 4112 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 4113 int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 4114 } 4115 4116 /* 4117 * Update the fanout of a client if its recorded link speed doesn't match 4118 * its current link speed. 4119 */ 4120 void 4121 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 4122 { 4123 uint64_t link_speed; 4124 mac_resource_props_t *mcip_mrp; 4125 flow_entry_t *flent = mcip->mci_flent; 4126 mac_soft_ring_set_t *rx_srs; 4127 mac_cpus_t *srs_cpu; 4128 int soft_ring_count, maxcpus; 4129 4130 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 4131 4132 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 4133 MAC_STAT_IFSPEED); 4134 4135 if ((link_speed != 0) && 4136 (link_speed != mcip->mci_flent->fe_nic_speed)) { 4137 mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 4138 /* 4139 * Before calling mac_fanout_setup(), check to see if 4140 * the SRSes already have the right number of soft 4141 * rings. mac_fanout_setup() is a heavy duty operation 4142 * where new cpu bindings are done for SRS and soft 4143 * ring threads and interrupts re-targeted. 4144 */ 4145 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 4146 soft_ring_count = mac_compute_soft_ring_count(flent, 4147 flent->fe_rx_srs_cnt - 1, maxcpus); 4148 /* 4149 * If soft_ring_count returned by 4150 * mac_compute_soft_ring_count() is 0, bump it 4151 * up by 1 because we always have atleast one 4152 * TCP, UDP, and OTH soft ring associated with 4153 * an SRS. 4154 */ 4155 soft_ring_count = (soft_ring_count == 0) ? 4156 1 : soft_ring_count; 4157 rx_srs = flent->fe_rx_srs[0]; 4158 srs_cpu = &rx_srs->srs_cpu; 4159 if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 4160 mac_fanout_setup(mcip, flent, mcip_mrp, 4161 mac_rx_deliver, mcip, NULL, cpupart); 4162 } 4163 } 4164 } 4165 4166 /* 4167 * Walk through the list of MAC clients for the MAC. 4168 * For each active MAC client, recompute the number of soft rings 4169 * associated with every client, only if current speed is different 4170 * from the speed that was previously used for soft ring computation. 4171 * If the cable is disconnected whlie the NIC is started, we would get 4172 * notification with speed set to 0. We do not recompute in that case. 4173 */ 4174 void 4175 mac_fanout_recompute(mac_impl_t *mip) 4176 { 4177 mac_client_impl_t *mcip; 4178 cpupart_t *cpupart; 4179 boolean_t use_default; 4180 mac_resource_props_t *mrp, *emrp; 4181 4182 i_mac_perim_enter(mip); 4183 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 4184 mip->mi_linkstate != LINK_STATE_UP) { 4185 i_mac_perim_exit(mip); 4186 return; 4187 } 4188 4189 for (mcip = mip->mi_clients_list; mcip != NULL; 4190 mcip = mcip->mci_client_next) { 4191 /* Aggr port clients don't have SRSes. */ 4192 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 4193 continue; 4194 4195 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 4196 !MCIP_DATAPATH_SETUP(mcip)) 4197 continue; 4198 mrp = MCIP_RESOURCE_PROPS(mcip); 4199 emrp = MCIP_EFFECTIVE_PROPS(mcip); 4200 use_default = B_FALSE; 4201 pool_lock(); 4202 cpupart = mac_pset_find(mrp, &use_default); 4203 mac_fanout_recompute_client(mcip, cpupart); 4204 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 4205 pool_unlock(); 4206 } 4207 4208 i_mac_perim_exit(mip); 4209 } 4210 4211 /* 4212 * Given a MAC, change the polling state for all its MAC clients. 'enable' is 4213 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 4214 * default. 4215 */ 4216 void 4217 mac_poll_state_change(mac_handle_t mh, boolean_t enable) 4218 { 4219 mac_impl_t *mip = (mac_impl_t *)mh; 4220 mac_client_impl_t *mcip; 4221 4222 i_mac_perim_enter(mip); 4223 if (enable) 4224 mip->mi_state_flags &= ~MIS_POLL_DISABLE; 4225 else 4226 mip->mi_state_flags |= MIS_POLL_DISABLE; 4227 for (mcip = mip->mi_clients_list; mcip != NULL; 4228 mcip = mcip->mci_client_next) 4229 mac_client_update_classifier(mcip, B_TRUE); 4230 i_mac_perim_exit(mip); 4231 } 4232