1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2018 Joyent, Inc. 24 * Copyright 2020 RackTop Systems. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/callb.h> 29 #include <sys/cpupart.h> 30 #include <sys/pool.h> 31 #include <sys/pool_pset.h> 32 #include <sys/sdt.h> 33 #include <sys/strsubr.h> 34 #include <sys/strsun.h> 35 #include <sys/vlan.h> 36 #include <inet/ipsec_impl.h> 37 #include <inet/ip_impl.h> 38 #include <inet/sadb.h> 39 #include <inet/ipsecesp.h> 40 #include <inet/ipsecah.h> 41 42 #include <sys/mac_impl.h> 43 #include <sys/mac_client_impl.h> 44 #include <sys/mac_client_priv.h> 45 #include <sys/mac_soft_ring.h> 46 #include <sys/mac_flow_impl.h> 47 #include <sys/mac_stat.h> 48 49 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 50 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 51 static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 52 static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 53 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 54 55 static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 56 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 57 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 58 static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 59 static void mac_srs_add_glist(mac_soft_ring_set_t *); 60 static void mac_srs_remove_glist(mac_soft_ring_set_t *); 61 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 62 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 63 64 static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 65 static void mac_walk_srs_and_bind(int); 66 static void mac_walk_srs_and_unbind(int); 67 68 extern boolean_t mac_latency_optimize; 69 70 static kmem_cache_t *mac_srs_cache; 71 kmem_cache_t *mac_soft_ring_cache; 72 73 /* 74 * The duration in msec we wait before signalling the soft ring 75 * worker thread in case packets get queued. 76 */ 77 uint32_t mac_soft_ring_worker_wait = 0; 78 79 /* 80 * A global tunable for turning polling on/off. By default, dynamic 81 * polling is always on and is always very beneficial. It should be 82 * turned off with absolute care and for the rare workload (very 83 * low latency sensitive traffic). 84 */ 85 int mac_poll_enable = B_TRUE; 86 87 /* 88 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 89 * Large values could end up in consuming lot of system memory and cause 90 * system hang. 91 */ 92 int mac_soft_ring_max_q_cnt = 1024; 93 int mac_soft_ring_min_q_cnt = 256; 94 int mac_soft_ring_poll_thres = 16; 95 96 boolean_t mac_tx_serialize = B_FALSE; 97 98 /* 99 * mac_tx_srs_hiwat is the queue depth threshold at which callers of 100 * mac_tx() will be notified of flow control condition. 101 * 102 * TCP does not honour flow control condition sent up by mac_tx(). 103 * Thus provision is made for TCP to allow more packets to be queued 104 * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 105 * 106 * Note that mac_tx_srs_hiwat is always be lesser than 107 * mac_tx_srs_max_q_cnt. 108 */ 109 uint32_t mac_tx_srs_max_q_cnt = 100000; 110 uint32_t mac_tx_srs_hiwat = 1000; 111 112 /* 113 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 114 * 115 * Global tunables that determines the number of soft rings to be used for 116 * fanning out incoming traffic on a link. These count will be used only 117 * when no explicit set of CPUs was assigned to the data-links. 118 * 119 * mac_rx_soft_ring_count tunable will come into effect only if 120 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 121 * default only for sun4v platforms. 122 * 123 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 124 * 10Gbps link and is not dependent upon mac_soft_ring_enable. 125 * 126 * The number of soft rings for fanout for a link or a flow is determined 127 * by mac_compute_soft_ring_count() routine. This routine will take into 128 * account mac_soft_ring_enable, mac_rx_soft_ring_count and 129 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 130 * 131 * If a bandwidth is specified, the determination of the number of soft 132 * rings is based on specified bandwidth, CPU speed and number of CPUs in 133 * the system. 134 */ 135 uint_t mac_rx_soft_ring_count = 8; 136 uint_t mac_rx_soft_ring_10gig_count = 8; 137 138 /* 139 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 140 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 141 * list is used to walk the list of all MAC threads when a CPU is 142 * coming online or going offline. 143 */ 144 static mac_soft_ring_set_t *mac_srs_g_list = NULL; 145 static krwlock_t mac_srs_g_lock; 146 147 /* 148 * Whether the SRS threads should be bound, or not. 149 */ 150 boolean_t mac_srs_thread_bind = B_TRUE; 151 152 /* 153 * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 154 * dladm command would override this. 155 */ 156 boolean_t mac_tx_intr_retarget = B_FALSE; 157 boolean_t mac_rx_intr_retarget = B_FALSE; 158 159 /* 160 * If cpu bindings are specified by user, then Tx SRS and its soft 161 * rings should also be bound to the CPUs specified by user. The 162 * CPUs for Tx bindings are at the end of the cpu list provided by 163 * the user. If enough CPUs are not available (for Tx and Rx 164 * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 165 */ 166 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 167 processorid_t cpuid; \ 168 int i; \ 169 mac_soft_ring_t *softring; \ 170 mac_cpus_t *srs_cpu; \ 171 \ 172 srs_cpu = &mac_tx_srs->srs_cpu; \ 173 cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 174 mac_srs_worker_bind(mac_tx_srs, cpuid); \ 175 if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 176 for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 177 cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 178 softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 179 if (cpuid != -1) { \ 180 (void) mac_soft_ring_bind(softring, \ 181 cpuid); \ 182 } \ 183 } \ 184 } \ 185 } 186 187 /* 188 * Re-targeting is allowed only for exclusive group or for primary. 189 */ 190 #define RETARGETABLE_CLIENT(group, mcip) \ 191 ((((group) != NULL) && \ 192 ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 193 mac_is_primary_client(mcip)) 194 195 #define MAC_RING_RETARGETABLE(ring) \ 196 (((ring) != NULL) && \ 197 ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 198 !((ring)->mr_info.mri_intr.mi_ddi_shared)) 199 200 201 /* INIT and FINI ROUTINES */ 202 203 void 204 mac_soft_ring_init(void) 205 { 206 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 207 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 208 209 mac_srs_cache = kmem_cache_create("mac_srs_cache", 210 sizeof (mac_soft_ring_set_t), 211 64, NULL, NULL, NULL, NULL, NULL, 0); 212 213 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 214 mutex_enter(&cpu_lock); 215 register_cpu_setup_func(mac_srs_cpu_setup, NULL); 216 mutex_exit(&cpu_lock); 217 } 218 219 void 220 mac_soft_ring_finish(void) 221 { 222 mutex_enter(&cpu_lock); 223 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 224 mutex_exit(&cpu_lock); 225 rw_destroy(&mac_srs_g_lock); 226 kmem_cache_destroy(mac_soft_ring_cache); 227 kmem_cache_destroy(mac_srs_cache); 228 } 229 230 static void 231 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 232 { 233 mac_soft_ring_t *softring, *next, *head; 234 235 /* 236 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 237 * DR. The callbacks from DR are called with cpu_lock held, and hence 238 * can't wait to grab the mac perimeter. The soft ring list is hence 239 * protected for read access by srs_lock. Changing the soft ring list 240 * needs the mac perimeter and the srs_lock. 241 */ 242 mutex_enter(&mac_srs->srs_lock); 243 244 head = mac_srs->srs_soft_ring_head; 245 mac_srs->srs_soft_ring_head = NULL; 246 mac_srs->srs_soft_ring_tail = NULL; 247 mac_srs->srs_soft_ring_count = 0; 248 249 mutex_exit(&mac_srs->srs_lock); 250 251 for (softring = head; softring != NULL; softring = next) { 252 next = softring->s_ring_next; 253 mac_soft_ring_free(softring); 254 } 255 } 256 257 static void 258 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 259 { 260 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 261 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 262 263 rw_enter(&mac_srs_g_lock, RW_WRITER); 264 mutex_enter(&mac_srs->srs_lock); 265 266 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 267 268 if (mac_srs_g_list == NULL) { 269 mac_srs_g_list = mac_srs; 270 } else { 271 mac_srs->srs_next = mac_srs_g_list; 272 mac_srs_g_list->srs_prev = mac_srs; 273 mac_srs->srs_prev = NULL; 274 mac_srs_g_list = mac_srs; 275 } 276 mac_srs->srs_state |= SRS_IN_GLIST; 277 278 mutex_exit(&mac_srs->srs_lock); 279 rw_exit(&mac_srs_g_lock); 280 } 281 282 static void 283 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 284 { 285 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 286 287 rw_enter(&mac_srs_g_lock, RW_WRITER); 288 mutex_enter(&mac_srs->srs_lock); 289 290 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 291 292 if (mac_srs == mac_srs_g_list) { 293 mac_srs_g_list = mac_srs->srs_next; 294 if (mac_srs_g_list != NULL) 295 mac_srs_g_list->srs_prev = NULL; 296 } else { 297 mac_srs->srs_prev->srs_next = mac_srs->srs_next; 298 if (mac_srs->srs_next != NULL) 299 mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 300 } 301 mac_srs->srs_state &= ~SRS_IN_GLIST; 302 303 mutex_exit(&mac_srs->srs_lock); 304 rw_exit(&mac_srs_g_lock); 305 } 306 307 /* POLLING SETUP AND TEAR DOWN ROUTINES */ 308 309 /* 310 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 311 * 312 * These routines are used to call back into the upper layer 313 * (primarily TCP squeue) to stop polling the soft rings or 314 * restart polling. 315 */ 316 void 317 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 318 mac_soft_ring_set_t *mac_srs) 319 { 320 mac_soft_ring_t *softring; 321 322 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 323 324 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 325 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 326 return; 327 } 328 329 for (softring = mac_srs->srs_soft_ring_head; 330 softring != NULL; softring = softring->s_ring_next) { 331 if ((softring->s_ring_type & ST_RING_TCP) && 332 (softring->s_ring_rx_arg2 != NULL)) { 333 mcip->mci_resource_quiesce(mcip->mci_resource_arg, 334 softring->s_ring_rx_arg2); 335 } 336 } 337 } 338 339 void 340 mac_srs_client_poll_restart(mac_client_impl_t *mcip, 341 mac_soft_ring_set_t *mac_srs) 342 { 343 mac_soft_ring_t *softring; 344 345 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 346 347 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 348 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 349 return; 350 } 351 352 for (softring = mac_srs->srs_soft_ring_head; 353 softring != NULL; softring = softring->s_ring_next) { 354 if ((softring->s_ring_type & ST_RING_TCP) && 355 (softring->s_ring_rx_arg2 != NULL)) { 356 mcip->mci_resource_restart(mcip->mci_resource_arg, 357 softring->s_ring_rx_arg2); 358 } 359 } 360 } 361 362 /* 363 * Register the given SRS and associated soft rings with the consumer and 364 * enable the polling interface used by the consumer.(i.e IP) over this 365 * SRS and associated soft rings. 366 */ 367 void 368 mac_srs_client_poll_enable(mac_client_impl_t *mcip, 369 mac_soft_ring_set_t *mac_srs) 370 { 371 mac_rx_fifo_t mrf; 372 mac_soft_ring_t *softring; 373 374 ASSERT(mac_srs->srs_mcip == mcip); 375 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 376 377 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 378 return; 379 380 bzero(&mrf, sizeof (mac_rx_fifo_t)); 381 mrf.mrf_type = MAC_RX_FIFO; 382 383 /* 384 * A SRS is capable of acting as a soft ring for cases 385 * where no fanout is needed. This is the case for userland 386 * flows. 387 */ 388 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 389 return; 390 391 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 392 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 393 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 394 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 395 396 softring = mac_srs->srs_soft_ring_head; 397 while (softring != NULL) { 398 if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 399 /* 400 * TCP and UDP support DLS bypass. Squeue polling 401 * support implies DLS bypass since the squeue poll 402 * path does not have DLS processing. 403 */ 404 mac_soft_ring_dls_bypass(softring, 405 mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 406 } 407 /* 408 * Non-TCP protocols don't support squeues. Hence we don't 409 * make any ring addition callbacks for non-TCP rings 410 */ 411 if (!(softring->s_ring_type & ST_RING_TCP)) { 412 softring->s_ring_rx_arg2 = NULL; 413 softring = softring->s_ring_next; 414 continue; 415 } 416 mrf.mrf_rx_arg = softring; 417 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 418 mrf.mrf_cpu_id = softring->s_ring_cpuid; 419 mrf.mrf_flow_priority = mac_srs->srs_pri; 420 421 softring->s_ring_rx_arg2 = mcip->mci_resource_add( 422 mcip->mci_resource_arg, (mac_resource_t *)&mrf); 423 424 softring = softring->s_ring_next; 425 } 426 } 427 428 /* 429 * Unregister the given SRS and associated soft rings with the consumer and 430 * disable the polling interface used by the consumer.(i.e IP) over this 431 * SRS and associated soft rings. 432 */ 433 void 434 mac_srs_client_poll_disable(mac_client_impl_t *mcip, 435 mac_soft_ring_set_t *mac_srs) 436 { 437 mac_soft_ring_t *softring; 438 439 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 440 441 /* 442 * A SRS is capable of acting as a soft ring for cases 443 * where no protocol fanout is needed. This is the case 444 * for userland flows. Nothing to do here. 445 */ 446 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 447 return; 448 449 mutex_enter(&mac_srs->srs_lock); 450 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 451 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 452 mutex_exit(&mac_srs->srs_lock); 453 return; 454 } 455 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 456 mutex_exit(&mac_srs->srs_lock); 457 458 /* 459 * DLS bypass is now disabled in the case of both TCP and UDP. 460 * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 461 * callback. In addition, in the case of TCP, invoke IP's callback 462 * for ring removal. 463 */ 464 for (softring = mac_srs->srs_soft_ring_head; 465 softring != NULL; softring = softring->s_ring_next) { 466 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 467 continue; 468 469 if ((softring->s_ring_type & ST_RING_TCP) && 470 softring->s_ring_rx_arg2 != NULL) { 471 mcip->mci_resource_remove(mcip->mci_resource_arg, 472 softring->s_ring_rx_arg2); 473 } 474 475 mutex_enter(&softring->s_ring_lock); 476 while (softring->s_ring_state & S_RING_PROC) { 477 softring->s_ring_state |= S_RING_CLIENT_WAIT; 478 cv_wait(&softring->s_ring_client_cv, 479 &softring->s_ring_lock); 480 } 481 softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 482 softring->s_ring_rx_arg2 = NULL; 483 softring->s_ring_rx_func = mac_rx_deliver; 484 softring->s_ring_rx_arg1 = mcip; 485 mutex_exit(&softring->s_ring_lock); 486 } 487 } 488 489 /* 490 * Enable or disable poll capability of the SRS on the underlying Rx ring. 491 * 492 * There is a need to enable or disable the poll capability of an SRS over an 493 * Rx ring depending on the number of mac clients sharing the ring and also 494 * whether user flows are configured on it. However the poll state is actively 495 * manipulated by the SRS worker and poll threads and uncoordinated changes by 496 * yet another thread to the underlying capability can surprise them leading 497 * to assert failures. Instead we quiesce the SRS, make the changes and then 498 * restart the SRS. 499 */ 500 static void 501 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 502 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 503 { 504 boolean_t need_restart = B_FALSE; 505 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 506 mac_ring_t *ring; 507 508 if (!SRS_QUIESCED(mac_srs)) { 509 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 510 need_restart = B_TRUE; 511 } 512 513 ring = mac_srs->srs_ring; 514 if ((ring != NULL) && 515 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 516 if (turn_off_poll_capab) 517 mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 518 else if (mac_poll_enable) 519 mac_srs->srs_state |= SRS_POLLING_CAPAB; 520 } 521 srs_rx->sr_lower_proc = rx_func; 522 523 if (need_restart) 524 mac_rx_srs_restart(mac_srs); 525 } 526 527 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 528 529 /* 530 * Return the next CPU to be used to bind a MAC kernel thread. 531 * If a cpupart is specified, the cpu chosen must be from that 532 * cpu partition. 533 */ 534 static processorid_t 535 mac_next_bind_cpu(cpupart_t *cpupart) 536 { 537 static cpu_t *cp = NULL; 538 cpu_t *cp_start; 539 540 ASSERT(MUTEX_HELD(&cpu_lock)); 541 542 if (cp == NULL) 543 cp = cpu_list; 544 545 cp = cp->cpu_next_onln; 546 cp_start = cp; 547 548 do { 549 if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 550 return (cp->cpu_id); 551 552 } while ((cp = cp->cpu_next_onln) != cp_start); 553 554 return (-1); /* No matching CPU found online */ 555 } 556 557 /* ARGSUSED */ 558 static int 559 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 560 { 561 ASSERT(MUTEX_HELD(&cpu_lock)); 562 switch (what) { 563 case CPU_CONFIG: 564 case CPU_ON: 565 case CPU_CPUPART_IN: 566 mac_walk_srs_and_bind(id); 567 break; 568 569 case CPU_UNCONFIG: 570 case CPU_OFF: 571 case CPU_CPUPART_OUT: 572 mac_walk_srs_and_unbind(id); 573 break; 574 575 default: 576 break; 577 } 578 return (0); 579 } 580 581 /* 582 * mac_compute_soft_ring_count(): 583 * 584 * This routine computes the number of soft rings needed to handle incoming 585 * load given a flow_entry. 586 * 587 * The routine does the following: 588 * 1) soft rings will be created if mac_soft_ring_enable is set. 589 * 2) If the underlying link is a 10Gbps link, then soft rings will be 590 * created even if mac_soft_ring_enable is not set. The number of soft 591 * rings, so created, will equal mac_rx_soft_ring_10gig_count. 592 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 593 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 594 * 10Gbps link. 595 * 596 * If a bandwidth limit is specified, the number that gets computed is 597 * dependent upon CPU speed, the number of Rx rings configured, and 598 * the bandwidth limit. 599 * If more Rx rings are available, less number of soft rings is needed. 600 * 601 * mac_use_bw_heuristic is another "hidden" variable that can be used to 602 * override the default use of soft ring count computation. Depending upon 603 * the usefulness of it, mac_use_bw_heuristic can later be made into a 604 * data-link property or removed altogether. 605 * 606 * TODO: Cleanup and tighten some of the assumptions. 607 */ 608 boolean_t mac_check_overlay = B_TRUE; 609 boolean_t mac_use_bw_heuristic = B_TRUE; 610 static int 611 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 612 { 613 uint64_t cpu_speed, bw = 0; 614 int srings = 0; 615 boolean_t bw_enabled = B_FALSE; 616 mac_client_impl_t *mcip = flent->fe_mcip; 617 618 ASSERT(!(flent->fe_type & FLOW_USER)); 619 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 620 mac_use_bw_heuristic) { 621 /* bandwidth enabled */ 622 bw_enabled = B_TRUE; 623 bw = flent->fe_resource_props.mrp_maxbw; 624 } 625 if (!bw_enabled) { 626 /* No bandwidth enabled */ 627 if (mac_soft_ring_enable) 628 srings = mac_rx_soft_ring_count; 629 630 /* Is this a 10Gig link? */ 631 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 632 MAC_STAT_IFSPEED); 633 /* convert to Mbps */ 634 if (((flent->fe_nic_speed)/1000000) > 1000 && 635 mac_rx_soft_ring_10gig_count > 0) { 636 /* This is a 10Gig link */ 637 srings = mac_rx_soft_ring_10gig_count; 638 /* 639 * Use 2 times mac_rx_soft_ring_10gig_count for 640 * sun4v systems. 641 */ 642 if (mac_soft_ring_enable) 643 srings = srings * 2; 644 } else if (mac_check_overlay == B_TRUE && 645 (mcip->mci_state_flags & MCIS_IS_VNIC) != 0) { 646 /* Is this a VNIC on an overlay? */ 647 mac_handle_t mh = (mac_handle_t)mcip->mci_mip; 648 if (mac_is_overlay(mh) == B_TRUE) { 649 srings = mac_rx_soft_ring_10gig_count; 650 } 651 } 652 653 654 } else { 655 /* 656 * Soft ring computation using CPU speed and specified 657 * bandwidth limit. 658 */ 659 /* Assumption: all CPUs have the same frequency */ 660 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 661 662 /* cpu_speed is in MHz; make bw in units of Mbps. */ 663 bw = bw/1000000; 664 665 if (bw >= 1000) { 666 /* 667 * bw is greater than or equal to 1Gbps. 668 * The number of soft rings required is a function 669 * of bandwidth and CPU speed. To keep this simple, 670 * let's use this rule: 1GHz CPU can handle 1Gbps. 671 * If bw is less than 1 Gbps, then there is no need 672 * for soft rings. Assumption is that CPU speeds 673 * (on modern systems) are at least 1GHz. 674 */ 675 srings = bw/cpu_speed; 676 if (srings <= 1 && mac_soft_ring_enable) { 677 /* 678 * Give at least 2 soft rings 679 * for sun4v systems 680 */ 681 srings = 2; 682 } 683 } 684 } 685 /* 686 * If the flent has multiple Rx SRSs, then each SRS need not 687 * have that many soft rings on top of it. The number of 688 * soft rings for each Rx SRS is found by dividing srings by 689 * rx_srs_cnt. 690 */ 691 if (rx_srs_cnt > 1) { 692 int remainder; 693 694 remainder = srings%rx_srs_cnt; 695 srings = srings/rx_srs_cnt; 696 if (remainder != 0) 697 srings++; 698 /* 699 * Fanning out to 1 soft ring is not very useful. 700 * Set it as well to 0 and mac_srs_fanout_init() 701 * will take care of creating a single soft ring 702 * for proto fanout. 703 */ 704 if (srings == 1) 705 srings = 0; 706 } 707 /* Do some more massaging */ 708 srings = min(srings, maxcpus); 709 srings = min(srings, MAX_SR_FANOUT); 710 return (srings); 711 } 712 713 /* 714 * mac_tx_cpu_init: 715 * set up CPUs for Tx interrupt re-targeting and Tx worker 716 * thread binding 717 */ 718 static void 719 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 720 cpupart_t *cpupart) 721 { 722 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 723 mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 724 mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 725 mac_soft_ring_t *sringp; 726 mac_ring_t *ring; 727 processorid_t worker_cpuid; 728 boolean_t retargetable_client = B_FALSE; 729 int i, j; 730 731 if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 732 flent->fe_mcip)) { 733 retargetable_client = B_TRUE; 734 } 735 736 if (MAC_TX_SOFT_RINGS(tx_srs)) { 737 if (mrp != NULL) 738 j = mrp->mrp_ncpus - 1; 739 for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 740 if (mrp != NULL) { 741 if (j < 0) 742 j = mrp->mrp_ncpus - 1; 743 worker_cpuid = mrp->mrp_cpu[j]; 744 } else { 745 /* 746 * Bind interrupt to the next CPU available 747 * and leave the worker unbound. 748 */ 749 worker_cpuid = -1; 750 } 751 sringp = tx_srs->srs_tx_soft_rings[i]; 752 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 753 srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 754 if (MAC_RING_RETARGETABLE(ring) && 755 retargetable_client) { 756 mutex_enter(&cpu_lock); 757 srs_cpu->mc_tx_intr_cpu[i] = 758 (mrp != NULL) ? mrp->mrp_cpu[j] : 759 (mac_tx_intr_retarget ? 760 mac_next_bind_cpu(cpupart) : -1); 761 mutex_exit(&cpu_lock); 762 } else { 763 srs_cpu->mc_tx_intr_cpu[i] = -1; 764 } 765 if (mrp != NULL) 766 j--; 767 } 768 } else { 769 /* Tx mac_ring_handle_t is stored in st_arg2 */ 770 srs_cpu->mc_tx_fanout_cpus[0] = 771 (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 772 ring = (mac_ring_t *)srs_tx->st_arg2; 773 if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 774 mutex_enter(&cpu_lock); 775 srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 776 mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 777 (mac_tx_intr_retarget ? 778 mac_next_bind_cpu(cpupart) : -1); 779 mutex_exit(&cpu_lock); 780 } else { 781 srs_cpu->mc_tx_intr_cpu[0] = -1; 782 } 783 } 784 } 785 786 /* 787 * Assignment of user specified CPUs to a link. 788 * 789 * Minimum CPUs required to get an optimal assignmet: 790 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 791 * flag is set -- one for polling, one for fanout soft ring. 792 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 793 * for polling, one for SRS worker thread and one for fanout soft ring. 794 * 795 * The CPUs needed for Tx side is equal to the number of Tx rings 796 * the link is using. 797 * 798 * mac_flow_user_cpu_init() categorizes the CPU assignment depending 799 * upon the number of CPUs in 3 different buckets. 800 * 801 * In the first bucket, the most optimal case is handled. The user has 802 * passed enough number of CPUs and every thread gets its own CPU. 803 * 804 * The second and third are the sub-optimal cases. Enough CPUs are not 805 * available. 806 * 807 * The second bucket handles the case where atleast one distinct CPU is 808 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 809 * SRS or soft rings). 810 * 811 * In the third case (worst case scenario), specified CPU count is less 812 * than the Rx rings configured for the link. In this case, we round 813 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 814 */ 815 static void 816 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 817 { 818 mac_soft_ring_set_t *rx_srs, *tx_srs; 819 int i, srs_cnt; 820 mac_cpus_t *srs_cpu; 821 int no_of_cpus, cpu_cnt; 822 int rx_srs_cnt, reqd_rx_cpu_cnt; 823 int fanout_cpu_cnt, reqd_tx_cpu_cnt; 824 int reqd_poll_worker_cnt, fanout_cnt_per_srs; 825 mac_resource_props_t *emrp = &flent->fe_effective_props; 826 827 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 828 /* 829 * The check for nbc_ncpus to be within limits for 830 * the user specified case was done earlier and if 831 * not within limits, an error would have been 832 * returned to the user. 833 */ 834 ASSERT(mrp->mrp_ncpus > 0); 835 836 no_of_cpus = mrp->mrp_ncpus; 837 838 if (mrp->mrp_rx_intr_cpu != -1) { 839 /* 840 * interrupt has been re-targetted. Poll 841 * thread needs to be bound to interrupt 842 * CPU. 843 * 844 * Find where in the list is the intr 845 * CPU and swap it with the first one. 846 * We will be using the first CPU in the 847 * list for poll. 848 */ 849 for (i = 0; i < no_of_cpus; i++) { 850 if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 851 break; 852 } 853 mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 854 mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 855 } 856 857 /* 858 * Requirements: 859 * The number of CPUs that each Rx ring needs is dependent 860 * upon mac_latency_optimize flag. 861 * 1) If set, atleast 2 CPUs are needed -- one for 862 * polling, one for fanout soft ring. 863 * 2) If not set, then atleast 3 CPUs are needed -- one 864 * for polling, one for srs worker thread, and one for 865 * fanout soft ring. 866 */ 867 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 868 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 869 reqd_rx_cpu_cnt = mac_latency_optimize ? 870 (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 871 872 /* How many CPUs are needed for Tx side? */ 873 tx_srs = flent->fe_tx_srs; 874 reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 875 tx_srs->srs_tx_ring_count : 1; 876 877 /* CPUs needed for Rx SRSes poll and worker threads */ 878 reqd_poll_worker_cnt = mac_latency_optimize ? 879 rx_srs_cnt : rx_srs_cnt * 2; 880 881 /* Has the user provided enough CPUs? */ 882 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 883 /* 884 * Best case scenario. There is enough CPUs. All 885 * Rx rings will get their own set of CPUs plus 886 * Tx soft rings will get their own. 887 */ 888 /* 889 * fanout_cpu_cnt is the number of CPUs available 890 * for Rx side fanout soft rings. 891 */ 892 fanout_cpu_cnt = no_of_cpus - 893 reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 894 895 /* 896 * Divide fanout_cpu_cnt by rx_srs_cnt to find 897 * out how many fanout soft rings each Rx SRS 898 * can have. 899 */ 900 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 901 902 /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 903 fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 904 905 /* Do the assignment for the default Rx ring */ 906 cpu_cnt = 0; 907 rx_srs = flent->fe_rx_srs[0]; 908 ASSERT(rx_srs->srs_ring == NULL); 909 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 910 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 911 srs_cpu = &rx_srs->srs_cpu; 912 srs_cpu->mc_ncpus = no_of_cpus; 913 bcopy(mrp->mrp_cpu, 914 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 915 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 916 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 917 /* Retarget the interrupt to the same CPU as the poll */ 918 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 919 srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 920 srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 921 for (i = 0; i < fanout_cnt_per_srs; i++) 922 srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 923 924 /* Do the assignment for h/w Rx SRSes */ 925 if (flent->fe_rx_srs_cnt > 1) { 926 cpu_cnt = 0; 927 for (srs_cnt = 1; 928 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 929 rx_srs = flent->fe_rx_srs[srs_cnt]; 930 ASSERT(rx_srs->srs_ring != NULL); 931 if (rx_srs->srs_fanout_state == 932 SRS_FANOUT_INIT) { 933 rx_srs->srs_fanout_state = 934 SRS_FANOUT_REINIT; 935 } 936 srs_cpu = &rx_srs->srs_cpu; 937 srs_cpu->mc_ncpus = no_of_cpus; 938 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 939 sizeof (srs_cpu->mc_cpus)); 940 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 941 /* The first CPU in the list is the intr CPU */ 942 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 943 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 944 srs_cpu->mc_rx_workerid = 945 (mac_latency_optimize ? 946 srs_cpu->mc_rx_pollid : 947 mrp->mrp_cpu[cpu_cnt++]); 948 for (i = 0; i < fanout_cnt_per_srs; i++) { 949 srs_cpu->mc_rx_fanout_cpus[i] = 950 mrp->mrp_cpu[cpu_cnt++]; 951 } 952 ASSERT(cpu_cnt <= no_of_cpus); 953 } 954 } 955 goto tx_cpu_init; 956 } 957 958 /* 959 * Sub-optimal case. 960 * We have the following information: 961 * no_of_cpus - no. of cpus that user passed. 962 * rx_srs_cnt - no. of rx rings. 963 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 964 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 965 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 966 */ 967 /* 968 * If we bind the Rx fanout soft rings to the same CPUs 969 * as poll/worker, would that be enough? 970 */ 971 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 972 boolean_t worker_assign = B_FALSE; 973 974 /* 975 * If mac_latency_optimize is not set, are there 976 * enough CPUs to assign a CPU for worker also? 977 */ 978 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 979 worker_assign = B_TRUE; 980 /* 981 * Zero'th Rx SRS is the default Rx ring. It is not 982 * associated with h/w Rx ring. 983 */ 984 rx_srs = flent->fe_rx_srs[0]; 985 ASSERT(rx_srs->srs_ring == NULL); 986 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 987 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 988 cpu_cnt = 0; 989 srs_cpu = &rx_srs->srs_cpu; 990 srs_cpu->mc_ncpus = no_of_cpus; 991 bcopy(mrp->mrp_cpu, 992 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 993 srs_cpu->mc_rx_fanout_cnt = 1; 994 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 995 /* Retarget the interrupt to the same CPU as the poll */ 996 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 997 srs_cpu->mc_rx_workerid = 998 ((!mac_latency_optimize && worker_assign) ? 999 mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 1000 1001 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1002 1003 /* Do CPU bindings for SRSes having h/w Rx rings */ 1004 if (flent->fe_rx_srs_cnt > 1) { 1005 cpu_cnt = 0; 1006 for (srs_cnt = 1; 1007 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1008 rx_srs = flent->fe_rx_srs[srs_cnt]; 1009 ASSERT(rx_srs->srs_ring != NULL); 1010 if (rx_srs->srs_fanout_state == 1011 SRS_FANOUT_INIT) { 1012 rx_srs->srs_fanout_state = 1013 SRS_FANOUT_REINIT; 1014 } 1015 srs_cpu = &rx_srs->srs_cpu; 1016 srs_cpu->mc_ncpus = no_of_cpus; 1017 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1018 sizeof (srs_cpu->mc_cpus)); 1019 srs_cpu->mc_rx_pollid = 1020 mrp->mrp_cpu[cpu_cnt]; 1021 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1022 srs_cpu->mc_rx_workerid = 1023 ((!mac_latency_optimize && worker_assign) ? 1024 mrp->mrp_cpu[++cpu_cnt] : 1025 srs_cpu->mc_rx_pollid); 1026 srs_cpu->mc_rx_fanout_cnt = 1; 1027 srs_cpu->mc_rx_fanout_cpus[0] = 1028 mrp->mrp_cpu[cpu_cnt]; 1029 cpu_cnt++; 1030 ASSERT(cpu_cnt <= no_of_cpus); 1031 } 1032 } 1033 goto tx_cpu_init; 1034 } 1035 1036 /* 1037 * Real sub-optimal case. Not enough CPUs for poll and 1038 * Tx soft rings. Do a round robin assignment where 1039 * each Rx SRS will get the same CPU for poll, worker 1040 * and fanout soft ring. 1041 */ 1042 cpu_cnt = 0; 1043 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1044 rx_srs = flent->fe_rx_srs[srs_cnt]; 1045 srs_cpu = &rx_srs->srs_cpu; 1046 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1047 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1048 srs_cpu->mc_ncpus = no_of_cpus; 1049 bcopy(mrp->mrp_cpu, 1050 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 1051 srs_cpu->mc_rx_fanout_cnt = 1; 1052 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 1053 /* Retarget the interrupt to the same CPU as the poll */ 1054 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1055 srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 1056 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1057 if (++cpu_cnt >= no_of_cpus) 1058 cpu_cnt = 0; 1059 } 1060 1061 tx_cpu_init: 1062 mac_tx_cpu_init(flent, mrp, NULL); 1063 1064 /* 1065 * Copy the user specified CPUs to the effective CPUs 1066 */ 1067 for (i = 0; i < mrp->mrp_ncpus; i++) { 1068 emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 1069 } 1070 emrp->mrp_ncpus = mrp->mrp_ncpus; 1071 emrp->mrp_mask = mrp->mrp_mask; 1072 bzero(emrp->mrp_pool, MAXPATHLEN); 1073 } 1074 1075 /* 1076 * mac_flow_cpu_init(): 1077 * 1078 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1079 * the CPU binding information in srs_cpu for all Rx SRSes associated 1080 * with a flent. 1081 */ 1082 static void 1083 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1084 { 1085 mac_soft_ring_set_t *rx_srs; 1086 processorid_t cpuid; 1087 int i, j, k, srs_cnt, maxcpus, soft_ring_cnt = 0; 1088 mac_cpus_t *srs_cpu; 1089 mac_resource_props_t *emrp = &flent->fe_effective_props; 1090 1091 /* 1092 * The maximum number of CPUs available can either be 1093 * the number of CPUs in the pool or the number of CPUs 1094 * in the system. 1095 */ 1096 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 1097 /* 1098 * We cannot exceed the hard limit imposed by data structures. 1099 * Leave space for polling CPU and the SRS worker thread when 1100 * "mac_latency_optimize" is not set. 1101 */ 1102 maxcpus = MIN(maxcpus, MRP_NCPUS - 2); 1103 1104 /* 1105 * Compute the number of soft rings needed on top for each Rx 1106 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1107 * associated with h/w Rx rings. Soft ring count needed for 1108 * each h/w Rx SRS is computed and the same is applied to 1109 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1110 * is the software classified Rx SRS. 1111 */ 1112 soft_ring_cnt = mac_compute_soft_ring_count(flent, 1113 flent->fe_rx_srs_cnt - 1, maxcpus); 1114 if (soft_ring_cnt == 0) { 1115 /* 1116 * Even when soft_ring_cnt is 0, we still need 1117 * to create a soft ring for TCP, UDP and 1118 * OTHER. So set it to 1. 1119 */ 1120 soft_ring_cnt = 1; 1121 } 1122 1123 emrp->mrp_ncpus = 0; 1124 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt && 1125 emrp->mrp_ncpus < MRP_NCPUS; srs_cnt++) { 1126 rx_srs = flent->fe_rx_srs[srs_cnt]; 1127 srs_cpu = &rx_srs->srs_cpu; 1128 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1129 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1130 srs_cpu->mc_ncpus = soft_ring_cnt; 1131 srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1132 mutex_enter(&cpu_lock); 1133 for (j = 0; j < soft_ring_cnt; j++) { 1134 cpuid = mac_next_bind_cpu(cpupart); 1135 srs_cpu->mc_cpus[j] = cpuid; 1136 srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1137 } 1138 cpuid = mac_next_bind_cpu(cpupart); 1139 srs_cpu->mc_rx_pollid = cpuid; 1140 srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 1141 srs_cpu->mc_rx_pollid : -1); 1142 /* increment ncpus to account for polling cpu */ 1143 srs_cpu->mc_ncpus++; 1144 srs_cpu->mc_cpus[j++] = cpuid; 1145 if (!mac_latency_optimize) { 1146 cpuid = mac_next_bind_cpu(cpupart); 1147 srs_cpu->mc_ncpus++; 1148 srs_cpu->mc_cpus[j++] = cpuid; 1149 } 1150 srs_cpu->mc_rx_workerid = cpuid; 1151 mutex_exit(&cpu_lock); 1152 1153 /* 1154 * Copy fanout CPUs to fe_effective_props without duplicates. 1155 */ 1156 for (i = 0; i < srs_cpu->mc_ncpus && 1157 emrp->mrp_ncpus < MRP_NCPUS; i++) { 1158 for (j = 0; j < emrp->mrp_ncpus; j++) { 1159 if (emrp->mrp_cpu[j] == srs_cpu->mc_cpus[i]) 1160 break; 1161 } 1162 if (j == emrp->mrp_ncpus) { 1163 emrp->mrp_cpu[emrp->mrp_ncpus++] = 1164 srs_cpu->mc_cpus[i]; 1165 } 1166 } 1167 } 1168 1169 mac_tx_cpu_init(flent, NULL, cpupart); 1170 } 1171 1172 /* 1173 * DATAPATH SETUP ROUTINES 1174 * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1175 */ 1176 1177 /* 1178 * mac_srs_fanout_list_alloc: 1179 * 1180 * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 1181 * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 1182 * array space is needed to store Tx soft rings. Thus we allocate so 1183 * much array space for srs_tx_soft_rings. 1184 * 1185 * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 1186 * of space to st_soft_rings. This array is used for quick access to 1187 * soft ring associated with a pseudo Tx ring based on the pseudo 1188 * ring's index (mr_index). 1189 */ 1190 static void 1191 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1192 { 1193 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1194 1195 if (mac_srs->srs_type & SRST_TX) { 1196 mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 1197 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1198 MAX_RINGS_PER_GROUP, KM_SLEEP); 1199 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 1200 mac_srs_tx_t *tx = &mac_srs->srs_tx; 1201 1202 tx->st_soft_rings = (mac_soft_ring_t **) 1203 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1204 MAX_RINGS_PER_GROUP, KM_SLEEP); 1205 } 1206 } else { 1207 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1208 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1209 KM_SLEEP); 1210 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1211 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1212 KM_SLEEP); 1213 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1214 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1215 KM_SLEEP); 1216 } 1217 } 1218 1219 static void 1220 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1221 { 1222 cpu_t *cp; 1223 boolean_t clear = B_FALSE; 1224 1225 ASSERT(MUTEX_HELD(&cpu_lock)); 1226 1227 if (!mac_srs_thread_bind) 1228 return; 1229 1230 cp = cpu_get(cpuid); 1231 if (cp == NULL || !cpu_is_online(cp)) 1232 return; 1233 1234 mutex_enter(&mac_srs->srs_lock); 1235 mac_srs->srs_state |= SRS_WORKER_BOUND; 1236 if (mac_srs->srs_worker_cpuid != -1) 1237 clear = B_TRUE; 1238 mac_srs->srs_worker_cpuid = cpuid; 1239 mutex_exit(&mac_srs->srs_lock); 1240 1241 if (clear) 1242 thread_affinity_clear(mac_srs->srs_worker); 1243 1244 thread_affinity_set(mac_srs->srs_worker, cpuid); 1245 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1246 } 1247 1248 static void 1249 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1250 { 1251 cpu_t *cp; 1252 boolean_t clear = B_FALSE; 1253 1254 ASSERT(MUTEX_HELD(&cpu_lock)); 1255 1256 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1257 return; 1258 1259 cp = cpu_get(cpuid); 1260 if (cp == NULL || !cpu_is_online(cp)) 1261 return; 1262 1263 mutex_enter(&mac_srs->srs_lock); 1264 mac_srs->srs_state |= SRS_POLL_BOUND; 1265 if (mac_srs->srs_poll_cpuid != -1) 1266 clear = B_TRUE; 1267 mac_srs->srs_poll_cpuid = cpuid; 1268 mutex_exit(&mac_srs->srs_lock); 1269 1270 if (clear) 1271 thread_affinity_clear(mac_srs->srs_poll_thr); 1272 1273 thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1274 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1275 } 1276 1277 /* 1278 * Re-target interrupt to the passed CPU. If re-target is successful, 1279 * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 1280 */ 1281 void 1282 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1283 { 1284 cpu_t *cp; 1285 mac_ring_t *ring = mac_srs->srs_ring; 1286 mac_intr_t *mintr = &ring->mr_info.mri_intr; 1287 flow_entry_t *flent = mac_srs->srs_flent; 1288 boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 1289 1290 ASSERT(MUTEX_HELD(&cpu_lock)); 1291 1292 /* 1293 * Don't re-target the interrupt for these cases: 1294 * 1) ring is NULL 1295 * 2) the interrupt is shared (mi_ddi_shared) 1296 * 3) ddi_handle is NULL and !primary 1297 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 1298 * Case 3 & 4 are because of mac_client_intr_cpu() routine. 1299 * This routine will re-target fixed interrupt for primary 1300 * mac client if the client has only one ring. In that 1301 * case, mc_rx_intr_cpu will already have the correct value. 1302 */ 1303 if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 1304 (mintr->mi_ddi_handle == NULL && !primary) || (primary && 1305 mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 1306 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1307 return; 1308 } 1309 1310 if (mintr->mi_ddi_handle == NULL) 1311 return; 1312 1313 cp = cpu_get(cpuid); 1314 if (cp == NULL || !cpu_is_online(cp)) 1315 return; 1316 1317 /* Drop the cpu_lock as set_intr_affinity() holds it */ 1318 mutex_exit(&cpu_lock); 1319 if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 1320 mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 1321 else 1322 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1323 mutex_enter(&cpu_lock); 1324 } 1325 1326 /* 1327 * Re-target Tx interrupts 1328 */ 1329 void 1330 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 1331 { 1332 cpu_t *cp; 1333 mac_ring_t *ring; 1334 mac_intr_t *mintr; 1335 mac_soft_ring_t *sringp; 1336 mac_srs_tx_t *srs_tx; 1337 mac_cpus_t *srs_cpu; 1338 processorid_t cpuid; 1339 int i; 1340 1341 ASSERT(MUTEX_HELD(&cpu_lock)); 1342 1343 srs_cpu = &mac_srs->srs_cpu; 1344 if (MAC_TX_SOFT_RINGS(mac_srs)) { 1345 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 1346 sringp = mac_srs->srs_tx_soft_rings[i]; 1347 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 1348 cpuid = srs_cpu->mc_tx_intr_cpu[i]; 1349 cp = cpu_get(cpuid); 1350 if (cp == NULL || !cpu_is_online(cp) || 1351 !MAC_RING_RETARGETABLE(ring)) { 1352 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1353 continue; 1354 } 1355 mintr = &ring->mr_info.mri_intr; 1356 /* 1357 * Drop the cpu_lock as set_intr_affinity() 1358 * holds it 1359 */ 1360 mutex_exit(&cpu_lock); 1361 if (set_intr_affinity(mintr->mi_ddi_handle, 1362 cpuid) == DDI_SUCCESS) { 1363 srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 1364 } else { 1365 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1366 } 1367 mutex_enter(&cpu_lock); 1368 } 1369 } else { 1370 cpuid = srs_cpu->mc_tx_intr_cpu[0]; 1371 cp = cpu_get(cpuid); 1372 if (cp == NULL || !cpu_is_online(cp)) { 1373 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1374 return; 1375 } 1376 srs_tx = &mac_srs->srs_tx; 1377 ring = (mac_ring_t *)srs_tx->st_arg2; 1378 if (MAC_RING_RETARGETABLE(ring)) { 1379 mintr = &ring->mr_info.mri_intr; 1380 mutex_exit(&cpu_lock); 1381 if ((set_intr_affinity(mintr->mi_ddi_handle, 1382 cpuid) == DDI_SUCCESS)) { 1383 srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 1384 } else { 1385 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1386 } 1387 mutex_enter(&cpu_lock); 1388 } 1389 } 1390 } 1391 1392 /* 1393 * When a CPU comes back online, bind the MAC kernel threads which 1394 * were previously bound to that CPU, and had to be unbound because 1395 * the CPU was going away. 1396 * 1397 * These functions are called with cpu_lock held and hence we can't 1398 * cv_wait to grab the mac perimeter. Since these functions walk the soft 1399 * ring list of an SRS without being in the perimeter, the list itself 1400 * is protected by the SRS lock. 1401 */ 1402 static void 1403 mac_walk_srs_and_bind(int cpuid) 1404 { 1405 mac_soft_ring_set_t *mac_srs; 1406 mac_soft_ring_t *soft_ring; 1407 1408 rw_enter(&mac_srs_g_lock, RW_READER); 1409 1410 if ((mac_srs = mac_srs_g_list) == NULL) 1411 goto done; 1412 1413 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1414 if (mac_srs->srs_worker_cpuid == -1 && 1415 mac_srs->srs_worker_cpuid_save == cpuid) { 1416 mac_srs->srs_worker_cpuid_save = -1; 1417 mac_srs_worker_bind(mac_srs, cpuid); 1418 } 1419 1420 if (!(mac_srs->srs_type & SRST_TX)) { 1421 if (mac_srs->srs_poll_cpuid == -1 && 1422 mac_srs->srs_poll_cpuid_save == cpuid) { 1423 mac_srs->srs_poll_cpuid_save = -1; 1424 mac_srs_poll_bind(mac_srs, cpuid); 1425 } 1426 } 1427 1428 /* Next tackle the soft rings associated with the srs */ 1429 mutex_enter(&mac_srs->srs_lock); 1430 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1431 soft_ring = soft_ring->s_ring_next) { 1432 if (soft_ring->s_ring_cpuid == -1 && 1433 soft_ring->s_ring_cpuid_save == cpuid) { 1434 soft_ring->s_ring_cpuid_save = -1; 1435 (void) mac_soft_ring_bind(soft_ring, cpuid); 1436 } 1437 } 1438 mutex_exit(&mac_srs->srs_lock); 1439 } 1440 done: 1441 rw_exit(&mac_srs_g_lock); 1442 } 1443 1444 /* 1445 * Change the priority of the SRS's poll and worker thread. Additionally, 1446 * update the priority of the worker threads for the SRS's soft rings. 1447 * Need to modify any associated squeue threads. 1448 */ 1449 void 1450 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1451 { 1452 mac_soft_ring_t *ringp; 1453 1454 mac_srs->srs_pri = prival; 1455 thread_lock(mac_srs->srs_worker); 1456 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1457 thread_unlock(mac_srs->srs_worker); 1458 if (mac_srs->srs_poll_thr != NULL) { 1459 thread_lock(mac_srs->srs_poll_thr); 1460 (void) thread_change_pri(mac_srs->srs_poll_thr, 1461 mac_srs->srs_pri, 0); 1462 thread_unlock(mac_srs->srs_poll_thr); 1463 } 1464 if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1465 return; 1466 while (ringp != mac_srs->srs_soft_ring_tail) { 1467 thread_lock(ringp->s_ring_worker); 1468 (void) thread_change_pri(ringp->s_ring_worker, 1469 mac_srs->srs_pri, 0); 1470 thread_unlock(ringp->s_ring_worker); 1471 ringp = ringp->s_ring_next; 1472 } 1473 ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1474 thread_lock(ringp->s_ring_worker); 1475 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1476 thread_unlock(ringp->s_ring_worker); 1477 } 1478 1479 /* 1480 * Change the receive bandwidth limit. 1481 */ 1482 static void 1483 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1484 { 1485 mac_soft_ring_t *softring; 1486 1487 mutex_enter(&srs->srs_lock); 1488 mutex_enter(&srs->srs_bw->mac_bw_lock); 1489 1490 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1491 /* Reset bandwidth limit */ 1492 if (srs->srs_type & SRST_BW_CONTROL) { 1493 softring = srs->srs_soft_ring_head; 1494 while (softring != NULL) { 1495 softring->s_ring_type &= ~ST_RING_BW_CTL; 1496 softring = softring->s_ring_next; 1497 } 1498 srs->srs_type &= ~SRST_BW_CONTROL; 1499 srs->srs_drain_func = mac_rx_srs_drain; 1500 } 1501 } else { 1502 /* Set/Modify bandwidth limit */ 1503 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1504 /* 1505 * Give twice the queuing capability before 1506 * dropping packets. The unit is bytes/tick. 1507 */ 1508 srs->srs_bw->mac_bw_drop_threshold = 1509 srs->srs_bw->mac_bw_limit << 1; 1510 if (!(srs->srs_type & SRST_BW_CONTROL)) { 1511 softring = srs->srs_soft_ring_head; 1512 while (softring != NULL) { 1513 softring->s_ring_type |= ST_RING_BW_CTL; 1514 softring = softring->s_ring_next; 1515 } 1516 srs->srs_type |= SRST_BW_CONTROL; 1517 srs->srs_drain_func = mac_rx_srs_drain_bw; 1518 } 1519 } 1520 done: 1521 mutex_exit(&srs->srs_bw->mac_bw_lock); 1522 mutex_exit(&srs->srs_lock); 1523 } 1524 1525 /* Change the transmit bandwidth limit */ 1526 static void 1527 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1528 { 1529 uint32_t tx_mode, ring_info = 0; 1530 mac_srs_tx_t *srs_tx = &srs->srs_tx; 1531 mac_client_impl_t *mcip = srs->srs_mcip; 1532 1533 /* 1534 * We need to quiesce/restart the client here because mac_tx() and 1535 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1536 * st_mode and related fields, which are modified by the code below. 1537 */ 1538 mac_tx_client_quiesce((mac_client_handle_t)mcip); 1539 1540 mutex_enter(&srs->srs_lock); 1541 mutex_enter(&srs->srs_bw->mac_bw_lock); 1542 1543 tx_mode = srs_tx->st_mode; 1544 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1545 /* Reset bandwidth limit */ 1546 if (tx_mode == SRS_TX_BW) { 1547 if (srs_tx->st_arg2 != NULL) 1548 ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1549 if (mac_tx_serialize || 1550 (ring_info & MAC_RING_TX_SERIALIZE)) { 1551 srs_tx->st_mode = SRS_TX_SERIALIZE; 1552 } else { 1553 srs_tx->st_mode = SRS_TX_DEFAULT; 1554 } 1555 } else if (tx_mode == SRS_TX_BW_FANOUT) { 1556 srs_tx->st_mode = SRS_TX_FANOUT; 1557 } else if (tx_mode == SRS_TX_BW_AGGR) { 1558 srs_tx->st_mode = SRS_TX_AGGR; 1559 } 1560 srs->srs_type &= ~SRST_BW_CONTROL; 1561 } else { 1562 /* Set/Modify bandwidth limit */ 1563 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1564 /* 1565 * Give twice the queuing capability before 1566 * dropping packets. The unit is bytes/tick. 1567 */ 1568 srs->srs_bw->mac_bw_drop_threshold = 1569 srs->srs_bw->mac_bw_limit << 1; 1570 srs->srs_type |= SRST_BW_CONTROL; 1571 if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 1572 tx_mode != SRS_TX_BW_AGGR) { 1573 if (tx_mode == SRS_TX_SERIALIZE || 1574 tx_mode == SRS_TX_DEFAULT) { 1575 srs_tx->st_mode = SRS_TX_BW; 1576 } else if (tx_mode == SRS_TX_FANOUT) { 1577 srs_tx->st_mode = SRS_TX_BW_FANOUT; 1578 } else if (tx_mode == SRS_TX_AGGR) { 1579 srs_tx->st_mode = SRS_TX_BW_AGGR; 1580 } else { 1581 ASSERT(0); 1582 } 1583 } 1584 } 1585 done: 1586 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1587 mutex_exit(&srs->srs_bw->mac_bw_lock); 1588 mutex_exit(&srs->srs_lock); 1589 1590 mac_tx_client_restart((mac_client_handle_t)mcip); 1591 } 1592 1593 /* 1594 * The uber function that deals with any update to bandwidth limits. 1595 */ 1596 void 1597 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1598 { 1599 int count; 1600 1601 for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1602 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1603 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1604 } 1605 1606 /* 1607 * When the first sub-flow is added to a link, we disable polling on the 1608 * link and also modify the entry point to mac_rx_srs_subflow_process(). 1609 * (polling is disabled because with the subflow added, accounting 1610 * for polling needs additional logic, it is assumed that when a subflow is 1611 * added, we can take some hit as a result of disabling polling rather than 1612 * adding more complexity - if this becomes a perf. issue we need to 1613 * re-rvaluate this logic). When the last subflow is removed, we turn back 1614 * polling and also reset the entry point to mac_rx_srs_process(). 1615 * 1616 * In the future if there are multiple SRS, we can simply 1617 * take one and give it to the flow rather than disabling polling and 1618 * resetting the entry point. 1619 */ 1620 void 1621 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1622 { 1623 flow_entry_t *flent = mcip->mci_flent; 1624 int i; 1625 mac_impl_t *mip = mcip->mci_mip; 1626 mac_rx_func_t rx_func; 1627 uint_t rx_srs_cnt; 1628 boolean_t enable_classifier; 1629 1630 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1631 1632 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1633 1634 rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1635 mac_rx_srs_process; 1636 1637 /* Tell mac_srs_poll_state_change to disable polling if necessary */ 1638 if (mip->mi_state_flags & MIS_POLL_DISABLE) 1639 enable_classifier = B_TRUE; 1640 1641 /* 1642 * If receive function has already been configured correctly for 1643 * current subflow configuration, do nothing. 1644 */ 1645 if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1646 return; 1647 1648 rx_srs_cnt = flent->fe_rx_srs_cnt; 1649 for (i = 0; i < rx_srs_cnt; i++) { 1650 ASSERT(flent->fe_rx_srs[i] != NULL); 1651 mac_srs_poll_state_change(flent->fe_rx_srs[i], 1652 enable_classifier, rx_func); 1653 } 1654 1655 /* 1656 * Change the S/W classifier so that we can land in the 1657 * correct processing function with correct argument. 1658 * If all subflows have been removed we can revert to 1659 * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process(). 1660 */ 1661 mutex_enter(&flent->fe_lock); 1662 flent->fe_cb_fn = (flow_fn_t)rx_func; 1663 flent->fe_cb_arg1 = (void *)mip; 1664 flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1665 mutex_exit(&flent->fe_lock); 1666 } 1667 1668 static void 1669 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1670 { 1671 int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1672 mac_soft_ring_t *softring; 1673 1674 softring = mac_srs->srs_soft_ring_head; 1675 if (softring == NULL) { 1676 ASSERT(mac_srs->srs_soft_ring_count == 0); 1677 mac_srs->srs_tcp_ring_count = 0; 1678 mac_srs->srs_udp_ring_count = 0; 1679 mac_srs->srs_oth_ring_count = 0; 1680 mac_srs->srs_tx_ring_count = 0; 1681 return; 1682 } 1683 1684 while (softring != NULL) { 1685 if (softring->s_ring_type & ST_RING_TCP) { 1686 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1687 } else if (softring->s_ring_type & ST_RING_UDP) { 1688 mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1689 } else if (softring->s_ring_type & ST_RING_OTH) { 1690 mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1691 } else { 1692 ASSERT(softring->s_ring_type & ST_RING_TX); 1693 mac_srs->srs_tx_soft_rings[tx_count++] = softring; 1694 } 1695 softring = softring->s_ring_next; 1696 } 1697 1698 ASSERT(mac_srs->srs_soft_ring_count == 1699 (tcp_count + udp_count + oth_count + tx_count)); 1700 mac_srs->srs_tcp_ring_count = tcp_count; 1701 mac_srs->srs_udp_ring_count = udp_count; 1702 mac_srs->srs_oth_ring_count = oth_count; 1703 mac_srs->srs_tx_ring_count = tx_count; 1704 } 1705 1706 void 1707 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 1708 mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1709 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1710 mac_resource_handle_t x_arg2, boolean_t set_bypass) 1711 { 1712 mac_soft_ring_t *softring; 1713 mac_rx_fifo_t mrf; 1714 1715 bzero(&mrf, sizeof (mac_rx_fifo_t)); 1716 mrf.mrf_type = MAC_RX_FIFO; 1717 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1718 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 1719 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1720 mrf.mrf_flow_priority = pri; 1721 1722 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1723 (type|ST_RING_TCP), pri, mcip, mac_srs, 1724 cpuid, rx_func, x_arg1, x_arg2); 1725 softring->s_ring_rx_arg2 = NULL; 1726 1727 /* 1728 * TCP and UDP support DLS bypass. In addition TCP 1729 * squeue can also poll their corresponding soft rings. 1730 */ 1731 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1732 mac_soft_ring_dls_bypass(softring, 1733 mcip->mci_direct_rx_fn, 1734 mcip->mci_direct_rx_arg); 1735 1736 mrf.mrf_rx_arg = softring; 1737 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1738 1739 /* 1740 * Make a call in IP to get a TCP squeue assigned to 1741 * this softring to maintain full CPU locality through 1742 * the stack and allow the squeue to be able to poll 1743 * the softring so the flow control can be pushed 1744 * all the way to H/W. 1745 */ 1746 softring->s_ring_rx_arg2 = 1747 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1748 (mac_resource_t *)&mrf); 1749 } 1750 1751 /* 1752 * Non-TCP protocols don't support squeues. Hence we 1753 * don't make any ring addition callbacks for non-TCP 1754 * rings. Now create the UDP softring and allow it to 1755 * bypass the DLS layer. 1756 */ 1757 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1758 (type|ST_RING_UDP), pri, mcip, mac_srs, 1759 cpuid, rx_func, x_arg1, x_arg2); 1760 softring->s_ring_rx_arg2 = NULL; 1761 1762 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1763 mac_soft_ring_dls_bypass(softring, 1764 mcip->mci_direct_rx_fn, 1765 mcip->mci_direct_rx_arg); 1766 } 1767 1768 /* Create the Oth softrings which has to go through the DLS */ 1769 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1770 (type|ST_RING_OTH), pri, mcip, mac_srs, 1771 cpuid, rx_func, x_arg1, x_arg2); 1772 softring->s_ring_rx_arg2 = NULL; 1773 } 1774 1775 /* 1776 * This routine associates a CPU or a set of CPU to process incoming 1777 * traffic from a mac client. If multiple CPUs are specified, then 1778 * so many soft rings are created with each soft ring worker thread 1779 * bound to a CPU in the set. Each soft ring in turn will be 1780 * associated with an squeue and the squeue will be moved to the 1781 * same CPU as that of the soft ring's. 1782 */ 1783 static void 1784 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 1785 void *x_arg1, mac_resource_handle_t x_arg2, 1786 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1787 { 1788 mac_soft_ring_t *softring; 1789 uint32_t soft_ring_flag = 0; 1790 processorid_t cpuid = -1; 1791 int i, srings_present, new_fanout_cnt; 1792 mac_cpus_t *srs_cpu; 1793 1794 /* fanout state is REINIT. Set it back to INIT */ 1795 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1796 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1797 1798 /* how many are present right now */ 1799 srings_present = mac_rx_srs->srs_tcp_ring_count; 1800 /* new request */ 1801 srs_cpu = &mac_rx_srs->srs_cpu; 1802 new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1803 1804 mutex_enter(&mac_rx_srs->srs_lock); 1805 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1806 soft_ring_flag |= ST_RING_BW_CTL; 1807 mutex_exit(&mac_rx_srs->srs_lock); 1808 1809 if (new_fanout_cnt > srings_present) { 1810 /* soft rings increased */ 1811 mutex_enter(&mac_rx_srs->srs_lock); 1812 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1813 mutex_exit(&mac_rx_srs->srs_lock); 1814 1815 for (i = mac_rx_srs->srs_tcp_ring_count; 1816 i < new_fanout_cnt; i++) { 1817 /* 1818 * Create the protocol softrings and set the 1819 * DLS bypass where possible. 1820 */ 1821 mac_srs_create_proto_softrings(i, soft_ring_flag, 1822 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1823 rx_func, x_arg1, x_arg2, B_TRUE); 1824 } 1825 mac_srs_update_fanout_list(mac_rx_srs); 1826 } else if (new_fanout_cnt < srings_present) { 1827 /* soft rings decreased */ 1828 if (new_fanout_cnt == 1) { 1829 mutex_enter(&mac_rx_srs->srs_lock); 1830 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1831 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1832 mutex_exit(&mac_rx_srs->srs_lock); 1833 } 1834 /* Get rid of extra soft rings */ 1835 for (i = new_fanout_cnt; 1836 i < mac_rx_srs->srs_tcp_ring_count; i++) { 1837 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1838 if (softring->s_ring_rx_arg2 != NULL) { 1839 mcip->mci_resource_remove( 1840 (void *)mcip->mci_resource_arg, 1841 softring->s_ring_rx_arg2); 1842 } 1843 mac_soft_ring_remove(mac_rx_srs, 1844 mac_rx_srs->srs_tcp_soft_rings[i]); 1845 mac_soft_ring_remove(mac_rx_srs, 1846 mac_rx_srs->srs_udp_soft_rings[i]); 1847 mac_soft_ring_remove(mac_rx_srs, 1848 mac_rx_srs->srs_oth_soft_rings[i]); 1849 } 1850 mac_srs_update_fanout_list(mac_rx_srs); 1851 } 1852 1853 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1854 mutex_enter(&cpu_lock); 1855 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1856 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1857 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1858 cpuid); 1859 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1860 cpuid); 1861 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1862 cpuid); 1863 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1864 if (softring->s_ring_rx_arg2 != NULL) { 1865 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1866 softring->s_ring_rx_arg2, cpuid); 1867 } 1868 } 1869 1870 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1871 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1872 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1873 /* 1874 * Bind Tx srs and soft ring threads too. Let's bind tx 1875 * srs to the last cpu in mrp list. 1876 */ 1877 if (mac_tx_srs != NULL) { 1878 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1879 mac_tx_srs_retarget_intr(mac_tx_srs); 1880 } 1881 mutex_exit(&cpu_lock); 1882 } 1883 1884 /* 1885 * Bind SRS threads and soft rings to CPUs/create fanout list. 1886 */ 1887 void 1888 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 1889 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1890 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 1891 cpupart_t *cpupart) 1892 { 1893 int i; 1894 processorid_t cpuid; 1895 uint32_t soft_ring_flag = 0; 1896 int soft_ring_cnt; 1897 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1898 1899 /* 1900 * Remove the no soft ring flag and we will adjust it 1901 * appropriately further down. 1902 */ 1903 mutex_enter(&mac_rx_srs->srs_lock); 1904 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1905 mutex_exit(&mac_rx_srs->srs_lock); 1906 1907 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1908 1909 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1910 soft_ring_flag |= ST_RING_BW_CTL; 1911 1912 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1913 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1914 /* 1915 * Ring count can be 0 if no fanout is required and no cpu 1916 * were specified. Leave the SRS worker and poll thread 1917 * unbound 1918 */ 1919 ASSERT(mrp != NULL); 1920 soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1921 1922 /* Step 1: bind cpu contains cpu list where threads need to bind */ 1923 if (soft_ring_cnt > 0) { 1924 mutex_enter(&cpu_lock); 1925 for (i = 0; i < soft_ring_cnt; i++) { 1926 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1927 /* Create the protocol softrings */ 1928 mac_srs_create_proto_softrings(i, soft_ring_flag, 1929 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1930 rx_func, x_arg1, x_arg2, B_FALSE); 1931 } 1932 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1933 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1934 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1935 /* 1936 * Bind Tx srs and soft ring threads too. 1937 * Let's bind tx srs to the last cpu in 1938 * mrp list. 1939 */ 1940 if (mac_tx_srs == NULL) { 1941 mutex_exit(&cpu_lock); 1942 goto alldone; 1943 } 1944 1945 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1946 mac_tx_srs_retarget_intr(mac_tx_srs); 1947 mutex_exit(&cpu_lock); 1948 } else { 1949 mutex_enter(&cpu_lock); 1950 /* 1951 * For a subflow, mrp_workerid and mrp_pollid 1952 * is not set. 1953 */ 1954 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 1955 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1956 mutex_exit(&cpu_lock); 1957 goto no_softrings; 1958 } 1959 1960 alldone: 1961 if (soft_ring_cnt > 1) 1962 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1963 mac_srs_update_fanout_list(mac_rx_srs); 1964 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1965 return; 1966 1967 no_softrings: 1968 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1969 mutex_enter(&cpu_lock); 1970 cpuid = mac_next_bind_cpu(cpupart); 1971 /* Create the protocol softrings */ 1972 mac_srs_create_proto_softrings(0, soft_ring_flag, 1973 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1974 rx_func, x_arg1, x_arg2, B_FALSE); 1975 mutex_exit(&cpu_lock); 1976 } else { 1977 /* 1978 * This is the case when there is no fanout which is 1979 * true for subflows. 1980 */ 1981 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1982 } 1983 mac_srs_update_fanout_list(mac_rx_srs); 1984 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1985 } 1986 1987 /* 1988 * Calls mac_srs_fanout_init() or modify() depending upon whether 1989 * the SRS is getting initialized or re-initialized. 1990 */ 1991 void 1992 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1993 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1994 mac_resource_handle_t x_arg2, cpupart_t *cpupart) 1995 { 1996 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1997 int i, rx_srs_cnt; 1998 1999 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 2000 2001 /* 2002 * Aggr ports do not have SRSes. This function should never be 2003 * called on an aggr port. 2004 */ 2005 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2006 mac_rx_srs = flent->fe_rx_srs[0]; 2007 2008 /* 2009 * Set up the fanout on the tx side only once, with the 2010 * first rx SRS. The CPU binding, fanout, and bandwidth 2011 * criteria are common to both RX and TX, so 2012 * initializing them along side avoids redundant code. 2013 */ 2014 mac_tx_srs = flent->fe_tx_srs; 2015 rx_srs_cnt = flent->fe_rx_srs_cnt; 2016 2017 /* No fanout for subflows */ 2018 if (flent->fe_type & FLOW_USER) { 2019 mac_srs_fanout_init(mcip, mrp, rx_func, 2020 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2021 cpupart); 2022 return; 2023 } 2024 2025 if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 2026 mac_flow_user_cpu_init(flent, mrp); 2027 else 2028 mac_flow_cpu_init(flent, cpupart); 2029 2030 mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2031 2032 /* 2033 * Set up fanout for both SW (0th SRS) and HW classified 2034 * SRS (the rest of Rx SRSs in flent). 2035 */ 2036 for (i = 0; i < rx_srs_cnt; i++) { 2037 mac_rx_srs = flent->fe_rx_srs[i]; 2038 if (i != 0) 2039 mac_tx_srs = NULL; 2040 switch (mac_rx_srs->srs_fanout_state) { 2041 case SRS_FANOUT_UNINIT: 2042 mac_srs_fanout_init(mcip, mrp, rx_func, 2043 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2044 cpupart); 2045 break; 2046 case SRS_FANOUT_INIT: 2047 break; 2048 case SRS_FANOUT_REINIT: 2049 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 2050 mac_srs_fanout_modify(mcip, rx_func, x_arg1, 2051 x_arg2, mac_rx_srs, mac_tx_srs); 2052 mac_rx_srs_restart(mac_rx_srs); 2053 break; 2054 default: 2055 VERIFY(mac_rx_srs->srs_fanout_state <= 2056 SRS_FANOUT_REINIT); 2057 break; 2058 } 2059 } 2060 } 2061 2062 /* 2063 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2064 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2065 * processing is created. 2066 * 2067 * Details on Rx SRS: 2068 * Create a SRS and also add the necessary soft rings for TCP and 2069 * non-TCP based on fanout type and count specified. 2070 * 2071 * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2072 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2073 * to be heavily modified. 2074 * 2075 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2076 */ 2077 mac_soft_ring_set_t * 2078 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2079 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2080 mac_ring_t *ring) 2081 { 2082 mac_soft_ring_set_t *mac_srs; 2083 mac_srs_rx_t *srs_rx; 2084 mac_srs_tx_t *srs_tx; 2085 mac_bw_ctl_t *mac_bw; 2086 mac_resource_props_t *mrp; 2087 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2088 2089 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2090 bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2091 srs_rx = &mac_srs->srs_rx; 2092 srs_tx = &mac_srs->srs_tx; 2093 2094 mutex_enter(&flent->fe_lock); 2095 2096 /* 2097 * Get the bandwidth control structure from the flent. Get 2098 * rid of any residual values in the control structure for 2099 * the tx bw struct and also for the rx, if the rx srs is 2100 * the 1st one being brought up (the rx bw ctl struct may 2101 * be shared by multiple SRSs) 2102 */ 2103 if (is_tx_srs) { 2104 mac_srs->srs_bw = &flent->fe_tx_bw; 2105 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2106 flent->fe_tx_srs = mac_srs; 2107 } else { 2108 /* 2109 * The bw counter (stored in the flent) is shared 2110 * by SRS's within an rx group. 2111 */ 2112 mac_srs->srs_bw = &flent->fe_rx_bw; 2113 /* First rx SRS, clear the bw structure */ 2114 if (flent->fe_rx_srs_cnt == 0) 2115 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2116 2117 /* 2118 * It is better to panic here rather than just assert because 2119 * on a non-debug kernel we might end up courrupting memory 2120 * and making it difficult to debug. 2121 */ 2122 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 2123 panic("Array Overrun detected due to MAC client %p " 2124 " having more rings than %d", (void *)mcip, 2125 MAX_RINGS_PER_GROUP); 2126 } 2127 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2128 flent->fe_rx_srs_cnt++; 2129 } 2130 mac_srs->srs_flent = flent; 2131 mutex_exit(&flent->fe_lock); 2132 2133 mac_srs->srs_state = 0; 2134 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2135 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2136 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 2137 mac_srs->srs_mcip = mcip; 2138 mac_srs_fanout_list_alloc(mac_srs); 2139 2140 /* 2141 * For a flow we use the underlying MAC client's priority range with 2142 * the priority value to find an absolute priority value. For a MAC 2143 * client we use the MAC client's maximum priority as the value. 2144 */ 2145 mrp = &flent->fe_effective_props; 2146 if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2147 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2148 mcip->mci_max_pri, mrp->mrp_priority); 2149 } else { 2150 mac_srs->srs_pri = mcip->mci_max_pri; 2151 } 2152 /* 2153 * We need to insert the SRS in the global list before 2154 * binding the SRS and SR threads. Otherwise there is a 2155 * is a small window where the cpu reconfig callbacks 2156 * may miss the SRS in the list walk and DR could fail 2157 * as there are bound threads. 2158 */ 2159 mac_srs_add_glist(mac_srs); 2160 2161 /* Initialize bw limit */ 2162 if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2163 mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2164 2165 mac_bw = mac_srs->srs_bw; 2166 mutex_enter(&mac_bw->mac_bw_lock); 2167 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2168 2169 /* 2170 * Give twice the queuing capability before 2171 * dropping packets. The unit is bytes/tick. 2172 */ 2173 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2174 mutex_exit(&mac_bw->mac_bw_lock); 2175 mac_srs->srs_type |= SRST_BW_CONTROL; 2176 } else { 2177 mac_srs->srs_drain_func = mac_rx_srs_drain; 2178 } 2179 2180 /* 2181 * We use the following policy to control Receive 2182 * Side Dynamic Polling: 2183 * 1) We switch to poll mode anytime the processing thread causes 2184 * a backlog to build up in SRS and its associated Soft Rings 2185 * (sr_poll_pkt_cnt > 0). 2186 * 2) As long as the backlog stays under the low water mark 2187 * (sr_lowat), we poll the H/W for more packets. 2188 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2189 * stay in poll mode but don't poll the H/W for more packets. 2190 * 4) Anytime in polling mode, if we poll the H/W for packets and 2191 * find nothing plus we have an existing backlog 2192 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2193 * the H/W for packets anymore (let the polling thread go to sleep). 2194 * 5) Once the backlog is relieved (packets are processed) we reenable 2195 * polling (by signalling the poll thread) only when the backlog 2196 * dips below sr_poll_thres. 2197 * 6) sr_hiwat is used exclusively when we are not polling capable 2198 * and is used to decide when to drop packets so the SRS queue 2199 * length doesn't grow infinitely. 2200 */ 2201 if (!is_tx_srs) { 2202 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2203 /* Low water mark needs to be less than high water mark */ 2204 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2205 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2206 (mac_soft_ring_max_q_cnt >> 2); 2207 /* Poll threshold need to be half of low water mark or less */ 2208 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2209 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2210 (srs_rx->sr_lowat >> 1); 2211 if (mac_latency_optimize) 2212 mac_srs->srs_state |= SRS_LATENCY_OPT; 2213 else 2214 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2215 } 2216 2217 /* 2218 * Create the srs_worker with twice the stack of a normal kernel thread 2219 * to reduce the likelihood of stack overflows in receive-side 2220 * processing. (The larger stacks are not the only precaution taken 2221 * against stack overflows; see the use of mac_rx_srs_stack_needed 2222 * in mac_sched.c). 2223 */ 2224 mac_srs->srs_worker = thread_create(NULL, default_stksize << 1, 2225 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2226 2227 if (is_tx_srs) { 2228 /* Handle everything about Tx SRS and return */ 2229 mac_srs->srs_drain_func = mac_tx_srs_drain; 2230 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2231 srs_tx->st_hiwat = 2232 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2233 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2234 srs_tx->st_arg1 = x_arg1; 2235 srs_tx->st_arg2 = x_arg2; 2236 goto done; 2237 } 2238 2239 if ((srs_type & SRST_FLOW) != 0 || 2240 FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2241 srs_rx->sr_lower_proc = mac_rx_srs_process; 2242 else 2243 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2244 2245 srs_rx->sr_func = rx_func; 2246 srs_rx->sr_arg1 = x_arg1; 2247 srs_rx->sr_arg2 = x_arg2; 2248 2249 if (ring != NULL) { 2250 uint_t ring_info; 2251 2252 /* Is the mac_srs created over the RX default group? */ 2253 if (ring->mr_gh == (mac_group_handle_t) 2254 MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2255 mac_srs->srs_type |= SRST_DEFAULT_GRP; 2256 } 2257 mac_srs->srs_ring = ring; 2258 ring->mr_srs = mac_srs; 2259 ring->mr_classify_type = MAC_HW_CLASSIFIER; 2260 ring->mr_flag |= MR_INCIPIENT; 2261 2262 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 2263 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2264 mac_srs->srs_state |= SRS_POLLING_CAPAB; 2265 2266 mac_srs->srs_poll_thr = thread_create(NULL, 0, 2267 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2268 mac_srs->srs_pri); 2269 /* 2270 * Some drivers require serialization and don't send 2271 * packet chains in interrupt context. For such 2272 * drivers, we should always queue in the soft ring 2273 * so that we get a chance to switch into polling 2274 * mode under backlog. 2275 */ 2276 ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 2277 if (ring_info & MAC_RING_RX_ENQUEUE) 2278 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2279 } 2280 done: 2281 mac_srs_stat_create(mac_srs); 2282 return (mac_srs); 2283 } 2284 2285 /* 2286 * Figure out the number of soft rings required. Its dependant on 2287 * if protocol fanout is required (for LINKs), global settings 2288 * require us to do fanout for performance (based on mac_soft_ring_enable), 2289 * or user has specifically requested fanout. 2290 */ 2291 static uint32_t 2292 mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2293 { 2294 uint32_t fanout_type; 2295 mac_resource_props_t *mrp = &flent->fe_effective_props; 2296 2297 /* no fanout for subflows */ 2298 switch (link_type) { 2299 case SRST_FLOW: 2300 fanout_type = SRST_NO_SOFT_RINGS; 2301 break; 2302 case SRST_LINK: 2303 fanout_type = SRST_FANOUT_PROTO; 2304 break; 2305 } 2306 2307 /* A primary NIC/link is being plumbed */ 2308 if (flent->fe_type & FLOW_PRIMARY_MAC) { 2309 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2310 fanout_type |= SRST_FANOUT_SRC_IP; 2311 } 2312 } else if (flent->fe_type & FLOW_VNIC) { 2313 /* A VNIC is being created */ 2314 if (mrp != NULL && mrp->mrp_ncpus > 0) { 2315 fanout_type |= SRST_FANOUT_SRC_IP; 2316 } 2317 } 2318 2319 return (fanout_type); 2320 } 2321 2322 /* 2323 * Change a group from h/w to s/w classification. 2324 */ 2325 void 2326 mac_rx_switch_grp_to_sw(mac_group_t *group) 2327 { 2328 mac_ring_t *ring; 2329 mac_soft_ring_set_t *mac_srs; 2330 2331 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2332 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2333 /* 2334 * Remove the SRS associated with the HW ring. 2335 * As a result, polling will be disabled. 2336 */ 2337 mac_srs = ring->mr_srs; 2338 ASSERT(mac_srs != NULL); 2339 mac_rx_srs_remove(mac_srs); 2340 ring->mr_srs = NULL; 2341 } 2342 2343 if (ring->mr_state != MR_INUSE) 2344 (void) mac_start_ring(ring); 2345 2346 /* 2347 * We need to perform SW classification 2348 * for packets landing in these rings 2349 */ 2350 ring->mr_flag = 0; 2351 ring->mr_classify_type = MAC_SW_CLASSIFIER; 2352 } 2353 } 2354 2355 /* 2356 * Create the Rx SRS for S/W classifier and for each ring in the 2357 * group (if exclusive group). Also create the Tx SRS. 2358 */ 2359 void 2360 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2361 uint32_t link_type) 2362 { 2363 cpupart_t *cpupart; 2364 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2365 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2366 boolean_t use_default = B_FALSE; 2367 2368 mac_rx_srs_group_setup(mcip, flent, link_type); 2369 mac_tx_srs_group_setup(mcip, flent, link_type); 2370 2371 /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */ 2372 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 2373 return; 2374 2375 pool_lock(); 2376 cpupart = mac_pset_find(mrp, &use_default); 2377 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2378 mac_rx_deliver, mcip, NULL, cpupart); 2379 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 2380 pool_unlock(); 2381 } 2382 2383 /* 2384 * Set up the Rx SRSes. If there is no group associated with the 2385 * client, then only setup SW classification. If the client has 2386 * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an 2387 * SRS for each HW ring. If the client is sharing a group, then make 2388 * sure to teardown the HW SRSes. 2389 */ 2390 void 2391 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2392 uint32_t link_type) 2393 { 2394 mac_impl_t *mip = mcip->mci_mip; 2395 mac_soft_ring_set_t *mac_srs; 2396 mac_ring_t *ring; 2397 uint32_t fanout_type; 2398 mac_group_t *rx_group = flent->fe_rx_ring_group; 2399 boolean_t no_unicast; 2400 2401 /* 2402 * If this is an an aggr port, then don't setup Rx SRS and Rx 2403 * soft rings as they won't be used. However, we still need to 2404 * start the rings to receive data on them. 2405 */ 2406 if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) { 2407 if (rx_group == NULL) 2408 return; 2409 2410 for (ring = rx_group->mrg_rings; ring != NULL; 2411 ring = ring->mr_next) { 2412 if (ring->mr_state != MR_INUSE) 2413 (void) mac_start_ring(ring); 2414 } 2415 2416 return; 2417 } 2418 2419 /* 2420 * Aggr ports should never have SRSes. 2421 */ 2422 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2423 2424 fanout_type = mac_find_fanout(flent, link_type); 2425 no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0; 2426 2427 /* Create the SRS for SW classification if none exists */ 2428 if (flent->fe_rx_srs[0] == NULL) { 2429 ASSERT(flent->fe_rx_srs_cnt == 0); 2430 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2431 mac_rx_deliver, mcip, NULL, NULL); 2432 mutex_enter(&flent->fe_lock); 2433 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2434 flent->fe_cb_arg1 = (void *)mip; 2435 flent->fe_cb_arg2 = (void *)mac_srs; 2436 mutex_exit(&flent->fe_lock); 2437 } 2438 2439 if (rx_group == NULL) 2440 return; 2441 2442 /* 2443 * If the group is marked RESERVED then setup an SRS and 2444 * fanout for each HW ring. 2445 */ 2446 switch (rx_group->mrg_state) { 2447 case MAC_GROUP_STATE_RESERVED: 2448 for (ring = rx_group->mrg_rings; ring != NULL; 2449 ring = ring->mr_next) { 2450 uint16_t vid = i_mac_flow_vid(mcip->mci_flent); 2451 2452 switch (ring->mr_state) { 2453 case MR_INUSE: 2454 case MR_FREE: 2455 if (ring->mr_srs != NULL) 2456 break; 2457 if (ring->mr_state != MR_INUSE) 2458 (void) mac_start_ring(ring); 2459 2460 /* 2461 * If a client requires SW VLAN 2462 * filtering or has no unicast address 2463 * then we don't create any HW ring 2464 * SRSes. 2465 */ 2466 if ((!MAC_GROUP_HW_VLAN(rx_group) && 2467 vid != VLAN_ID_NONE) || no_unicast) 2468 break; 2469 2470 /* 2471 * When a client has exclusive use of 2472 * a group, and that group's traffic 2473 * is fully HW classified, we create 2474 * an SRS for each HW ring in order to 2475 * make use of dynamic polling of said 2476 * HW rings. 2477 */ 2478 mac_srs = mac_srs_create(mcip, flent, 2479 fanout_type | link_type, 2480 mac_rx_deliver, mcip, NULL, ring); 2481 break; 2482 default: 2483 cmn_err(CE_PANIC, 2484 "srs_setup: mcip = %p " 2485 "trying to add UNKNOWN ring = %p\n", 2486 (void *)mcip, (void *)ring); 2487 break; 2488 } 2489 } 2490 break; 2491 case MAC_GROUP_STATE_SHARED: 2492 /* 2493 * When a group is shared by multiple clients, we must 2494 * use SW classifiction to ensure packets are 2495 * delivered to the correct client. 2496 */ 2497 mac_rx_switch_grp_to_sw(rx_group); 2498 break; 2499 default: 2500 ASSERT(B_FALSE); 2501 break; 2502 } 2503 } 2504 2505 /* 2506 * Set up the TX SRS. 2507 */ 2508 void 2509 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2510 uint32_t link_type) 2511 { 2512 /* 2513 * If this is an exclusive client (e.g. an aggr port), then 2514 * don't setup Tx SRS and Tx soft rings as they won't be used. 2515 * However, we still need to start the rings to send data 2516 * across them. 2517 */ 2518 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 2519 mac_ring_t *ring; 2520 mac_group_t *grp; 2521 2522 grp = (mac_group_t *)flent->fe_tx_ring_group; 2523 2524 if (grp == NULL) 2525 return; 2526 2527 for (ring = grp->mrg_rings; ring != NULL; 2528 ring = ring->mr_next) { 2529 if (ring->mr_state != MR_INUSE) 2530 (void) mac_start_ring(ring); 2531 } 2532 2533 return; 2534 } 2535 2536 /* 2537 * Aggr ports should never have SRSes. 2538 */ 2539 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2540 2541 if (flent->fe_tx_srs == NULL) { 2542 (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 2543 NULL, mcip, NULL, NULL); 2544 } 2545 2546 mac_tx_srs_setup(mcip, flent); 2547 } 2548 2549 /* 2550 * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown 2551 * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set 2552 * when we wish to move a MAC client from one group to another. In 2553 * that case, we need to release the current HW SRSes but keep the SW 2554 * SRS for continued traffic classifiction. 2555 */ 2556 void 2557 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 2558 { 2559 mac_soft_ring_set_t *mac_srs; 2560 int i; 2561 int count = flent->fe_rx_srs_cnt; 2562 2563 for (i = 0; i < count; i++) { 2564 if (i == 0 && hwonly) 2565 continue; 2566 mac_srs = flent->fe_rx_srs[i]; 2567 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2568 mac_srs_free(mac_srs); 2569 flent->fe_rx_srs[i] = NULL; 2570 flent->fe_rx_srs_cnt--; 2571 } 2572 2573 /* 2574 * If we are only tearing down the HW SRSes then there must be 2575 * one SRS left for SW classification. Otherwise we are tearing 2576 * down both HW and SW and there should be no SRSes left. 2577 */ 2578 if (hwonly) 2579 VERIFY3S(flent->fe_rx_srs_cnt, ==, 1); 2580 else 2581 VERIFY3S(flent->fe_rx_srs_cnt, ==, 0); 2582 } 2583 2584 /* 2585 * Remove the TX SRS. 2586 */ 2587 void 2588 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2589 uint32_t link_type) 2590 { 2591 mac_soft_ring_set_t *tx_srs; 2592 mac_srs_tx_t *tx; 2593 2594 if ((tx_srs = flent->fe_tx_srs) == NULL) 2595 return; 2596 2597 tx = &tx_srs->srs_tx; 2598 switch (link_type) { 2599 case SRST_FLOW: 2600 /* 2601 * For flows, we need to work with passed 2602 * flent to find the Rx/Tx SRS. 2603 */ 2604 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2605 break; 2606 case SRST_LINK: 2607 mac_tx_client_condemn((mac_client_handle_t)mcip); 2608 if (tx->st_arg2 != NULL) { 2609 ASSERT(tx_srs->srs_type & SRST_TX); 2610 /* 2611 * The ring itself will be stopped when 2612 * we release the group or in the 2613 * mac_datapath_teardown (for the default 2614 * group) 2615 */ 2616 tx->st_arg2 = NULL; 2617 } 2618 break; 2619 default: 2620 ASSERT(B_FALSE); 2621 break; 2622 } 2623 mac_srs_free(tx_srs); 2624 flent->fe_tx_srs = NULL; 2625 } 2626 2627 /* 2628 * This is the group state machine. 2629 * 2630 * The state of an Rx group is given by 2631 * the following table. The default group and its rings are started in 2632 * mac_start itself and the default group stays in SHARED state until 2633 * mac_stop at which time the group and rings are stopped and and it 2634 * reverts to the Registered state. 2635 * 2636 * Typically this function is called on a group after adding or removing a 2637 * client from it, to find out what should be the new state of the group. 2638 * If the new state is RESERVED, then the client that owns this group 2639 * exclusively is also returned. Note that adding or removing a client from 2640 * a group could also impact the default group and the caller needs to 2641 * evaluate the effect on the default group. 2642 * 2643 * Group type # of clients mi_nactiveclients Group State 2644 * in the group 2645 * 2646 * Non-default 0 N.A. REGISTERED 2647 * Non-default 1 N.A. RESERVED 2648 * 2649 * Default 0 N.A. SHARED 2650 * Default 1 1 RESERVED 2651 * Default 1 > 1 SHARED 2652 * Default > 1 N.A. SHARED 2653 * 2654 * For a TX group, the following is the state table. 2655 * 2656 * Group type # of clients Group State 2657 * in the group 2658 * 2659 * Non-default 0 REGISTERED 2660 * Non-default 1 RESERVED 2661 * 2662 * Default 0 REGISTERED 2663 * Default 1 RESERVED 2664 * Default > 1 SHARED 2665 */ 2666 mac_group_state_t 2667 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 2668 mac_group_t *defgrp, boolean_t rx_group) 2669 { 2670 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2671 2672 *group_only_mcip = NULL; 2673 2674 /* Non-default group */ 2675 2676 if (grp != defgrp) { 2677 if (MAC_GROUP_NO_CLIENT(grp)) 2678 return (MAC_GROUP_STATE_REGISTERED); 2679 2680 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2681 if (*group_only_mcip != NULL) 2682 return (MAC_GROUP_STATE_RESERVED); 2683 2684 return (MAC_GROUP_STATE_SHARED); 2685 } 2686 2687 /* Default group */ 2688 2689 if (MAC_GROUP_NO_CLIENT(grp)) { 2690 if (rx_group) 2691 return (MAC_GROUP_STATE_SHARED); 2692 else 2693 return (MAC_GROUP_STATE_REGISTERED); 2694 } 2695 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2696 if (*group_only_mcip == NULL) 2697 return (MAC_GROUP_STATE_SHARED); 2698 2699 if (rx_group && mip->mi_nactiveclients != 1) 2700 return (MAC_GROUP_STATE_SHARED); 2701 2702 ASSERT(*group_only_mcip != NULL); 2703 return (MAC_GROUP_STATE_RESERVED); 2704 } 2705 2706 /* 2707 * OVERVIEW NOTES FOR DATAPATH 2708 * =========================== 2709 * 2710 * Create an SRS and setup the corresponding flow function and args. 2711 * Add a classification rule for the flow specified by 'flent' and program 2712 * the hardware classifier when applicable. 2713 * 2714 * Rx ring assignment, SRS, polling and B/W enforcement 2715 * ---------------------------------------------------- 2716 * 2717 * We try to use H/W classification on NIC and assign traffic to a 2718 * MAC address to a particular Rx ring. There is a 1-1 mapping 2719 * between a SRS and a Rx ring. The SRS (short for soft ring set) 2720 * dynamically switches the underlying Rx ring between interrupt 2721 * and polling mode and enforces any specified B/W control. 2722 * 2723 * There is always a SRS created and tied to each H/W and S/W rule. 2724 * Whenever we create a H/W rule, we always add the the same rule to 2725 * S/W classifier and tie a SRS to it. 2726 * 2727 * In case a B/W control is specified, its broken into bytes 2728 * per ticks and as soon as the quota for a tick is exhausted, 2729 * the underlying Rx ring is forced into poll mode for remianing 2730 * tick. The SRS poll thread only polls for bytes that are 2731 * allowed to come in the SRS. We typically let 4x the configured 2732 * B/W worth of packets to come in the SRS (to prevent unnecessary 2733 * drops due to bursts) but only process the specified amount. 2734 * 2735 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2736 * Rx rings (and corresponding SRSs) assigned to it. The SRS 2737 * in turn can have softrings to do protocol level fanout or 2738 * softrings to do S/W based fanout or both. In case the NIC 2739 * has no Rx rings, we do S/W classification to respective SRS. 2740 * The S/W classification rule is always setup and ready. This 2741 * allows the MAC layer to reassign Rx rings whenever needed 2742 * but packets still continue to flow via the default path and 2743 * getting S/W classified to correct SRS. 2744 * 2745 * In other cases where a NIC or VNIC is plumbed, our goal is use 2746 * H/W classifier and get two Rx ring assigned for the Link. One 2747 * for TCP and one for UDP|SCTP. The respective SRS still do the 2748 * polling on the Rx ring. For Link that is plumbed for IP, there 2749 * is a TCP squeue which also does polling and can control the 2750 * the Rx ring directly (where SRS is just pass through). For 2751 * the following cases, the SRS does the polling underneath. 2752 * 1) non IP based Links (Links which are not plumbed via ifconfig) 2753 * and paths which have no IP squeues (UDP & SCTP) 2754 * 2) If B/W control is specified on the Link 2755 * 3) If S/W fanout is secified 2756 * 2757 * Note1: As of current implementation, we try to assign only 1 Rx 2758 * ring per Link and more than 1 Rx ring for primary Link for 2759 * H/W based fanout. We always create following softrings per SRS: 2760 * 1) TCP softring which is polled by TCP squeue where possible 2761 * (and also bypasses DLS) 2762 * 2) UDP/SCTP based which bypasses DLS 2763 * 3) OTH softring which goes via DLS (currently deal with IPv6 2764 * and non TCP/UDP/SCTP for IPv4 packets). 2765 * 2766 * It is necessary to create 3 softrings since SRS has to poll 2767 * the single Rx ring underneath and enforce any link level B/W 2768 * control (we can't switch the Rx ring in poll mode just based 2769 * on TCP squeue if the same Rx ring is sharing UDP and other 2770 * traffic as well). Once polling is done and any Link level B/W 2771 * control is specified, the packets are assigned to respective 2772 * softring based on protocol. Since TCP has IP based squeue 2773 * which benefits by polling, we separate TCP packets into 2774 * its own softring which can be polled by IP squeue. We need 2775 * to separate out UDP/SCTP to UDP softring since it can bypass 2776 * the DLS layer which has heavy performance advanatges and we 2777 * need a softring (OTH) for rest. 2778 * 2779 * ToDo: The 3 softrings for protocol are needed only till we can 2780 * get rid of DLS from datapath, make IPv4 and IPv6 paths 2781 * symmetric (deal with mac_header_info for v6 and polling for 2782 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2783 * are generic), and bring SAP based classification to MAC layer 2784 * 2785 * H/W and S/W based fanout and multiple Rx rings per Link 2786 * ------------------------------------------------------- 2787 * 2788 * In case, fanout is requested (or determined automatically based 2789 * on Link speed and processor speed), we try to assign multiple 2790 * Rx rings per Link with their respective SRS. In this case 2791 * the NIC should be capable of fanning out incoming packets between 2792 * the assigned Rx rings (H/W based fanout). All the SRS 2793 * individually switch their Rx ring between interrupt and polling 2794 * mode but share a common B/W control counter in case of Link 2795 * level B/W is specified. 2796 * 2797 * If S/W based fanout is specified in lieu of H/W based fanout, 2798 * the Link SRS creates the specified number of softrings for 2799 * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2800 * out to the correct softring based on their protocol and 2801 * protocol specific hash function. 2802 * 2803 * Primary and non primary MAC clients 2804 * ----------------------------------- 2805 * 2806 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2807 * and are a Layer 2 construct. 2808 * 2809 * Primary NIC: 2810 * The Link that owns the primary MAC address and typically 2811 * is used as the data NIC in non virtualized cases. As such 2812 * H/W resources are preferntially given to primary NIC. As 2813 * far as code is concerned, there is no difference in the 2814 * primary NIC vs VNICs. They are all treated as Links. 2815 * At the very first call to mac_unicast_add() we program the S/W 2816 * classifier for the primary MAC address, get a soft ring set 2817 * (and soft rings based on 'ip_soft_ring_cnt') 2818 * and a Rx ring assigned for polling to get enabled. 2819 * When IP get plumbed and negotiates polling, we can 2820 * let squeue do the polling on TCP softring. 2821 * 2822 * VNICs: 2823 * Same as any other Link. As long as the H/W resource assignments 2824 * are equal, the data path and setup for all Links is same. 2825 * 2826 * Flows: 2827 * Can be configured on Links. They have their own SRS and the 2828 * S/W classifier is programmed appropriately based on the flow. 2829 * The flows typically deal with layer 3 and above and 2830 * creates a soft ring set specific to the flow. The receive 2831 * side function is switched from mac_rx_srs_process to 2832 * mac_rx_srs_subflow_process which first tries to assign the 2833 * packet to appropriate flow SRS and failing which assigns it 2834 * to link SRS. This allows us to avoid the layered approach 2835 * which gets complex. 2836 * 2837 * By the time mac_datapath_setup() completes, we already have the 2838 * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2839 * and S/W classifiers programmed. IP is not plumbed yet (and might 2840 * never be for Virtual Machines guest OS path). When IP is plumbed 2841 * (for both NIC and VNIC), we do a capability negotiation for polling 2842 * and upcall functions etc. 2843 * 2844 * Rx ring Assignement NOTES 2845 * ------------------------- 2846 * 2847 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2848 * as NIC with a single default ring), we assign the only ring to 2849 * primary Link. The primary Link SRS can do polling on it as long as 2850 * it is the only link in use and we compare the MAC address for unicast 2851 * packets before accepting an incoming packet (there is no need for S/W 2852 * classification in this case). We disable polling on the only ring the 2853 * moment 2nd link gets created (the polling remains enabled even though 2854 * there are broadcast and * multicast flows created). 2855 * 2856 * If the NIC has more than 1 Rx ring, we assign the default ring (the 2857 * 1st ring) to deal with broadcast, multicast and traffic for other 2858 * NICs which needs S/W classification. We assign the primary mac 2859 * addresses to another ring by specifiying a classification rule for 2860 * primary unicast MAC address to the selected ring. The primary Link 2861 * (and its SRS) can continue to poll the assigned Rx ring at all times 2862 * independantly. 2863 * 2864 * Note: In future, if no fanout is specified, we try to assign 2 Rx 2865 * rings for the primary Link with the primary MAC address + TCP going 2866 * to one ring and primary MAC address + UDP|SCTP going to other ring. 2867 * Any remaining traffic for primary MAC address can go to the default 2868 * Rx ring and get S/W classified. This way the respective SRSs don't 2869 * need to do proto fanout and don't need to have softrings at all and 2870 * can poll their respective Rx rings. 2871 * 2872 * As an optimization, when a new NIC or VNIC is created, we can get 2873 * only one Rx ring and make it a TCP specific Rx ring and use the 2874 * H/W default Rx ring for the rest (this Rx ring is never polled). 2875 * 2876 * For clients that don't have MAC address, but want to receive and 2877 * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 2878 * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 2879 * always give the default group and use software classification (i.e. 2880 * even if this is the only client in the default group, we will 2881 * leave group as shared). 2882 */ 2883 2884 int 2885 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2886 uint32_t link_type) 2887 { 2888 mac_impl_t *mip = mcip->mci_mip; 2889 mac_group_t *rgroup = NULL; 2890 mac_group_t *tgroup = NULL; 2891 mac_group_t *default_rgroup; 2892 mac_group_t *default_tgroup; 2893 int err; 2894 uint16_t vid; 2895 uint8_t *mac_addr; 2896 mac_group_state_t next_state; 2897 mac_client_impl_t *group_only_mcip; 2898 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2899 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2900 boolean_t rxhw; 2901 boolean_t txhw; 2902 boolean_t use_default = B_FALSE; 2903 cpupart_t *cpupart; 2904 boolean_t no_unicast; 2905 boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 2906 mac_client_impl_t *reloc_pmcip = NULL; 2907 boolean_t use_hw; 2908 2909 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2910 2911 switch (link_type) { 2912 case SRST_FLOW: 2913 mac_srs_group_setup(mcip, flent, link_type); 2914 return (0); 2915 2916 case SRST_LINK: 2917 no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2918 mac_addr = flent->fe_flow_desc.fd_dst_mac; 2919 2920 /* Default RX group */ 2921 default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2922 2923 /* Default TX group */ 2924 default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 2925 2926 if (no_unicast) { 2927 rgroup = default_rgroup; 2928 tgroup = default_tgroup; 2929 goto grp_found; 2930 } 2931 rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 2932 (mrp->mrp_nrxrings > 0 || 2933 (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 2934 txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 2935 (mrp->mrp_ntxrings > 0 || 2936 (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 2937 2938 /* 2939 * All the rings initially belong to the default group 2940 * under dynamic grouping. The primary client uses the 2941 * default group when it is the only client. The 2942 * default group is also used as the destination for 2943 * all multicast and broadcast traffic of all clients. 2944 * Therefore, the primary client loses its ability to 2945 * poll the softrings on addition of a second client. 2946 * To avoid a performance penalty, MAC will move the 2947 * primary client to a dedicated group when it can. 2948 * 2949 * When using static grouping, the primary client 2950 * begins life on a non-default group. There is 2951 * no moving needed upon addition of a second client. 2952 */ 2953 if (!isprimary && mip->mi_nactiveclients == 2 && 2954 (group_only_mcip = mac_primary_client_handle(mip)) != 2955 NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 2956 reloc_pmcip = mac_check_primary_relocation( 2957 group_only_mcip, rxhw); 2958 } 2959 2960 /* 2961 * Check to see if we can get an exclusive group for 2962 * this mac address or if there already exists a 2963 * group that has this mac address (case of VLANs). 2964 * If no groups are available, use the default group. 2965 */ 2966 rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 2967 if (rgroup == NULL && rxhw) { 2968 err = ENOSPC; 2969 goto setup_failed; 2970 } else if (rgroup == NULL) { 2971 rgroup = default_rgroup; 2972 } 2973 2974 /* 2975 * If we are adding a second client to a 2976 * non-default group then we need to move the 2977 * existing client to the default group and 2978 * add the new client to the default group as 2979 * well. 2980 */ 2981 if (rgroup != default_rgroup && 2982 rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) { 2983 group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup); 2984 err = mac_rx_switch_group(group_only_mcip, rgroup, 2985 default_rgroup); 2986 2987 if (err != 0) 2988 goto setup_failed; 2989 2990 rgroup = default_rgroup; 2991 } 2992 2993 /* 2994 * Check to see if we can get an exclusive group for 2995 * this mac client. If no groups are available, use 2996 * the default group. 2997 */ 2998 tgroup = mac_reserve_tx_group(mcip, B_FALSE); 2999 if (tgroup == NULL && txhw) { 3000 if (rgroup != NULL && rgroup != default_rgroup) 3001 mac_release_rx_group(mcip, rgroup); 3002 err = ENOSPC; 3003 goto setup_failed; 3004 } else if (tgroup == NULL) { 3005 tgroup = default_tgroup; 3006 } 3007 3008 /* 3009 * Some NICs don't support any Rx rings, so there may not 3010 * even be a default group. 3011 */ 3012 grp_found: 3013 if (rgroup != NULL) { 3014 if (rgroup != default_rgroup && 3015 MAC_GROUP_NO_CLIENT(rgroup) && 3016 (rxhw || mcip->mci_share != 0)) { 3017 MAC_RX_GRP_RESERVED(mip); 3018 if (mip->mi_rx_group_type == 3019 MAC_GROUP_TYPE_DYNAMIC) { 3020 MAC_RX_RING_RESERVED(mip, 3021 rgroup->mrg_cur_count); 3022 } 3023 } 3024 3025 flent->fe_rx_ring_group = rgroup; 3026 /* 3027 * Add the client to the group and update the 3028 * group's state. If rgroup != default_group 3029 * then the rgroup should only ever have one 3030 * client and be in the RESERVED state. But no 3031 * matter what, the default_rgroup will enter 3032 * the SHARED state since it has to receive 3033 * all broadcast and multicast traffic. This 3034 * case is handled later in the function. 3035 */ 3036 mac_group_add_client(rgroup, mcip); 3037 next_state = mac_group_next_state(rgroup, 3038 &group_only_mcip, default_rgroup, B_TRUE); 3039 mac_set_group_state(rgroup, next_state); 3040 } 3041 3042 if (tgroup != NULL) { 3043 if (tgroup != default_tgroup && 3044 MAC_GROUP_NO_CLIENT(tgroup) && 3045 (txhw || mcip->mci_share != 0)) { 3046 MAC_TX_GRP_RESERVED(mip); 3047 if (mip->mi_tx_group_type == 3048 MAC_GROUP_TYPE_DYNAMIC) { 3049 MAC_TX_RING_RESERVED(mip, 3050 tgroup->mrg_cur_count); 3051 } 3052 } 3053 flent->fe_tx_ring_group = tgroup; 3054 mac_group_add_client(tgroup, mcip); 3055 next_state = mac_group_next_state(tgroup, 3056 &group_only_mcip, default_tgroup, B_FALSE); 3057 tgroup->mrg_state = next_state; 3058 } 3059 3060 /* We are setting up minimal datapath only */ 3061 if (no_unicast) { 3062 mac_srs_group_setup(mcip, flent, link_type); 3063 break; 3064 } 3065 3066 /* Program software classification. */ 3067 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3068 goto setup_failed; 3069 3070 /* Program hardware classification. */ 3071 vid = i_mac_flow_vid(flent); 3072 use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0; 3073 err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw); 3074 3075 if (err != 0) 3076 goto setup_failed; 3077 3078 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 3079 VERIFY3P(mcip->mci_unicast, !=, NULL); 3080 3081 /* 3082 * Setup the Rx and Tx SRSes. If the client has a 3083 * reserved group, then mac_srs_group_setup() creates 3084 * the required SRSes for the HW rings. If we have a 3085 * shared group, mac_srs_group_setup() dismantles the 3086 * HW SRSes of the previously exclusive group. 3087 */ 3088 mac_srs_group_setup(mcip, flent, link_type); 3089 3090 /* (Re)init the v6 token & local addr used by link protection */ 3091 mac_protect_update_mac_token(mcip); 3092 break; 3093 3094 default: 3095 ASSERT(B_FALSE); 3096 break; 3097 } 3098 3099 /* 3100 * All broadcast and multicast traffic is received only on the default 3101 * group. If we have setup the datapath for a non-default group above 3102 * then move the default group to shared state to allow distribution of 3103 * incoming broadcast traffic to the other groups and dismantle the 3104 * SRSes over the default group. 3105 */ 3106 if (rgroup != NULL) { 3107 if (rgroup != default_rgroup) { 3108 if (default_rgroup->mrg_state == 3109 MAC_GROUP_STATE_RESERVED) { 3110 group_only_mcip = MAC_GROUP_ONLY_CLIENT( 3111 default_rgroup); 3112 ASSERT(group_only_mcip != NULL && 3113 mip->mi_nactiveclients > 1); 3114 3115 mac_set_group_state(default_rgroup, 3116 MAC_GROUP_STATE_SHARED); 3117 mac_rx_srs_group_setup(group_only_mcip, 3118 group_only_mcip->mci_flent, SRST_LINK); 3119 pool_lock(); 3120 cpupart = mac_pset_find(mrp, &use_default); 3121 mac_fanout_setup(group_only_mcip, 3122 group_only_mcip->mci_flent, 3123 MCIP_RESOURCE_PROPS(group_only_mcip), 3124 mac_rx_deliver, group_only_mcip, NULL, 3125 cpupart); 3126 mac_set_pool_effective(use_default, cpupart, 3127 mrp, emrp); 3128 pool_unlock(); 3129 } 3130 ASSERT(default_rgroup->mrg_state == 3131 MAC_GROUP_STATE_SHARED); 3132 } 3133 3134 /* 3135 * A VLAN MAC client on a reserved group still 3136 * requires SW classification if the MAC doesn't 3137 * provide VLAN HW filtering. 3138 * 3139 * Clients with no unicast address also require SW 3140 * classification. 3141 */ 3142 if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 3143 ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) || 3144 no_unicast)) { 3145 mac_rx_switch_grp_to_sw(rgroup); 3146 } 3147 3148 } 3149 3150 mac_set_rings_effective(mcip); 3151 return (0); 3152 3153 setup_failed: 3154 /* Switch the primary back to default group */ 3155 if (reloc_pmcip != NULL) { 3156 (void) mac_rx_switch_group(reloc_pmcip, 3157 reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 3158 } 3159 mac_datapath_teardown(mcip, flent, link_type); 3160 return (err); 3161 } 3162 3163 void 3164 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3165 uint32_t link_type) 3166 { 3167 mac_impl_t *mip = mcip->mci_mip; 3168 mac_group_t *group = NULL; 3169 mac_client_impl_t *grp_only_mcip; 3170 flow_entry_t *group_only_flent; 3171 mac_group_t *default_group; 3172 boolean_t check_default_group = B_FALSE; 3173 mac_group_state_t next_state; 3174 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 3175 uint16_t vid; 3176 3177 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3178 3179 switch (link_type) { 3180 case SRST_FLOW: 3181 mac_rx_srs_group_teardown(flent, B_FALSE); 3182 mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3183 return; 3184 3185 case SRST_LINK: 3186 /* Stop sending packets */ 3187 mac_tx_client_block(mcip); 3188 group = flent->fe_rx_ring_group; 3189 vid = i_mac_flow_vid(flent); 3190 3191 /* 3192 * Stop the packet flow from the hardware by disabling 3193 * any hardware filters assigned to this client. 3194 */ 3195 if (mcip->mci_unicast != NULL) { 3196 int err; 3197 3198 err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid); 3199 3200 if (err != 0) { 3201 cmn_err(CE_WARN, "%s: failed to remove a MAC HW" 3202 " filters because of error 0x%x", 3203 mip->mi_name, err); 3204 } 3205 3206 mcip->mci_unicast = NULL; 3207 } 3208 3209 /* Stop the packets coming from the S/W classifier */ 3210 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3211 mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3212 3213 /* Quiesce and destroy all the SRSes. */ 3214 mac_rx_srs_group_teardown(flent, B_FALSE); 3215 mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 3216 3217 ASSERT3P(mcip->mci_flent, ==, flent); 3218 ASSERT3P(flent->fe_next, ==, NULL); 3219 3220 /* 3221 * Release our hold on the group as well. We need 3222 * to check if the shared group has only one client 3223 * left who can use it exclusively. Also, if we 3224 * were the last client, release the group. 3225 */ 3226 default_group = MAC_DEFAULT_RX_GROUP(mip); 3227 if (group != NULL) { 3228 mac_group_remove_client(group, mcip); 3229 next_state = mac_group_next_state(group, 3230 &grp_only_mcip, default_group, B_TRUE); 3231 3232 if (next_state == MAC_GROUP_STATE_RESERVED) { 3233 /* 3234 * Only one client left on this RX group. 3235 */ 3236 VERIFY3P(grp_only_mcip, !=, NULL); 3237 mac_set_group_state(group, 3238 MAC_GROUP_STATE_RESERVED); 3239 group_only_flent = grp_only_mcip->mci_flent; 3240 3241 /* 3242 * The only remaining client has exclusive 3243 * access on the group. Allow it to 3244 * dynamically poll the H/W rings etc. 3245 */ 3246 mac_rx_srs_group_setup(grp_only_mcip, 3247 group_only_flent, SRST_LINK); 3248 mac_fanout_setup(grp_only_mcip, 3249 group_only_flent, 3250 MCIP_RESOURCE_PROPS(grp_only_mcip), 3251 mac_rx_deliver, grp_only_mcip, NULL, NULL); 3252 mac_rx_group_unmark(group, MR_INCIPIENT); 3253 mac_set_rings_effective(grp_only_mcip); 3254 } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3255 /* 3256 * This is a non-default group being freed up. 3257 * We need to reevaluate the default group 3258 * to see if the primary client can get 3259 * exclusive access to the default group. 3260 */ 3261 VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip)); 3262 if (mrp->mrp_mask & MRP_RX_RINGS) { 3263 MAC_RX_GRP_RELEASED(mip); 3264 if (mip->mi_rx_group_type == 3265 MAC_GROUP_TYPE_DYNAMIC) { 3266 MAC_RX_RING_RELEASED(mip, 3267 group->mrg_cur_count); 3268 } 3269 } 3270 mac_release_rx_group(mcip, group); 3271 mac_set_group_state(group, 3272 MAC_GROUP_STATE_REGISTERED); 3273 check_default_group = B_TRUE; 3274 } else { 3275 VERIFY3S(next_state, ==, 3276 MAC_GROUP_STATE_SHARED); 3277 mac_set_group_state(group, 3278 MAC_GROUP_STATE_SHARED); 3279 mac_rx_group_unmark(group, MR_CONDEMNED); 3280 } 3281 flent->fe_rx_ring_group = NULL; 3282 } 3283 /* 3284 * Remove the client from the TX group. Additionally, if 3285 * this a non-default group, then we also need to release 3286 * the group. 3287 */ 3288 group = flent->fe_tx_ring_group; 3289 default_group = MAC_DEFAULT_TX_GROUP(mip); 3290 if (group != NULL) { 3291 mac_group_remove_client(group, mcip); 3292 next_state = mac_group_next_state(group, 3293 &grp_only_mcip, default_group, B_FALSE); 3294 if (next_state == MAC_GROUP_STATE_REGISTERED) { 3295 if (group != default_group) { 3296 if (mrp->mrp_mask & MRP_TX_RINGS) { 3297 MAC_TX_GRP_RELEASED(mip); 3298 if (mip->mi_tx_group_type == 3299 MAC_GROUP_TYPE_DYNAMIC) { 3300 MAC_TX_RING_RELEASED( 3301 mip, group-> 3302 mrg_cur_count); 3303 } 3304 } 3305 mac_release_tx_group(mcip, group); 3306 /* 3307 * If the default group is reserved, 3308 * then we need to set the effective 3309 * rings as we would have given 3310 * back some rings when the group 3311 * was released 3312 */ 3313 if (mip->mi_tx_group_type == 3314 MAC_GROUP_TYPE_DYNAMIC && 3315 default_group->mrg_state == 3316 MAC_GROUP_STATE_RESERVED) { 3317 grp_only_mcip = 3318 MAC_GROUP_ONLY_CLIENT 3319 (default_group); 3320 mac_set_rings_effective( 3321 grp_only_mcip); 3322 } 3323 } else { 3324 mac_ring_t *ring; 3325 int cnt; 3326 int ringcnt; 3327 3328 /* 3329 * Stop all the rings except the 3330 * default ring. 3331 */ 3332 ringcnt = group->mrg_cur_count; 3333 ring = group->mrg_rings; 3334 for (cnt = 0; cnt < ringcnt; cnt++) { 3335 if (ring->mr_state == 3336 MR_INUSE && ring != 3337 (mac_ring_t *) 3338 mip->mi_default_tx_ring) { 3339 mac_stop_ring(ring); 3340 ring->mr_flag = 0; 3341 } 3342 ring = ring->mr_next; 3343 } 3344 } 3345 } else if (next_state == MAC_GROUP_STATE_RESERVED) { 3346 mac_set_rings_effective(grp_only_mcip); 3347 } 3348 flent->fe_tx_ring_group = NULL; 3349 group->mrg_state = next_state; 3350 } 3351 break; 3352 default: 3353 ASSERT(B_FALSE); 3354 break; 3355 } 3356 3357 /* 3358 * The mac client using the default group gets exclusive access to the 3359 * default group if and only if it is the sole client on the entire 3360 * mip. If so set the group state to reserved, and set up the SRSes 3361 * over the default group. 3362 */ 3363 if (check_default_group) { 3364 default_group = MAC_DEFAULT_RX_GROUP(mip); 3365 VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED); 3366 next_state = mac_group_next_state(default_group, 3367 &grp_only_mcip, default_group, B_TRUE); 3368 if (next_state == MAC_GROUP_STATE_RESERVED) { 3369 VERIFY3P(grp_only_mcip, !=, NULL); 3370 VERIFY3U(mip->mi_nactiveclients, ==, 1); 3371 mac_set_group_state(default_group, 3372 MAC_GROUP_STATE_RESERVED); 3373 mac_rx_srs_group_setup(grp_only_mcip, 3374 grp_only_mcip->mci_flent, SRST_LINK); 3375 mac_fanout_setup(grp_only_mcip, 3376 grp_only_mcip->mci_flent, 3377 MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 3378 grp_only_mcip, NULL, NULL); 3379 mac_rx_group_unmark(default_group, MR_INCIPIENT); 3380 mac_set_rings_effective(grp_only_mcip); 3381 } 3382 } 3383 3384 /* 3385 * If the primary is the only one left and the MAC supports 3386 * dynamic grouping, we need to see if the primary needs to 3387 * be moved to the default group so that it can use all the 3388 * H/W rings. 3389 */ 3390 if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 3391 mip->mi_nactiveclients == 1 && 3392 mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 3393 default_group = MAC_DEFAULT_RX_GROUP(mip); 3394 grp_only_mcip = mac_primary_client_handle(mip); 3395 if (grp_only_mcip == NULL) 3396 return; 3397 group_only_flent = grp_only_mcip->mci_flent; 3398 mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 3399 /* 3400 * If the primary has an explicit property set, leave it 3401 * alone. 3402 */ 3403 if (mrp->mrp_mask & MRP_RX_RINGS) 3404 return; 3405 /* 3406 * Switch the primary to the default group. 3407 */ 3408 (void) mac_rx_switch_group(grp_only_mcip, 3409 group_only_flent->fe_rx_ring_group, default_group); 3410 } 3411 } 3412 3413 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3414 3415 static void 3416 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3417 { 3418 if (mac_srs->srs_type & SRST_TX) { 3419 mac_srs_tx_t *tx; 3420 3421 ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 3422 ASSERT(mac_srs->srs_udp_soft_rings == NULL); 3423 ASSERT(mac_srs->srs_oth_soft_rings == NULL); 3424 ASSERT(mac_srs->srs_tx_soft_rings != NULL); 3425 kmem_free(mac_srs->srs_tx_soft_rings, 3426 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3427 mac_srs->srs_tx_soft_rings = NULL; 3428 tx = &mac_srs->srs_tx; 3429 if (tx->st_soft_rings != NULL) { 3430 kmem_free(tx->st_soft_rings, 3431 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3432 } 3433 } else { 3434 ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3435 ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3436 kmem_free(mac_srs->srs_tcp_soft_rings, 3437 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3438 mac_srs->srs_tcp_soft_rings = NULL; 3439 ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3440 kmem_free(mac_srs->srs_udp_soft_rings, 3441 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3442 mac_srs->srs_udp_soft_rings = NULL; 3443 ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3444 kmem_free(mac_srs->srs_oth_soft_rings, 3445 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3446 mac_srs->srs_oth_soft_rings = NULL; 3447 } 3448 } 3449 3450 /* 3451 * An RX SRS is attached to at most one mac_ring. 3452 * A TX SRS has no rings. 3453 */ 3454 static void 3455 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3456 { 3457 mac_client_impl_t *mcip; 3458 mac_ring_t *ring; 3459 flow_entry_t *flent; 3460 3461 ring = mac_srs->srs_ring; 3462 if (mac_srs->srs_type & SRST_TX) { 3463 ASSERT(ring == NULL); 3464 return; 3465 } 3466 3467 if (ring == NULL) 3468 return; 3469 3470 /* 3471 * Broadcast flows don't have a client impl association, but they 3472 * use only soft rings. 3473 */ 3474 flent = mac_srs->srs_flent; 3475 mcip = flent->fe_mcip; 3476 ASSERT(mcip != NULL); 3477 3478 ring->mr_classify_type = MAC_NO_CLASSIFIER; 3479 ring->mr_srs = NULL; 3480 } 3481 3482 /* 3483 * Physical unlink and free of the data structures happen below. This is 3484 * driven from mac_flow_destroy(), on the last refrele of a flow. 3485 * 3486 * Assumes Rx srs is 1-1 mapped with an ring. 3487 */ 3488 void 3489 mac_srs_free(mac_soft_ring_set_t *mac_srs) 3490 { 3491 ASSERT(mac_srs->srs_mcip == NULL || 3492 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3493 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3494 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3495 3496 mac_drop_chain(mac_srs->srs_first, "SRS free"); 3497 mac_srs_ring_free(mac_srs); 3498 mac_srs_soft_rings_free(mac_srs); 3499 mac_srs_fanout_list_free(mac_srs); 3500 3501 mac_srs->srs_bw = NULL; 3502 mac_srs_stat_delete(mac_srs); 3503 kmem_cache_free(mac_srs_cache, mac_srs); 3504 } 3505 3506 static void 3507 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3508 { 3509 mac_soft_ring_t *softring; 3510 3511 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3512 3513 mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3514 if (s_ring_flag == S_RING_CONDEMNED) { 3515 while (mac_srs->srs_soft_ring_condemned_count != 3516 mac_srs->srs_soft_ring_count) 3517 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3518 } else { 3519 while (mac_srs->srs_soft_ring_quiesced_count != 3520 mac_srs->srs_soft_ring_count) 3521 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3522 } 3523 mutex_exit(&mac_srs->srs_lock); 3524 3525 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3526 softring = softring->s_ring_next) { 3527 (void) untimeout(softring->s_ring_tid); 3528 softring->s_ring_tid = NULL; 3529 } 3530 3531 (void) untimeout(mac_srs->srs_tid); 3532 mac_srs->srs_tid = NULL; 3533 3534 mutex_enter(&mac_srs->srs_lock); 3535 } 3536 3537 /* 3538 * The block comment above mac_rx_classify_flow_state_change explains the 3539 * background. At this point upcalls from the driver (both hardware classified 3540 * and software classified) have been cut off. We now need to quiesce the 3541 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3542 * the master controller. The steps involved are described below in the function 3543 */ 3544 void 3545 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3546 { 3547 uint_t s_ring_flag; 3548 uint_t srs_poll_wait_flag; 3549 3550 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3551 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3552 3553 if (mac_srs->srs_state & SRS_CONDEMNED) { 3554 s_ring_flag = S_RING_CONDEMNED; 3555 srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3556 } else { 3557 s_ring_flag = S_RING_QUIESCE; 3558 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3559 } 3560 3561 /* 3562 * In the case of Rx SRS wait till the poll thread is done. 3563 */ 3564 if ((mac_srs->srs_type & SRST_TX) == 0 && 3565 mac_srs->srs_poll_thr != NULL) { 3566 while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3567 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3568 3569 /* 3570 * Turn off polling as part of the quiesce operation. 3571 */ 3572 MAC_SRS_POLLING_OFF(mac_srs); 3573 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3574 } 3575 3576 /* 3577 * Then signal the soft ring worker threads to quiesce or quit 3578 * as needed and then wait till that happens. 3579 */ 3580 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3581 3582 if (mac_srs->srs_state & SRS_CONDEMNED) 3583 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3584 else 3585 mac_srs->srs_state |= SRS_QUIESCE_DONE; 3586 cv_signal(&mac_srs->srs_quiesce_done_cv); 3587 } 3588 3589 /* 3590 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3591 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3592 * or poll thread. This function is internal to the quiescing logic and is 3593 * called internally from the SRS quiesce or flow quiesce or client quiesce 3594 * higher level functions. 3595 */ 3596 void 3597 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3598 { 3599 mac_ring_t *ring; 3600 3601 ring = mac_srs->srs_ring; 3602 ASSERT(ring == NULL || ring->mr_refcnt == 0); 3603 3604 if (srs_flag == SRS_CONDEMNED) { 3605 /* 3606 * The SRS is going away. We need to unbind the SRS and SR 3607 * threads before removing from the global SRS list. Otherwise 3608 * there is a small window where the cpu reconfig callbacks 3609 * may miss the SRS in the list walk and DR could fail since 3610 * there are still bound threads. 3611 */ 3612 mac_srs_threads_unbind(mac_srs); 3613 mac_srs_remove_glist(mac_srs); 3614 } 3615 /* 3616 * Wakeup the SRS worker and poll threads. 3617 */ 3618 mutex_enter(&mac_srs->srs_lock); 3619 mac_srs->srs_state |= srs_flag; 3620 cv_signal(&mac_srs->srs_async); 3621 cv_signal(&mac_srs->srs_cv); 3622 mutex_exit(&mac_srs->srs_lock); 3623 } 3624 3625 /* 3626 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3627 * from the driver are done, then the Rx SRS is quiesced and only then can 3628 * we signal the soft rings. Thus this function can't be called arbitrarily 3629 * without satisfying the prerequisites. On the Tx side, the threads from 3630 * top need to quiesced, then the Tx SRS and only then can we signal the 3631 * Tx soft rings. 3632 */ 3633 static void 3634 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3635 { 3636 mac_soft_ring_t *softring; 3637 3638 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3639 softring = softring->s_ring_next) 3640 mac_soft_ring_signal(softring, sr_flag); 3641 } 3642 3643 /* 3644 * The block comment above mac_rx_classify_flow_state_change explains the 3645 * background. At this point the SRS is quiesced and we need to restart the 3646 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3647 * the master controller. The steps involved are described below in the function 3648 */ 3649 void 3650 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3651 { 3652 boolean_t iam_rx_srs; 3653 mac_soft_ring_t *softring; 3654 3655 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3656 if ((mac_srs->srs_type & SRST_TX) != 0) { 3657 iam_rx_srs = B_FALSE; 3658 ASSERT((mac_srs->srs_state & 3659 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3660 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3661 } else { 3662 iam_rx_srs = B_TRUE; 3663 ASSERT((mac_srs->srs_state & 3664 (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3665 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3666 if (mac_srs->srs_poll_thr != NULL) { 3667 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3668 SRS_POLL_THR_QUIESCED); 3669 } 3670 } 3671 3672 /* 3673 * Signal any quiesced soft ring workers to restart and wait for the 3674 * soft ring down count to come down to zero. 3675 */ 3676 if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3677 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3678 softring = softring->s_ring_next) { 3679 if (!(softring->s_ring_state & S_RING_QUIESCE)) 3680 continue; 3681 mac_soft_ring_signal(softring, S_RING_RESTART); 3682 } 3683 while (mac_srs->srs_soft_ring_quiesced_count != 0) 3684 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3685 } 3686 3687 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3688 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3689 /* 3690 * Signal the poll thread and ask it to restart. Wait till it 3691 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3692 * cleared. 3693 */ 3694 mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3695 cv_signal(&mac_srs->srs_cv); 3696 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3697 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3698 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3699 } 3700 /* Wake up any waiter waiting for the restart to complete */ 3701 mac_srs->srs_state |= SRS_RESTART_DONE; 3702 cv_signal(&mac_srs->srs_quiesce_done_cv); 3703 } 3704 3705 static void 3706 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3707 { 3708 mutex_enter(&mac_srs->srs_lock); 3709 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3710 ASSERT(mac_srs->srs_worker_cpuid == -1); 3711 mutex_exit(&mac_srs->srs_lock); 3712 return; 3713 } 3714 3715 mac_srs->srs_worker_cpuid = -1; 3716 mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3717 thread_affinity_clear(mac_srs->srs_worker); 3718 mutex_exit(&mac_srs->srs_lock); 3719 } 3720 3721 static void 3722 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3723 { 3724 mutex_enter(&mac_srs->srs_lock); 3725 if (mac_srs->srs_poll_thr == NULL || 3726 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3727 ASSERT(mac_srs->srs_poll_cpuid == -1); 3728 mutex_exit(&mac_srs->srs_lock); 3729 return; 3730 } 3731 3732 mac_srs->srs_poll_cpuid = -1; 3733 mac_srs->srs_state &= ~SRS_POLL_BOUND; 3734 thread_affinity_clear(mac_srs->srs_poll_thr); 3735 mutex_exit(&mac_srs->srs_lock); 3736 } 3737 3738 static void 3739 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3740 { 3741 mac_soft_ring_t *soft_ring; 3742 3743 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3744 3745 mutex_enter(&cpu_lock); 3746 mac_srs_worker_unbind(mac_srs); 3747 if (!(mac_srs->srs_type & SRST_TX)) 3748 mac_srs_poll_unbind(mac_srs); 3749 3750 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3751 soft_ring = soft_ring->s_ring_next) { 3752 mac_soft_ring_unbind(soft_ring); 3753 } 3754 mutex_exit(&cpu_lock); 3755 } 3756 3757 /* 3758 * When a CPU is going away, unbind all MAC threads which are bound 3759 * to that CPU. The affinity of the thread to the CPU is saved to allow 3760 * the thread to be rebound to the CPU if it comes back online. 3761 */ 3762 static void 3763 mac_walk_srs_and_unbind(int cpuid) 3764 { 3765 mac_soft_ring_set_t *mac_srs; 3766 mac_soft_ring_t *soft_ring; 3767 3768 rw_enter(&mac_srs_g_lock, RW_READER); 3769 3770 if ((mac_srs = mac_srs_g_list) == NULL) 3771 goto done; 3772 3773 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3774 if (mac_srs->srs_worker_cpuid == cpuid) { 3775 mac_srs->srs_worker_cpuid_save = cpuid; 3776 mac_srs_worker_unbind(mac_srs); 3777 } 3778 3779 if (!(mac_srs->srs_type & SRST_TX)) { 3780 if (mac_srs->srs_poll_cpuid == cpuid) { 3781 mac_srs->srs_poll_cpuid_save = cpuid; 3782 mac_srs_poll_unbind(mac_srs); 3783 } 3784 } 3785 3786 /* Next tackle the soft rings associated with the srs */ 3787 mutex_enter(&mac_srs->srs_lock); 3788 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3789 soft_ring = soft_ring->s_ring_next) { 3790 if (soft_ring->s_ring_cpuid == cpuid) { 3791 soft_ring->s_ring_cpuid_save = cpuid; 3792 mac_soft_ring_unbind(soft_ring); 3793 } 3794 } 3795 mutex_exit(&mac_srs->srs_lock); 3796 } 3797 done: 3798 rw_exit(&mac_srs_g_lock); 3799 } 3800 3801 /* TX SETUP and TEARDOWN ROUTINES */ 3802 3803 /* 3804 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3805 * handle the case where the number of rings is one. I.e. there is 3806 * a ring pointed to by mac_srs->srs_tx_arg2. 3807 */ 3808 void 3809 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3810 { 3811 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3812 mac_soft_ring_t *soft_ring; 3813 int count = mac_srs->srs_tx_ring_count; 3814 uint32_t soft_ring_type = ST_RING_TX; 3815 uint_t ring_info; 3816 3817 ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3818 ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 3819 if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 3820 soft_ring_type |= ST_RING_WORKER_ONLY; 3821 soft_ring = mac_soft_ring_create(count, 0, 3822 soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3823 NULL, mcip, (mac_resource_handle_t)tx_ring); 3824 mac_srs->srs_tx_ring_count++; 3825 mac_srs_update_fanout_list(mac_srs); 3826 /* 3827 * put this soft ring in quiesce mode too so when we restart 3828 * all soft rings in the srs are in the same state. 3829 */ 3830 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3831 } 3832 3833 static void 3834 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3835 { 3836 int sringcnt; 3837 3838 mutex_enter(&mac_srs->srs_lock); 3839 sringcnt = mac_srs->srs_soft_ring_count; 3840 ASSERT(sringcnt > 0); 3841 mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3842 3843 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3844 while (mac_srs->srs_soft_ring_condemned_count != 1) 3845 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3846 3847 if (softring == mac_srs->srs_soft_ring_head) { 3848 mac_srs->srs_soft_ring_head = softring->s_ring_next; 3849 if (mac_srs->srs_soft_ring_head != NULL) { 3850 mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3851 } else { 3852 mac_srs->srs_soft_ring_tail = NULL; 3853 } 3854 } else { 3855 softring->s_ring_prev->s_ring_next = 3856 softring->s_ring_next; 3857 if (softring->s_ring_next != NULL) { 3858 softring->s_ring_next->s_ring_prev = 3859 softring->s_ring_prev; 3860 } else { 3861 mac_srs->srs_soft_ring_tail = 3862 softring->s_ring_prev; 3863 } 3864 } 3865 mac_srs->srs_soft_ring_count--; 3866 3867 mac_srs->srs_soft_ring_condemned_count--; 3868 mutex_exit(&mac_srs->srs_lock); 3869 3870 mac_soft_ring_free(softring); 3871 } 3872 3873 void 3874 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3875 { 3876 int i; 3877 mac_soft_ring_t *soft_ring, *remove_sring; 3878 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3879 3880 mutex_enter(&mac_srs->srs_lock); 3881 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 3882 soft_ring = mac_srs->srs_tx_soft_rings[i]; 3883 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3884 break; 3885 } 3886 mutex_exit(&mac_srs->srs_lock); 3887 ASSERT(i < mac_srs->srs_tx_ring_count); 3888 remove_sring = soft_ring; 3889 /* 3890 * In the case of aggr, the soft ring associated with a Tx ring 3891 * is also stored in st_soft_rings[] array. That entry should 3892 * be removed. 3893 */ 3894 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 3895 mac_srs_tx_t *tx = &mac_srs->srs_tx; 3896 3897 ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 3898 tx->st_soft_rings[tx_ring->mr_index] = NULL; 3899 } 3900 mac_soft_ring_remove(mac_srs, remove_sring); 3901 mac_srs_update_fanout_list(mac_srs); 3902 } 3903 3904 /* 3905 * mac_tx_srs_setup(): 3906 * Used to setup Tx rings. If no free Tx ring is available, then default 3907 * Tx ring is used. 3908 */ 3909 void 3910 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3911 { 3912 mac_impl_t *mip = mcip->mci_mip; 3913 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 3914 int i; 3915 int tx_ring_count = 0; 3916 uint32_t soft_ring_type; 3917 mac_group_t *grp = NULL; 3918 mac_ring_t *ring; 3919 mac_srs_tx_t *tx = &tx_srs->srs_tx; 3920 boolean_t is_aggr; 3921 uint_t ring_info = 0; 3922 3923 is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0; 3924 grp = flent->fe_tx_ring_group; 3925 if (grp == NULL) { 3926 ring = (mac_ring_t *)mip->mi_default_tx_ring; 3927 goto no_group; 3928 } 3929 tx_ring_count = grp->mrg_cur_count; 3930 ring = grp->mrg_rings; 3931 /* 3932 * An attempt is made to reserve 'tx_ring_count' number 3933 * of Tx rings. If tx_ring_count is 0, default Tx ring 3934 * is used. If it is 1, an attempt is made to reserve one 3935 * Tx ring. In both the cases, the ring information is 3936 * stored in Tx SRS. If multiple Tx rings are specified, 3937 * then each Tx ring will have a Tx-side soft ring. All 3938 * these soft rings will be hang off Tx SRS. 3939 */ 3940 switch (grp->mrg_state) { 3941 case MAC_GROUP_STATE_SHARED: 3942 case MAC_GROUP_STATE_RESERVED: 3943 if (tx_ring_count <= 1 && !is_aggr) { 3944 no_group: 3945 if (ring != NULL && 3946 ring->mr_state != MR_INUSE) { 3947 (void) mac_start_ring(ring); 3948 ring_info = mac_hwring_getinfo( 3949 (mac_ring_handle_t)ring); 3950 } 3951 tx->st_arg2 = (void *)ring; 3952 mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 3953 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3954 tx->st_mode = SRS_TX_BW; 3955 } else if (mac_tx_serialize || 3956 (ring_info & MAC_RING_TX_SERIALIZE)) { 3957 tx->st_mode = SRS_TX_SERIALIZE; 3958 } else { 3959 tx->st_mode = SRS_TX_DEFAULT; 3960 } 3961 break; 3962 } 3963 soft_ring_type = ST_RING_TX; 3964 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3965 tx->st_mode = is_aggr ? 3966 SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3967 } else { 3968 tx->st_mode = is_aggr ? SRS_TX_AGGR : 3969 SRS_TX_FANOUT; 3970 } 3971 for (i = 0; i < tx_ring_count; i++) { 3972 ASSERT(ring != NULL); 3973 switch (ring->mr_state) { 3974 case MR_INUSE: 3975 case MR_FREE: 3976 ASSERT(ring->mr_srs == NULL); 3977 3978 if (ring->mr_state != MR_INUSE) 3979 (void) mac_start_ring(ring); 3980 ring_info = mac_hwring_getinfo( 3981 (mac_ring_handle_t)ring); 3982 if (mac_tx_serialize || (ring_info & 3983 MAC_RING_TX_SERIALIZE)) { 3984 soft_ring_type |= 3985 ST_RING_WORKER_ONLY; 3986 } 3987 (void) mac_soft_ring_create(i, 0, 3988 soft_ring_type, maxclsyspri, 3989 mcip, tx_srs, -1, NULL, mcip, 3990 (mac_resource_handle_t)ring); 3991 break; 3992 default: 3993 cmn_err(CE_PANIC, 3994 "srs_setup: mcip = %p " 3995 "trying to add UNKNOWN ring = %p\n", 3996 (void *)mcip, (void *)ring); 3997 break; 3998 } 3999 ring = ring->mr_next; 4000 } 4001 mac_srs_update_fanout_list(tx_srs); 4002 break; 4003 default: 4004 ASSERT(B_FALSE); 4005 break; 4006 } 4007 tx->st_func = mac_tx_get_func(tx->st_mode); 4008 if (is_aggr) { 4009 VERIFY(i_mac_capab_get((mac_handle_t)mip, 4010 MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 4011 } 4012 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 4013 int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 4014 } 4015 4016 /* 4017 * Update the fanout of a client if its recorded link speed doesn't match 4018 * its current link speed. 4019 */ 4020 void 4021 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 4022 { 4023 uint64_t link_speed; 4024 mac_resource_props_t *mcip_mrp; 4025 flow_entry_t *flent = mcip->mci_flent; 4026 mac_soft_ring_set_t *rx_srs; 4027 mac_cpus_t *srs_cpu; 4028 int soft_ring_count, maxcpus; 4029 4030 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 4031 4032 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 4033 MAC_STAT_IFSPEED); 4034 4035 if ((link_speed != 0) && 4036 (link_speed != mcip->mci_flent->fe_nic_speed)) { 4037 mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 4038 /* 4039 * Before calling mac_fanout_setup(), check to see if 4040 * the SRSes already have the right number of soft 4041 * rings. mac_fanout_setup() is a heavy duty operation 4042 * where new cpu bindings are done for SRS and soft 4043 * ring threads and interrupts re-targeted. 4044 */ 4045 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 4046 soft_ring_count = mac_compute_soft_ring_count(flent, 4047 flent->fe_rx_srs_cnt - 1, maxcpus); 4048 /* 4049 * If soft_ring_count returned by 4050 * mac_compute_soft_ring_count() is 0, bump it 4051 * up by 1 because we always have atleast one 4052 * TCP, UDP, and OTH soft ring associated with 4053 * an SRS. 4054 */ 4055 soft_ring_count = (soft_ring_count == 0) ? 4056 1 : soft_ring_count; 4057 rx_srs = flent->fe_rx_srs[0]; 4058 srs_cpu = &rx_srs->srs_cpu; 4059 if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 4060 mac_fanout_setup(mcip, flent, mcip_mrp, 4061 mac_rx_deliver, mcip, NULL, cpupart); 4062 } 4063 } 4064 } 4065 4066 /* 4067 * Walk through the list of MAC clients for the MAC. 4068 * For each active MAC client, recompute the number of soft rings 4069 * associated with every client, only if current speed is different 4070 * from the speed that was previously used for soft ring computation. 4071 * If the cable is disconnected whlie the NIC is started, we would get 4072 * notification with speed set to 0. We do not recompute in that case. 4073 */ 4074 void 4075 mac_fanout_recompute(mac_impl_t *mip) 4076 { 4077 mac_client_impl_t *mcip; 4078 cpupart_t *cpupart; 4079 boolean_t use_default; 4080 mac_resource_props_t *mrp, *emrp; 4081 4082 i_mac_perim_enter(mip); 4083 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 4084 mip->mi_linkstate != LINK_STATE_UP) { 4085 i_mac_perim_exit(mip); 4086 return; 4087 } 4088 4089 for (mcip = mip->mi_clients_list; mcip != NULL; 4090 mcip = mcip->mci_client_next) { 4091 /* Aggr port clients don't have SRSes. */ 4092 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 4093 continue; 4094 4095 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 4096 !MCIP_DATAPATH_SETUP(mcip)) 4097 continue; 4098 mrp = MCIP_RESOURCE_PROPS(mcip); 4099 emrp = MCIP_EFFECTIVE_PROPS(mcip); 4100 use_default = B_FALSE; 4101 pool_lock(); 4102 cpupart = mac_pset_find(mrp, &use_default); 4103 mac_fanout_recompute_client(mcip, cpupart); 4104 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 4105 pool_unlock(); 4106 } 4107 4108 i_mac_perim_exit(mip); 4109 } 4110 4111 /* 4112 * Given a MAC, change the polling state for all its MAC clients. 'enable' is 4113 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 4114 * default. 4115 */ 4116 void 4117 mac_poll_state_change(mac_handle_t mh, boolean_t enable) 4118 { 4119 mac_impl_t *mip = (mac_impl_t *)mh; 4120 mac_client_impl_t *mcip; 4121 4122 i_mac_perim_enter(mip); 4123 if (enable) 4124 mip->mi_state_flags &= ~MIS_POLL_DISABLE; 4125 else 4126 mip->mi_state_flags |= MIS_POLL_DISABLE; 4127 for (mcip = mip->mi_clients_list; mcip != NULL; 4128 mcip = mcip->mci_client_next) 4129 mac_client_update_classifier(mcip, B_TRUE); 4130 i_mac_perim_exit(mip); 4131 } 4132