1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2018 Joyent, Inc. 24 * Copyright 2020 RackTop Systems. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/callb.h> 29 #include <sys/cpupart.h> 30 #include <sys/pool.h> 31 #include <sys/pool_pset.h> 32 #include <sys/sdt.h> 33 #include <sys/strsubr.h> 34 #include <sys/strsun.h> 35 #include <sys/vlan.h> 36 #include <inet/ipsec_impl.h> 37 #include <inet/ip_impl.h> 38 #include <inet/sadb.h> 39 #include <inet/ipsecesp.h> 40 #include <inet/ipsecah.h> 41 42 #include <sys/mac_impl.h> 43 #include <sys/mac_client_impl.h> 44 #include <sys/mac_client_priv.h> 45 #include <sys/mac_soft_ring.h> 46 #include <sys/mac_flow_impl.h> 47 #include <sys/mac_stat.h> 48 49 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 50 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 51 static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 52 static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 53 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 54 55 static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 56 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 57 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 58 static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 59 static void mac_srs_add_glist(mac_soft_ring_set_t *); 60 static void mac_srs_remove_glist(mac_soft_ring_set_t *); 61 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 62 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 63 64 static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 65 static void mac_walk_srs_and_bind(int); 66 static void mac_walk_srs_and_unbind(int); 67 68 extern boolean_t mac_latency_optimize; 69 70 static kmem_cache_t *mac_srs_cache; 71 kmem_cache_t *mac_soft_ring_cache; 72 73 /* 74 * The duration in msec we wait before signalling the soft ring 75 * worker thread in case packets get queued. 76 */ 77 uint32_t mac_soft_ring_worker_wait = 0; 78 79 /* 80 * A global tunable for turning polling on/off. By default, dynamic 81 * polling is always on and is always very beneficial. It should be 82 * turned off with absolute care and for the rare workload (very 83 * low latency sensitive traffic). 84 */ 85 int mac_poll_enable = B_TRUE; 86 87 /* 88 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 89 * Large values could end up in consuming lot of system memory and cause 90 * system hang. 91 */ 92 int mac_soft_ring_max_q_cnt = 1024; 93 int mac_soft_ring_min_q_cnt = 256; 94 int mac_soft_ring_poll_thres = 16; 95 96 boolean_t mac_tx_serialize = B_FALSE; 97 98 /* 99 * mac_tx_srs_hiwat is the queue depth threshold at which callers of 100 * mac_tx() will be notified of flow control condition. 101 * 102 * TCP does not honour flow control condition sent up by mac_tx(). 103 * Thus provision is made for TCP to allow more packets to be queued 104 * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 105 * 106 * Note that mac_tx_srs_hiwat is always be lesser than 107 * mac_tx_srs_max_q_cnt. 108 */ 109 uint32_t mac_tx_srs_max_q_cnt = 100000; 110 uint32_t mac_tx_srs_hiwat = 1000; 111 112 /* 113 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 114 * 115 * Global tunables that determines the number of soft rings to be used for 116 * fanning out incoming traffic on a link. These count will be used only 117 * when no explicit set of CPUs was assigned to the data-links. 118 * 119 * mac_rx_soft_ring_count tunable will come into effect only if 120 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 121 * default only for sun4v platforms. 122 * 123 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 124 * 10Gbps link and is not dependent upon mac_soft_ring_enable. 125 * 126 * The number of soft rings for fanout for a link or a flow is determined 127 * by mac_compute_soft_ring_count() routine. This routine will take into 128 * account mac_soft_ring_enable, mac_rx_soft_ring_count and 129 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 130 * 131 * If a bandwidth is specified, the determination of the number of soft 132 * rings is based on specified bandwidth, CPU speed and number of CPUs in 133 * the system. 134 */ 135 uint_t mac_rx_soft_ring_count = 8; 136 uint_t mac_rx_soft_ring_10gig_count = 8; 137 138 /* 139 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 140 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 141 * list is used to walk the list of all MAC threads when a CPU is 142 * coming online or going offline. 143 */ 144 static mac_soft_ring_set_t *mac_srs_g_list = NULL; 145 static krwlock_t mac_srs_g_lock; 146 147 /* 148 * Whether the SRS threads should be bound, or not. 149 */ 150 boolean_t mac_srs_thread_bind = B_TRUE; 151 152 /* 153 * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 154 * dladm command would override this. 155 */ 156 boolean_t mac_tx_intr_retarget = B_FALSE; 157 boolean_t mac_rx_intr_retarget = B_FALSE; 158 159 /* 160 * If cpu bindings are specified by user, then Tx SRS and its soft 161 * rings should also be bound to the CPUs specified by user. The 162 * CPUs for Tx bindings are at the end of the cpu list provided by 163 * the user. If enough CPUs are not available (for Tx and Rx 164 * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 165 */ 166 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 167 processorid_t cpuid; \ 168 int i; \ 169 mac_soft_ring_t *softring; \ 170 mac_cpus_t *srs_cpu; \ 171 \ 172 srs_cpu = &mac_tx_srs->srs_cpu; \ 173 cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 174 mac_srs_worker_bind(mac_tx_srs, cpuid); \ 175 if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 176 for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 177 cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 178 softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 179 if (cpuid != -1) { \ 180 (void) mac_soft_ring_bind(softring, \ 181 cpuid); \ 182 } \ 183 } \ 184 } \ 185 } 186 187 /* 188 * Re-targeting is allowed only for exclusive group or for primary. 189 */ 190 #define RETARGETABLE_CLIENT(group, mcip) \ 191 ((((group) != NULL) && \ 192 ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 193 mac_is_primary_client(mcip)) 194 195 #define MAC_RING_RETARGETABLE(ring) \ 196 (((ring) != NULL) && \ 197 ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 198 !((ring)->mr_info.mri_intr.mi_ddi_shared)) 199 200 201 /* INIT and FINI ROUTINES */ 202 203 void 204 mac_soft_ring_init(void) 205 { 206 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 207 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 208 209 mac_srs_cache = kmem_cache_create("mac_srs_cache", 210 sizeof (mac_soft_ring_set_t), 211 64, NULL, NULL, NULL, NULL, NULL, 0); 212 213 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 214 mutex_enter(&cpu_lock); 215 register_cpu_setup_func(mac_srs_cpu_setup, NULL); 216 mutex_exit(&cpu_lock); 217 } 218 219 void 220 mac_soft_ring_finish(void) 221 { 222 mutex_enter(&cpu_lock); 223 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 224 mutex_exit(&cpu_lock); 225 rw_destroy(&mac_srs_g_lock); 226 kmem_cache_destroy(mac_soft_ring_cache); 227 kmem_cache_destroy(mac_srs_cache); 228 } 229 230 static void 231 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 232 { 233 mac_soft_ring_t *softring, *next, *head; 234 235 /* 236 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 237 * DR. The callbacks from DR are called with cpu_lock held, and hence 238 * can't wait to grab the mac perimeter. The soft ring list is hence 239 * protected for read access by srs_lock. Changing the soft ring list 240 * needs the mac perimeter and the srs_lock. 241 */ 242 mutex_enter(&mac_srs->srs_lock); 243 244 head = mac_srs->srs_soft_ring_head; 245 mac_srs->srs_soft_ring_head = NULL; 246 mac_srs->srs_soft_ring_tail = NULL; 247 mac_srs->srs_soft_ring_count = 0; 248 249 mutex_exit(&mac_srs->srs_lock); 250 251 for (softring = head; softring != NULL; softring = next) { 252 next = softring->s_ring_next; 253 mac_soft_ring_free(softring); 254 } 255 } 256 257 static void 258 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 259 { 260 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 261 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 262 263 rw_enter(&mac_srs_g_lock, RW_WRITER); 264 mutex_enter(&mac_srs->srs_lock); 265 266 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 267 268 if (mac_srs_g_list == NULL) { 269 mac_srs_g_list = mac_srs; 270 } else { 271 mac_srs->srs_next = mac_srs_g_list; 272 mac_srs_g_list->srs_prev = mac_srs; 273 mac_srs->srs_prev = NULL; 274 mac_srs_g_list = mac_srs; 275 } 276 mac_srs->srs_state |= SRS_IN_GLIST; 277 278 mutex_exit(&mac_srs->srs_lock); 279 rw_exit(&mac_srs_g_lock); 280 } 281 282 static void 283 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 284 { 285 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 286 287 rw_enter(&mac_srs_g_lock, RW_WRITER); 288 mutex_enter(&mac_srs->srs_lock); 289 290 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 291 292 if (mac_srs == mac_srs_g_list) { 293 mac_srs_g_list = mac_srs->srs_next; 294 if (mac_srs_g_list != NULL) 295 mac_srs_g_list->srs_prev = NULL; 296 } else { 297 mac_srs->srs_prev->srs_next = mac_srs->srs_next; 298 if (mac_srs->srs_next != NULL) 299 mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 300 } 301 mac_srs->srs_state &= ~SRS_IN_GLIST; 302 303 mutex_exit(&mac_srs->srs_lock); 304 rw_exit(&mac_srs_g_lock); 305 } 306 307 /* POLLING SETUP AND TEAR DOWN ROUTINES */ 308 309 /* 310 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 311 * 312 * These routines are used to call back into the upper layer 313 * (primarily TCP squeue) to stop polling the soft rings or 314 * restart polling. 315 */ 316 void 317 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 318 mac_soft_ring_set_t *mac_srs) 319 { 320 mac_soft_ring_t *softring; 321 322 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 323 324 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 325 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 326 return; 327 } 328 329 for (softring = mac_srs->srs_soft_ring_head; 330 softring != NULL; softring = softring->s_ring_next) { 331 if ((softring->s_ring_type & ST_RING_TCP) && 332 (softring->s_ring_rx_arg2 != NULL)) { 333 mcip->mci_resource_quiesce(mcip->mci_resource_arg, 334 softring->s_ring_rx_arg2); 335 } 336 } 337 } 338 339 void 340 mac_srs_client_poll_restart(mac_client_impl_t *mcip, 341 mac_soft_ring_set_t *mac_srs) 342 { 343 mac_soft_ring_t *softring; 344 345 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 346 347 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 348 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 349 return; 350 } 351 352 for (softring = mac_srs->srs_soft_ring_head; 353 softring != NULL; softring = softring->s_ring_next) { 354 if ((softring->s_ring_type & ST_RING_TCP) && 355 (softring->s_ring_rx_arg2 != NULL)) { 356 mcip->mci_resource_restart(mcip->mci_resource_arg, 357 softring->s_ring_rx_arg2); 358 } 359 } 360 } 361 362 /* 363 * Register the given SRS and associated soft rings with the consumer and 364 * enable the polling interface used by the consumer.(i.e IP) over this 365 * SRS and associated soft rings. 366 */ 367 void 368 mac_srs_client_poll_enable(mac_client_impl_t *mcip, 369 mac_soft_ring_set_t *mac_srs) 370 { 371 mac_rx_fifo_t mrf; 372 mac_soft_ring_t *softring; 373 374 ASSERT(mac_srs->srs_mcip == mcip); 375 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 376 377 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 378 return; 379 380 bzero(&mrf, sizeof (mac_rx_fifo_t)); 381 mrf.mrf_type = MAC_RX_FIFO; 382 383 /* 384 * A SRS is capable of acting as a soft ring for cases 385 * where no fanout is needed. This is the case for userland 386 * flows. 387 */ 388 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 389 return; 390 391 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 392 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 393 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 394 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 395 396 softring = mac_srs->srs_soft_ring_head; 397 while (softring != NULL) { 398 if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 399 /* 400 * TCP and UDP support DLS bypass. Squeue polling 401 * support implies DLS bypass since the squeue poll 402 * path does not have DLS processing. 403 */ 404 mac_soft_ring_dls_bypass(softring, 405 mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 406 } 407 /* 408 * Non-TCP protocols don't support squeues. Hence we don't 409 * make any ring addition callbacks for non-TCP rings 410 */ 411 if (!(softring->s_ring_type & ST_RING_TCP)) { 412 softring->s_ring_rx_arg2 = NULL; 413 softring = softring->s_ring_next; 414 continue; 415 } 416 mrf.mrf_rx_arg = softring; 417 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 418 mrf.mrf_cpu_id = softring->s_ring_cpuid; 419 mrf.mrf_flow_priority = mac_srs->srs_pri; 420 421 softring->s_ring_rx_arg2 = mcip->mci_resource_add( 422 mcip->mci_resource_arg, (mac_resource_t *)&mrf); 423 424 softring = softring->s_ring_next; 425 } 426 } 427 428 /* 429 * Unregister the given SRS and associated soft rings with the consumer and 430 * disable the polling interface used by the consumer.(i.e IP) over this 431 * SRS and associated soft rings. 432 */ 433 void 434 mac_srs_client_poll_disable(mac_client_impl_t *mcip, 435 mac_soft_ring_set_t *mac_srs) 436 { 437 mac_soft_ring_t *softring; 438 439 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 440 441 /* 442 * A SRS is capable of acting as a soft ring for cases 443 * where no protocol fanout is needed. This is the case 444 * for userland flows. Nothing to do here. 445 */ 446 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 447 return; 448 449 mutex_enter(&mac_srs->srs_lock); 450 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 451 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 452 mutex_exit(&mac_srs->srs_lock); 453 return; 454 } 455 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 456 mutex_exit(&mac_srs->srs_lock); 457 458 /* 459 * DLS bypass is now disabled in the case of both TCP and UDP. 460 * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 461 * callback. In addition, in the case of TCP, invoke IP's callback 462 * for ring removal. 463 */ 464 for (softring = mac_srs->srs_soft_ring_head; 465 softring != NULL; softring = softring->s_ring_next) { 466 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 467 continue; 468 469 if ((softring->s_ring_type & ST_RING_TCP) && 470 softring->s_ring_rx_arg2 != NULL) { 471 mcip->mci_resource_remove(mcip->mci_resource_arg, 472 softring->s_ring_rx_arg2); 473 } 474 475 mutex_enter(&softring->s_ring_lock); 476 while (softring->s_ring_state & S_RING_PROC) { 477 softring->s_ring_state |= S_RING_CLIENT_WAIT; 478 cv_wait(&softring->s_ring_client_cv, 479 &softring->s_ring_lock); 480 } 481 softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 482 softring->s_ring_rx_arg2 = NULL; 483 softring->s_ring_rx_func = mac_rx_deliver; 484 softring->s_ring_rx_arg1 = mcip; 485 mutex_exit(&softring->s_ring_lock); 486 } 487 } 488 489 /* 490 * Enable or disable poll capability of the SRS on the underlying Rx ring. 491 * 492 * There is a need to enable or disable the poll capability of an SRS over an 493 * Rx ring depending on the number of mac clients sharing the ring and also 494 * whether user flows are configured on it. However the poll state is actively 495 * manipulated by the SRS worker and poll threads and uncoordinated changes by 496 * yet another thread to the underlying capability can surprise them leading 497 * to assert failures. Instead we quiesce the SRS, make the changes and then 498 * restart the SRS. 499 */ 500 static void 501 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 502 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 503 { 504 boolean_t need_restart = B_FALSE; 505 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 506 mac_ring_t *ring; 507 508 if (!SRS_QUIESCED(mac_srs)) { 509 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 510 need_restart = B_TRUE; 511 } 512 513 ring = mac_srs->srs_ring; 514 if ((ring != NULL) && 515 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 516 if (turn_off_poll_capab) 517 mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 518 else if (mac_poll_enable) 519 mac_srs->srs_state |= SRS_POLLING_CAPAB; 520 } 521 srs_rx->sr_lower_proc = rx_func; 522 523 if (need_restart) 524 mac_rx_srs_restart(mac_srs); 525 } 526 527 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 528 529 /* 530 * Return the next CPU to be used to bind a MAC kernel thread. 531 * If a cpupart is specified, the cpu chosen must be from that 532 * cpu partition. 533 */ 534 static processorid_t 535 mac_next_bind_cpu(cpupart_t *cpupart) 536 { 537 static cpu_t *cp = NULL; 538 cpu_t *cp_start; 539 540 ASSERT(MUTEX_HELD(&cpu_lock)); 541 542 if (cp == NULL) 543 cp = cpu_list; 544 545 cp = cp->cpu_next_onln; 546 cp_start = cp; 547 548 do { 549 if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 550 return (cp->cpu_id); 551 552 } while ((cp = cp->cpu_next_onln) != cp_start); 553 554 return (-1); /* No matching CPU found online */ 555 } 556 557 /* ARGSUSED */ 558 static int 559 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 560 { 561 ASSERT(MUTEX_HELD(&cpu_lock)); 562 switch (what) { 563 case CPU_CONFIG: 564 case CPU_ON: 565 case CPU_CPUPART_IN: 566 mac_walk_srs_and_bind(id); 567 break; 568 569 case CPU_UNCONFIG: 570 case CPU_OFF: 571 case CPU_CPUPART_OUT: 572 mac_walk_srs_and_unbind(id); 573 break; 574 575 default: 576 break; 577 } 578 return (0); 579 } 580 581 /* 582 * mac_compute_soft_ring_count(): 583 * 584 * This routine computes the number of soft rings needed to handle incoming 585 * load given a flow_entry. 586 * 587 * The routine does the following: 588 * 1) soft rings will be created if mac_soft_ring_enable is set. 589 * 2) If the underlying link is a 10Gbps link, then soft rings will be 590 * created even if mac_soft_ring_enable is not set. The number of soft 591 * rings, so created, will equal mac_rx_soft_ring_10gig_count. 592 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 593 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 594 * 10Gbps link. 595 * 596 * If a bandwidth limit is specified, the number that gets computed is 597 * dependent upon CPU speed, the number of Rx rings configured, and 598 * the bandwidth limit. 599 * If more Rx rings are available, less number of soft rings is needed. 600 * 601 * mac_use_bw_heuristic is another "hidden" variable that can be used to 602 * override the default use of soft ring count computation. Depending upon 603 * the usefulness of it, mac_use_bw_heuristic can later be made into a 604 * data-link property or removed altogether. 605 * 606 * TODO: Cleanup and tighten some of the assumptions. 607 */ 608 boolean_t mac_use_bw_heuristic = B_TRUE; 609 static int 610 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 611 { 612 uint64_t cpu_speed, bw = 0; 613 int srings = 0; 614 boolean_t bw_enabled = B_FALSE; 615 616 ASSERT(!(flent->fe_type & FLOW_USER)); 617 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 618 mac_use_bw_heuristic) { 619 /* bandwidth enabled */ 620 bw_enabled = B_TRUE; 621 bw = flent->fe_resource_props.mrp_maxbw; 622 } 623 if (!bw_enabled) { 624 /* No bandwidth enabled */ 625 if (mac_soft_ring_enable) 626 srings = mac_rx_soft_ring_count; 627 628 /* Is this a 10Gig link? */ 629 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 630 MAC_STAT_IFSPEED); 631 /* convert to Mbps */ 632 if (((flent->fe_nic_speed)/1000000) > 1000 && 633 mac_rx_soft_ring_10gig_count > 0) { 634 /* This is a 10Gig link */ 635 srings = mac_rx_soft_ring_10gig_count; 636 /* 637 * Use 2 times mac_rx_soft_ring_10gig_count for 638 * sun4v systems. 639 */ 640 if (mac_soft_ring_enable) 641 srings = srings * 2; 642 } 643 } else { 644 /* 645 * Soft ring computation using CPU speed and specified 646 * bandwidth limit. 647 */ 648 /* Assumption: all CPUs have the same frequency */ 649 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 650 651 /* cpu_speed is in MHz; make bw in units of Mbps. */ 652 bw = bw/1000000; 653 654 if (bw >= 1000) { 655 /* 656 * bw is greater than or equal to 1Gbps. 657 * The number of soft rings required is a function 658 * of bandwidth and CPU speed. To keep this simple, 659 * let's use this rule: 1GHz CPU can handle 1Gbps. 660 * If bw is less than 1 Gbps, then there is no need 661 * for soft rings. Assumption is that CPU speeds 662 * (on modern systems) are at least 1GHz. 663 */ 664 srings = bw/cpu_speed; 665 if (srings <= 1 && mac_soft_ring_enable) { 666 /* 667 * Give at least 2 soft rings 668 * for sun4v systems 669 */ 670 srings = 2; 671 } 672 } 673 } 674 /* 675 * If the flent has multiple Rx SRSs, then each SRS need not 676 * have that many soft rings on top of it. The number of 677 * soft rings for each Rx SRS is found by dividing srings by 678 * rx_srs_cnt. 679 */ 680 if (rx_srs_cnt > 1) { 681 int remainder; 682 683 remainder = srings%rx_srs_cnt; 684 srings = srings/rx_srs_cnt; 685 if (remainder != 0) 686 srings++; 687 /* 688 * Fanning out to 1 soft ring is not very useful. 689 * Set it as well to 0 and mac_srs_fanout_init() 690 * will take care of creating a single soft ring 691 * for proto fanout. 692 */ 693 if (srings == 1) 694 srings = 0; 695 } 696 /* Do some more massaging */ 697 srings = min(srings, maxcpus); 698 srings = min(srings, MAX_SR_FANOUT); 699 return (srings); 700 } 701 702 /* 703 * mac_tx_cpu_init: 704 * set up CPUs for Tx interrupt re-targeting and Tx worker 705 * thread binding 706 */ 707 static void 708 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 709 cpupart_t *cpupart) 710 { 711 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 712 mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 713 mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 714 mac_soft_ring_t *sringp; 715 mac_ring_t *ring; 716 processorid_t worker_cpuid; 717 boolean_t retargetable_client = B_FALSE; 718 int i, j; 719 720 if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 721 flent->fe_mcip)) { 722 retargetable_client = B_TRUE; 723 } 724 725 if (MAC_TX_SOFT_RINGS(tx_srs)) { 726 if (mrp != NULL) 727 j = mrp->mrp_ncpus - 1; 728 for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 729 if (mrp != NULL) { 730 if (j < 0) 731 j = mrp->mrp_ncpus - 1; 732 worker_cpuid = mrp->mrp_cpu[j]; 733 } else { 734 /* 735 * Bind interrupt to the next CPU available 736 * and leave the worker unbound. 737 */ 738 worker_cpuid = -1; 739 } 740 sringp = tx_srs->srs_tx_soft_rings[i]; 741 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 742 srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 743 if (MAC_RING_RETARGETABLE(ring) && 744 retargetable_client) { 745 mutex_enter(&cpu_lock); 746 srs_cpu->mc_tx_intr_cpu[i] = 747 (mrp != NULL) ? mrp->mrp_cpu[j] : 748 (mac_tx_intr_retarget ? 749 mac_next_bind_cpu(cpupart) : -1); 750 mutex_exit(&cpu_lock); 751 } else { 752 srs_cpu->mc_tx_intr_cpu[i] = -1; 753 } 754 if (mrp != NULL) 755 j--; 756 } 757 } else { 758 /* Tx mac_ring_handle_t is stored in st_arg2 */ 759 srs_cpu->mc_tx_fanout_cpus[0] = 760 (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 761 ring = (mac_ring_t *)srs_tx->st_arg2; 762 if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 763 mutex_enter(&cpu_lock); 764 srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 765 mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 766 (mac_tx_intr_retarget ? 767 mac_next_bind_cpu(cpupart) : -1); 768 mutex_exit(&cpu_lock); 769 } else { 770 srs_cpu->mc_tx_intr_cpu[0] = -1; 771 } 772 } 773 } 774 775 /* 776 * Assignment of user specified CPUs to a link. 777 * 778 * Minimum CPUs required to get an optimal assignmet: 779 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 780 * flag is set -- one for polling, one for fanout soft ring. 781 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 782 * for polling, one for SRS worker thread and one for fanout soft ring. 783 * 784 * The CPUs needed for Tx side is equal to the number of Tx rings 785 * the link is using. 786 * 787 * mac_flow_user_cpu_init() categorizes the CPU assignment depending 788 * upon the number of CPUs in 3 different buckets. 789 * 790 * In the first bucket, the most optimal case is handled. The user has 791 * passed enough number of CPUs and every thread gets its own CPU. 792 * 793 * The second and third are the sub-optimal cases. Enough CPUs are not 794 * available. 795 * 796 * The second bucket handles the case where atleast one distinct CPU is 797 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 798 * SRS or soft rings). 799 * 800 * In the third case (worst case scenario), specified CPU count is less 801 * than the Rx rings configured for the link. In this case, we round 802 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 803 */ 804 static void 805 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 806 { 807 mac_soft_ring_set_t *rx_srs, *tx_srs; 808 int i, srs_cnt; 809 mac_cpus_t *srs_cpu; 810 int no_of_cpus, cpu_cnt; 811 int rx_srs_cnt, reqd_rx_cpu_cnt; 812 int fanout_cpu_cnt, reqd_tx_cpu_cnt; 813 int reqd_poll_worker_cnt, fanout_cnt_per_srs; 814 mac_resource_props_t *emrp = &flent->fe_effective_props; 815 816 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 817 /* 818 * The check for nbc_ncpus to be within limits for 819 * the user specified case was done earlier and if 820 * not within limits, an error would have been 821 * returned to the user. 822 */ 823 ASSERT(mrp->mrp_ncpus > 0); 824 825 no_of_cpus = mrp->mrp_ncpus; 826 827 if (mrp->mrp_rx_intr_cpu != -1) { 828 /* 829 * interrupt has been re-targetted. Poll 830 * thread needs to be bound to interrupt 831 * CPU. 832 * 833 * Find where in the list is the intr 834 * CPU and swap it with the first one. 835 * We will be using the first CPU in the 836 * list for poll. 837 */ 838 for (i = 0; i < no_of_cpus; i++) { 839 if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 840 break; 841 } 842 mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 843 mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 844 } 845 846 /* 847 * Requirements: 848 * The number of CPUs that each Rx ring needs is dependent 849 * upon mac_latency_optimize flag. 850 * 1) If set, atleast 2 CPUs are needed -- one for 851 * polling, one for fanout soft ring. 852 * 2) If not set, then atleast 3 CPUs are needed -- one 853 * for polling, one for srs worker thread, and one for 854 * fanout soft ring. 855 */ 856 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 857 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 858 reqd_rx_cpu_cnt = mac_latency_optimize ? 859 (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 860 861 /* How many CPUs are needed for Tx side? */ 862 tx_srs = flent->fe_tx_srs; 863 reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 864 tx_srs->srs_tx_ring_count : 1; 865 866 /* CPUs needed for Rx SRSes poll and worker threads */ 867 reqd_poll_worker_cnt = mac_latency_optimize ? 868 rx_srs_cnt : rx_srs_cnt * 2; 869 870 /* Has the user provided enough CPUs? */ 871 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 872 /* 873 * Best case scenario. There is enough CPUs. All 874 * Rx rings will get their own set of CPUs plus 875 * Tx soft rings will get their own. 876 */ 877 /* 878 * fanout_cpu_cnt is the number of CPUs available 879 * for Rx side fanout soft rings. 880 */ 881 fanout_cpu_cnt = no_of_cpus - 882 reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 883 884 /* 885 * Divide fanout_cpu_cnt by rx_srs_cnt to find 886 * out how many fanout soft rings each Rx SRS 887 * can have. 888 */ 889 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 890 891 /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 892 fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 893 894 /* Do the assignment for the default Rx ring */ 895 cpu_cnt = 0; 896 rx_srs = flent->fe_rx_srs[0]; 897 ASSERT(rx_srs->srs_ring == NULL); 898 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 899 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 900 srs_cpu = &rx_srs->srs_cpu; 901 srs_cpu->mc_ncpus = no_of_cpus; 902 bcopy(mrp->mrp_cpu, 903 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 904 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 905 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 906 /* Retarget the interrupt to the same CPU as the poll */ 907 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 908 srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 909 srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 910 for (i = 0; i < fanout_cnt_per_srs; i++) 911 srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 912 913 /* Do the assignment for h/w Rx SRSes */ 914 if (flent->fe_rx_srs_cnt > 1) { 915 cpu_cnt = 0; 916 for (srs_cnt = 1; 917 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 918 rx_srs = flent->fe_rx_srs[srs_cnt]; 919 ASSERT(rx_srs->srs_ring != NULL); 920 if (rx_srs->srs_fanout_state == 921 SRS_FANOUT_INIT) { 922 rx_srs->srs_fanout_state = 923 SRS_FANOUT_REINIT; 924 } 925 srs_cpu = &rx_srs->srs_cpu; 926 srs_cpu->mc_ncpus = no_of_cpus; 927 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 928 sizeof (srs_cpu->mc_cpus)); 929 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 930 /* The first CPU in the list is the intr CPU */ 931 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 932 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 933 srs_cpu->mc_rx_workerid = 934 (mac_latency_optimize ? 935 srs_cpu->mc_rx_pollid : 936 mrp->mrp_cpu[cpu_cnt++]); 937 for (i = 0; i < fanout_cnt_per_srs; i++) { 938 srs_cpu->mc_rx_fanout_cpus[i] = 939 mrp->mrp_cpu[cpu_cnt++]; 940 } 941 ASSERT(cpu_cnt <= no_of_cpus); 942 } 943 } 944 goto tx_cpu_init; 945 } 946 947 /* 948 * Sub-optimal case. 949 * We have the following information: 950 * no_of_cpus - no. of cpus that user passed. 951 * rx_srs_cnt - no. of rx rings. 952 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 953 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 954 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 955 */ 956 /* 957 * If we bind the Rx fanout soft rings to the same CPUs 958 * as poll/worker, would that be enough? 959 */ 960 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 961 boolean_t worker_assign = B_FALSE; 962 963 /* 964 * If mac_latency_optimize is not set, are there 965 * enough CPUs to assign a CPU for worker also? 966 */ 967 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 968 worker_assign = B_TRUE; 969 /* 970 * Zero'th Rx SRS is the default Rx ring. It is not 971 * associated with h/w Rx ring. 972 */ 973 rx_srs = flent->fe_rx_srs[0]; 974 ASSERT(rx_srs->srs_ring == NULL); 975 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 976 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 977 cpu_cnt = 0; 978 srs_cpu = &rx_srs->srs_cpu; 979 srs_cpu->mc_ncpus = no_of_cpus; 980 bcopy(mrp->mrp_cpu, 981 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 982 srs_cpu->mc_rx_fanout_cnt = 1; 983 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 984 /* Retarget the interrupt to the same CPU as the poll */ 985 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 986 srs_cpu->mc_rx_workerid = 987 ((!mac_latency_optimize && worker_assign) ? 988 mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 989 990 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 991 992 /* Do CPU bindings for SRSes having h/w Rx rings */ 993 if (flent->fe_rx_srs_cnt > 1) { 994 cpu_cnt = 0; 995 for (srs_cnt = 1; 996 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 997 rx_srs = flent->fe_rx_srs[srs_cnt]; 998 ASSERT(rx_srs->srs_ring != NULL); 999 if (rx_srs->srs_fanout_state == 1000 SRS_FANOUT_INIT) { 1001 rx_srs->srs_fanout_state = 1002 SRS_FANOUT_REINIT; 1003 } 1004 srs_cpu = &rx_srs->srs_cpu; 1005 srs_cpu->mc_ncpus = no_of_cpus; 1006 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1007 sizeof (srs_cpu->mc_cpus)); 1008 srs_cpu->mc_rx_pollid = 1009 mrp->mrp_cpu[cpu_cnt]; 1010 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1011 srs_cpu->mc_rx_workerid = 1012 ((!mac_latency_optimize && worker_assign) ? 1013 mrp->mrp_cpu[++cpu_cnt] : 1014 srs_cpu->mc_rx_pollid); 1015 srs_cpu->mc_rx_fanout_cnt = 1; 1016 srs_cpu->mc_rx_fanout_cpus[0] = 1017 mrp->mrp_cpu[cpu_cnt]; 1018 cpu_cnt++; 1019 ASSERT(cpu_cnt <= no_of_cpus); 1020 } 1021 } 1022 goto tx_cpu_init; 1023 } 1024 1025 /* 1026 * Real sub-optimal case. Not enough CPUs for poll and 1027 * Tx soft rings. Do a round robin assignment where 1028 * each Rx SRS will get the same CPU for poll, worker 1029 * and fanout soft ring. 1030 */ 1031 cpu_cnt = 0; 1032 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1033 rx_srs = flent->fe_rx_srs[srs_cnt]; 1034 srs_cpu = &rx_srs->srs_cpu; 1035 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1036 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1037 srs_cpu->mc_ncpus = no_of_cpus; 1038 bcopy(mrp->mrp_cpu, 1039 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 1040 srs_cpu->mc_rx_fanout_cnt = 1; 1041 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 1042 /* Retarget the interrupt to the same CPU as the poll */ 1043 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1044 srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 1045 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1046 if (++cpu_cnt >= no_of_cpus) 1047 cpu_cnt = 0; 1048 } 1049 1050 tx_cpu_init: 1051 mac_tx_cpu_init(flent, mrp, NULL); 1052 1053 /* 1054 * Copy the user specified CPUs to the effective CPUs 1055 */ 1056 for (i = 0; i < mrp->mrp_ncpus; i++) { 1057 emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 1058 } 1059 emrp->mrp_ncpus = mrp->mrp_ncpus; 1060 emrp->mrp_mask = mrp->mrp_mask; 1061 bzero(emrp->mrp_pool, MAXPATHLEN); 1062 } 1063 1064 /* 1065 * mac_flow_cpu_init(): 1066 * 1067 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1068 * the CPU binding information in srs_cpu for all Rx SRSes associated 1069 * with a flent. 1070 */ 1071 static void 1072 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1073 { 1074 mac_soft_ring_set_t *rx_srs; 1075 processorid_t cpuid; 1076 int i, j, k, srs_cnt, maxcpus, soft_ring_cnt = 0; 1077 mac_cpus_t *srs_cpu; 1078 mac_resource_props_t *emrp = &flent->fe_effective_props; 1079 1080 /* 1081 * The maximum number of CPUs available can either be 1082 * the number of CPUs in the pool or the number of CPUs 1083 * in the system. 1084 */ 1085 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 1086 /* 1087 * We cannot exceed the hard limit imposed by data structures. 1088 * Leave space for polling CPU and the SRS worker thread when 1089 * "mac_latency_optimize" is not set. 1090 */ 1091 maxcpus = MIN(maxcpus, MRP_NCPUS - 2); 1092 1093 /* 1094 * Compute the number of soft rings needed on top for each Rx 1095 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1096 * associated with h/w Rx rings. Soft ring count needed for 1097 * each h/w Rx SRS is computed and the same is applied to 1098 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1099 * is the software classified Rx SRS. 1100 */ 1101 soft_ring_cnt = mac_compute_soft_ring_count(flent, 1102 flent->fe_rx_srs_cnt - 1, maxcpus); 1103 if (soft_ring_cnt == 0) { 1104 /* 1105 * Even when soft_ring_cnt is 0, we still need 1106 * to create a soft ring for TCP, UDP and 1107 * OTHER. So set it to 1. 1108 */ 1109 soft_ring_cnt = 1; 1110 } 1111 1112 emrp->mrp_ncpus = 0; 1113 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt && 1114 emrp->mrp_ncpus < MRP_NCPUS; srs_cnt++) { 1115 rx_srs = flent->fe_rx_srs[srs_cnt]; 1116 srs_cpu = &rx_srs->srs_cpu; 1117 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1118 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1119 srs_cpu->mc_ncpus = soft_ring_cnt; 1120 srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1121 mutex_enter(&cpu_lock); 1122 for (j = 0; j < soft_ring_cnt; j++) { 1123 cpuid = mac_next_bind_cpu(cpupart); 1124 srs_cpu->mc_cpus[j] = cpuid; 1125 srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1126 } 1127 cpuid = mac_next_bind_cpu(cpupart); 1128 srs_cpu->mc_rx_pollid = cpuid; 1129 srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 1130 srs_cpu->mc_rx_pollid : -1); 1131 /* increment ncpus to account for polling cpu */ 1132 srs_cpu->mc_ncpus++; 1133 srs_cpu->mc_cpus[j++] = cpuid; 1134 if (!mac_latency_optimize) { 1135 cpuid = mac_next_bind_cpu(cpupart); 1136 srs_cpu->mc_ncpus++; 1137 srs_cpu->mc_cpus[j++] = cpuid; 1138 } 1139 srs_cpu->mc_rx_workerid = cpuid; 1140 mutex_exit(&cpu_lock); 1141 1142 /* 1143 * Copy fanout CPUs to fe_effective_props without duplicates. 1144 */ 1145 for (i = 0; i < srs_cpu->mc_ncpus && 1146 emrp->mrp_ncpus < MRP_NCPUS; i++) { 1147 for (j = 0; j < emrp->mrp_ncpus; j++) { 1148 if (emrp->mrp_cpu[j] == srs_cpu->mc_cpus[i]) 1149 break; 1150 } 1151 if (j == emrp->mrp_ncpus) { 1152 emrp->mrp_cpu[emrp->mrp_ncpus++] = 1153 srs_cpu->mc_cpus[i]; 1154 } 1155 } 1156 } 1157 1158 mac_tx_cpu_init(flent, NULL, cpupart); 1159 } 1160 1161 /* 1162 * DATAPATH SETUP ROUTINES 1163 * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1164 */ 1165 1166 /* 1167 * mac_srs_fanout_list_alloc: 1168 * 1169 * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 1170 * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 1171 * array space is needed to store Tx soft rings. Thus we allocate so 1172 * much array space for srs_tx_soft_rings. 1173 * 1174 * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 1175 * of space to st_soft_rings. This array is used for quick access to 1176 * soft ring associated with a pseudo Tx ring based on the pseudo 1177 * ring's index (mr_index). 1178 */ 1179 static void 1180 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1181 { 1182 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1183 1184 if (mac_srs->srs_type & SRST_TX) { 1185 mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 1186 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1187 MAX_RINGS_PER_GROUP, KM_SLEEP); 1188 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 1189 mac_srs_tx_t *tx = &mac_srs->srs_tx; 1190 1191 tx->st_soft_rings = (mac_soft_ring_t **) 1192 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1193 MAX_RINGS_PER_GROUP, KM_SLEEP); 1194 } 1195 } else { 1196 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1197 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1198 KM_SLEEP); 1199 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1200 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1201 KM_SLEEP); 1202 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1203 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1204 KM_SLEEP); 1205 } 1206 } 1207 1208 static void 1209 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1210 { 1211 cpu_t *cp; 1212 boolean_t clear = B_FALSE; 1213 1214 ASSERT(MUTEX_HELD(&cpu_lock)); 1215 1216 if (!mac_srs_thread_bind) 1217 return; 1218 1219 cp = cpu_get(cpuid); 1220 if (cp == NULL || !cpu_is_online(cp)) 1221 return; 1222 1223 mutex_enter(&mac_srs->srs_lock); 1224 mac_srs->srs_state |= SRS_WORKER_BOUND; 1225 if (mac_srs->srs_worker_cpuid != -1) 1226 clear = B_TRUE; 1227 mac_srs->srs_worker_cpuid = cpuid; 1228 mutex_exit(&mac_srs->srs_lock); 1229 1230 if (clear) 1231 thread_affinity_clear(mac_srs->srs_worker); 1232 1233 thread_affinity_set(mac_srs->srs_worker, cpuid); 1234 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1235 } 1236 1237 static void 1238 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1239 { 1240 cpu_t *cp; 1241 boolean_t clear = B_FALSE; 1242 1243 ASSERT(MUTEX_HELD(&cpu_lock)); 1244 1245 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1246 return; 1247 1248 cp = cpu_get(cpuid); 1249 if (cp == NULL || !cpu_is_online(cp)) 1250 return; 1251 1252 mutex_enter(&mac_srs->srs_lock); 1253 mac_srs->srs_state |= SRS_POLL_BOUND; 1254 if (mac_srs->srs_poll_cpuid != -1) 1255 clear = B_TRUE; 1256 mac_srs->srs_poll_cpuid = cpuid; 1257 mutex_exit(&mac_srs->srs_lock); 1258 1259 if (clear) 1260 thread_affinity_clear(mac_srs->srs_poll_thr); 1261 1262 thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1263 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1264 } 1265 1266 /* 1267 * Re-target interrupt to the passed CPU. If re-target is successful, 1268 * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 1269 */ 1270 void 1271 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1272 { 1273 cpu_t *cp; 1274 mac_ring_t *ring = mac_srs->srs_ring; 1275 mac_intr_t *mintr = &ring->mr_info.mri_intr; 1276 flow_entry_t *flent = mac_srs->srs_flent; 1277 boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 1278 1279 ASSERT(MUTEX_HELD(&cpu_lock)); 1280 1281 /* 1282 * Don't re-target the interrupt for these cases: 1283 * 1) ring is NULL 1284 * 2) the interrupt is shared (mi_ddi_shared) 1285 * 3) ddi_handle is NULL and !primary 1286 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 1287 * Case 3 & 4 are because of mac_client_intr_cpu() routine. 1288 * This routine will re-target fixed interrupt for primary 1289 * mac client if the client has only one ring. In that 1290 * case, mc_rx_intr_cpu will already have the correct value. 1291 */ 1292 if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 1293 (mintr->mi_ddi_handle == NULL && !primary) || (primary && 1294 mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 1295 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1296 return; 1297 } 1298 1299 if (mintr->mi_ddi_handle == NULL) 1300 return; 1301 1302 cp = cpu_get(cpuid); 1303 if (cp == NULL || !cpu_is_online(cp)) 1304 return; 1305 1306 /* Drop the cpu_lock as set_intr_affinity() holds it */ 1307 mutex_exit(&cpu_lock); 1308 if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 1309 mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 1310 else 1311 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1312 mutex_enter(&cpu_lock); 1313 } 1314 1315 /* 1316 * Re-target Tx interrupts 1317 */ 1318 void 1319 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 1320 { 1321 cpu_t *cp; 1322 mac_ring_t *ring; 1323 mac_intr_t *mintr; 1324 mac_soft_ring_t *sringp; 1325 mac_srs_tx_t *srs_tx; 1326 mac_cpus_t *srs_cpu; 1327 processorid_t cpuid; 1328 int i; 1329 1330 ASSERT(MUTEX_HELD(&cpu_lock)); 1331 1332 srs_cpu = &mac_srs->srs_cpu; 1333 if (MAC_TX_SOFT_RINGS(mac_srs)) { 1334 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 1335 sringp = mac_srs->srs_tx_soft_rings[i]; 1336 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 1337 cpuid = srs_cpu->mc_tx_intr_cpu[i]; 1338 cp = cpu_get(cpuid); 1339 if (cp == NULL || !cpu_is_online(cp) || 1340 !MAC_RING_RETARGETABLE(ring)) { 1341 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1342 continue; 1343 } 1344 mintr = &ring->mr_info.mri_intr; 1345 /* 1346 * Drop the cpu_lock as set_intr_affinity() 1347 * holds it 1348 */ 1349 mutex_exit(&cpu_lock); 1350 if (set_intr_affinity(mintr->mi_ddi_handle, 1351 cpuid) == DDI_SUCCESS) { 1352 srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 1353 } else { 1354 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1355 } 1356 mutex_enter(&cpu_lock); 1357 } 1358 } else { 1359 cpuid = srs_cpu->mc_tx_intr_cpu[0]; 1360 cp = cpu_get(cpuid); 1361 if (cp == NULL || !cpu_is_online(cp)) { 1362 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1363 return; 1364 } 1365 srs_tx = &mac_srs->srs_tx; 1366 ring = (mac_ring_t *)srs_tx->st_arg2; 1367 if (MAC_RING_RETARGETABLE(ring)) { 1368 mintr = &ring->mr_info.mri_intr; 1369 mutex_exit(&cpu_lock); 1370 if ((set_intr_affinity(mintr->mi_ddi_handle, 1371 cpuid) == DDI_SUCCESS)) { 1372 srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 1373 } else { 1374 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1375 } 1376 mutex_enter(&cpu_lock); 1377 } 1378 } 1379 } 1380 1381 /* 1382 * When a CPU comes back online, bind the MAC kernel threads which 1383 * were previously bound to that CPU, and had to be unbound because 1384 * the CPU was going away. 1385 * 1386 * These functions are called with cpu_lock held and hence we can't 1387 * cv_wait to grab the mac perimeter. Since these functions walk the soft 1388 * ring list of an SRS without being in the perimeter, the list itself 1389 * is protected by the SRS lock. 1390 */ 1391 static void 1392 mac_walk_srs_and_bind(int cpuid) 1393 { 1394 mac_soft_ring_set_t *mac_srs; 1395 mac_soft_ring_t *soft_ring; 1396 1397 rw_enter(&mac_srs_g_lock, RW_READER); 1398 1399 if ((mac_srs = mac_srs_g_list) == NULL) 1400 goto done; 1401 1402 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1403 if (mac_srs->srs_worker_cpuid == -1 && 1404 mac_srs->srs_worker_cpuid_save == cpuid) { 1405 mac_srs->srs_worker_cpuid_save = -1; 1406 mac_srs_worker_bind(mac_srs, cpuid); 1407 } 1408 1409 if (!(mac_srs->srs_type & SRST_TX)) { 1410 if (mac_srs->srs_poll_cpuid == -1 && 1411 mac_srs->srs_poll_cpuid_save == cpuid) { 1412 mac_srs->srs_poll_cpuid_save = -1; 1413 mac_srs_poll_bind(mac_srs, cpuid); 1414 } 1415 } 1416 1417 /* Next tackle the soft rings associated with the srs */ 1418 mutex_enter(&mac_srs->srs_lock); 1419 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1420 soft_ring = soft_ring->s_ring_next) { 1421 if (soft_ring->s_ring_cpuid == -1 && 1422 soft_ring->s_ring_cpuid_save == cpuid) { 1423 soft_ring->s_ring_cpuid_save = -1; 1424 (void) mac_soft_ring_bind(soft_ring, cpuid); 1425 } 1426 } 1427 mutex_exit(&mac_srs->srs_lock); 1428 } 1429 done: 1430 rw_exit(&mac_srs_g_lock); 1431 } 1432 1433 /* 1434 * Change the priority of the SRS's poll and worker thread. Additionally, 1435 * update the priority of the worker threads for the SRS's soft rings. 1436 * Need to modify any associated squeue threads. 1437 */ 1438 void 1439 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1440 { 1441 mac_soft_ring_t *ringp; 1442 1443 mac_srs->srs_pri = prival; 1444 thread_lock(mac_srs->srs_worker); 1445 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1446 thread_unlock(mac_srs->srs_worker); 1447 if (mac_srs->srs_poll_thr != NULL) { 1448 thread_lock(mac_srs->srs_poll_thr); 1449 (void) thread_change_pri(mac_srs->srs_poll_thr, 1450 mac_srs->srs_pri, 0); 1451 thread_unlock(mac_srs->srs_poll_thr); 1452 } 1453 if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1454 return; 1455 while (ringp != mac_srs->srs_soft_ring_tail) { 1456 thread_lock(ringp->s_ring_worker); 1457 (void) thread_change_pri(ringp->s_ring_worker, 1458 mac_srs->srs_pri, 0); 1459 thread_unlock(ringp->s_ring_worker); 1460 ringp = ringp->s_ring_next; 1461 } 1462 ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1463 thread_lock(ringp->s_ring_worker); 1464 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1465 thread_unlock(ringp->s_ring_worker); 1466 } 1467 1468 /* 1469 * Change the receive bandwidth limit. 1470 */ 1471 static void 1472 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1473 { 1474 mac_soft_ring_t *softring; 1475 1476 mutex_enter(&srs->srs_lock); 1477 mutex_enter(&srs->srs_bw->mac_bw_lock); 1478 1479 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1480 /* Reset bandwidth limit */ 1481 if (srs->srs_type & SRST_BW_CONTROL) { 1482 softring = srs->srs_soft_ring_head; 1483 while (softring != NULL) { 1484 softring->s_ring_type &= ~ST_RING_BW_CTL; 1485 softring = softring->s_ring_next; 1486 } 1487 srs->srs_type &= ~SRST_BW_CONTROL; 1488 srs->srs_drain_func = mac_rx_srs_drain; 1489 } 1490 } else { 1491 /* Set/Modify bandwidth limit */ 1492 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1493 /* 1494 * Give twice the queuing capability before 1495 * dropping packets. The unit is bytes/tick. 1496 */ 1497 srs->srs_bw->mac_bw_drop_threshold = 1498 srs->srs_bw->mac_bw_limit << 1; 1499 if (!(srs->srs_type & SRST_BW_CONTROL)) { 1500 softring = srs->srs_soft_ring_head; 1501 while (softring != NULL) { 1502 softring->s_ring_type |= ST_RING_BW_CTL; 1503 softring = softring->s_ring_next; 1504 } 1505 srs->srs_type |= SRST_BW_CONTROL; 1506 srs->srs_drain_func = mac_rx_srs_drain_bw; 1507 } 1508 } 1509 done: 1510 mutex_exit(&srs->srs_bw->mac_bw_lock); 1511 mutex_exit(&srs->srs_lock); 1512 } 1513 1514 /* Change the transmit bandwidth limit */ 1515 static void 1516 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1517 { 1518 uint32_t tx_mode, ring_info = 0; 1519 mac_srs_tx_t *srs_tx = &srs->srs_tx; 1520 mac_client_impl_t *mcip = srs->srs_mcip; 1521 1522 /* 1523 * We need to quiesce/restart the client here because mac_tx() and 1524 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1525 * st_mode and related fields, which are modified by the code below. 1526 */ 1527 mac_tx_client_quiesce((mac_client_handle_t)mcip); 1528 1529 mutex_enter(&srs->srs_lock); 1530 mutex_enter(&srs->srs_bw->mac_bw_lock); 1531 1532 tx_mode = srs_tx->st_mode; 1533 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1534 /* Reset bandwidth limit */ 1535 if (tx_mode == SRS_TX_BW) { 1536 if (srs_tx->st_arg2 != NULL) 1537 ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1538 if (mac_tx_serialize || 1539 (ring_info & MAC_RING_TX_SERIALIZE)) { 1540 srs_tx->st_mode = SRS_TX_SERIALIZE; 1541 } else { 1542 srs_tx->st_mode = SRS_TX_DEFAULT; 1543 } 1544 } else if (tx_mode == SRS_TX_BW_FANOUT) { 1545 srs_tx->st_mode = SRS_TX_FANOUT; 1546 } else if (tx_mode == SRS_TX_BW_AGGR) { 1547 srs_tx->st_mode = SRS_TX_AGGR; 1548 } 1549 srs->srs_type &= ~SRST_BW_CONTROL; 1550 } else { 1551 /* Set/Modify bandwidth limit */ 1552 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1553 /* 1554 * Give twice the queuing capability before 1555 * dropping packets. The unit is bytes/tick. 1556 */ 1557 srs->srs_bw->mac_bw_drop_threshold = 1558 srs->srs_bw->mac_bw_limit << 1; 1559 srs->srs_type |= SRST_BW_CONTROL; 1560 if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 1561 tx_mode != SRS_TX_BW_AGGR) { 1562 if (tx_mode == SRS_TX_SERIALIZE || 1563 tx_mode == SRS_TX_DEFAULT) { 1564 srs_tx->st_mode = SRS_TX_BW; 1565 } else if (tx_mode == SRS_TX_FANOUT) { 1566 srs_tx->st_mode = SRS_TX_BW_FANOUT; 1567 } else if (tx_mode == SRS_TX_AGGR) { 1568 srs_tx->st_mode = SRS_TX_BW_AGGR; 1569 } else { 1570 ASSERT(0); 1571 } 1572 } 1573 } 1574 done: 1575 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1576 mutex_exit(&srs->srs_bw->mac_bw_lock); 1577 mutex_exit(&srs->srs_lock); 1578 1579 mac_tx_client_restart((mac_client_handle_t)mcip); 1580 } 1581 1582 /* 1583 * The uber function that deals with any update to bandwidth limits. 1584 */ 1585 void 1586 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1587 { 1588 int count; 1589 1590 for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1591 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1592 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1593 } 1594 1595 /* 1596 * When the first sub-flow is added to a link, we disable polling on the 1597 * link and also modify the entry point to mac_rx_srs_subflow_process(). 1598 * (polling is disabled because with the subflow added, accounting 1599 * for polling needs additional logic, it is assumed that when a subflow is 1600 * added, we can take some hit as a result of disabling polling rather than 1601 * adding more complexity - if this becomes a perf. issue we need to 1602 * re-rvaluate this logic). When the last subflow is removed, we turn back 1603 * polling and also reset the entry point to mac_rx_srs_process(). 1604 * 1605 * In the future if there are multiple SRS, we can simply 1606 * take one and give it to the flow rather than disabling polling and 1607 * resetting the entry point. 1608 */ 1609 void 1610 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1611 { 1612 flow_entry_t *flent = mcip->mci_flent; 1613 int i; 1614 mac_impl_t *mip = mcip->mci_mip; 1615 mac_rx_func_t rx_func; 1616 uint_t rx_srs_cnt; 1617 boolean_t enable_classifier; 1618 1619 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1620 1621 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1622 1623 rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1624 mac_rx_srs_process; 1625 1626 /* Tell mac_srs_poll_state_change to disable polling if necessary */ 1627 if (mip->mi_state_flags & MIS_POLL_DISABLE) 1628 enable_classifier = B_TRUE; 1629 1630 /* 1631 * If receive function has already been configured correctly for 1632 * current subflow configuration, do nothing. 1633 */ 1634 if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1635 return; 1636 1637 rx_srs_cnt = flent->fe_rx_srs_cnt; 1638 for (i = 0; i < rx_srs_cnt; i++) { 1639 ASSERT(flent->fe_rx_srs[i] != NULL); 1640 mac_srs_poll_state_change(flent->fe_rx_srs[i], 1641 enable_classifier, rx_func); 1642 } 1643 1644 /* 1645 * Change the S/W classifier so that we can land in the 1646 * correct processing function with correct argument. 1647 * If all subflows have been removed we can revert to 1648 * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process(). 1649 */ 1650 mutex_enter(&flent->fe_lock); 1651 flent->fe_cb_fn = (flow_fn_t)rx_func; 1652 flent->fe_cb_arg1 = (void *)mip; 1653 flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1654 mutex_exit(&flent->fe_lock); 1655 } 1656 1657 static void 1658 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1659 { 1660 int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1661 mac_soft_ring_t *softring; 1662 1663 softring = mac_srs->srs_soft_ring_head; 1664 if (softring == NULL) { 1665 ASSERT(mac_srs->srs_soft_ring_count == 0); 1666 mac_srs->srs_tcp_ring_count = 0; 1667 mac_srs->srs_udp_ring_count = 0; 1668 mac_srs->srs_oth_ring_count = 0; 1669 mac_srs->srs_tx_ring_count = 0; 1670 return; 1671 } 1672 1673 while (softring != NULL) { 1674 if (softring->s_ring_type & ST_RING_TCP) { 1675 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1676 } else if (softring->s_ring_type & ST_RING_UDP) { 1677 mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1678 } else if (softring->s_ring_type & ST_RING_OTH) { 1679 mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1680 } else { 1681 ASSERT(softring->s_ring_type & ST_RING_TX); 1682 mac_srs->srs_tx_soft_rings[tx_count++] = softring; 1683 } 1684 softring = softring->s_ring_next; 1685 } 1686 1687 ASSERT(mac_srs->srs_soft_ring_count == 1688 (tcp_count + udp_count + oth_count + tx_count)); 1689 mac_srs->srs_tcp_ring_count = tcp_count; 1690 mac_srs->srs_udp_ring_count = udp_count; 1691 mac_srs->srs_oth_ring_count = oth_count; 1692 mac_srs->srs_tx_ring_count = tx_count; 1693 } 1694 1695 void 1696 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 1697 mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1698 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1699 mac_resource_handle_t x_arg2, boolean_t set_bypass) 1700 { 1701 mac_soft_ring_t *softring; 1702 mac_rx_fifo_t mrf; 1703 1704 bzero(&mrf, sizeof (mac_rx_fifo_t)); 1705 mrf.mrf_type = MAC_RX_FIFO; 1706 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1707 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 1708 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1709 mrf.mrf_flow_priority = pri; 1710 1711 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1712 (type|ST_RING_TCP), pri, mcip, mac_srs, 1713 cpuid, rx_func, x_arg1, x_arg2); 1714 softring->s_ring_rx_arg2 = NULL; 1715 1716 /* 1717 * TCP and UDP support DLS bypass. In addition TCP 1718 * squeue can also poll their corresponding soft rings. 1719 */ 1720 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1721 mac_soft_ring_dls_bypass(softring, 1722 mcip->mci_direct_rx_fn, 1723 mcip->mci_direct_rx_arg); 1724 1725 mrf.mrf_rx_arg = softring; 1726 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1727 1728 /* 1729 * Make a call in IP to get a TCP squeue assigned to 1730 * this softring to maintain full CPU locality through 1731 * the stack and allow the squeue to be able to poll 1732 * the softring so the flow control can be pushed 1733 * all the way to H/W. 1734 */ 1735 softring->s_ring_rx_arg2 = 1736 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1737 (mac_resource_t *)&mrf); 1738 } 1739 1740 /* 1741 * Non-TCP protocols don't support squeues. Hence we 1742 * don't make any ring addition callbacks for non-TCP 1743 * rings. Now create the UDP softring and allow it to 1744 * bypass the DLS layer. 1745 */ 1746 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1747 (type|ST_RING_UDP), pri, mcip, mac_srs, 1748 cpuid, rx_func, x_arg1, x_arg2); 1749 softring->s_ring_rx_arg2 = NULL; 1750 1751 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1752 mac_soft_ring_dls_bypass(softring, 1753 mcip->mci_direct_rx_fn, 1754 mcip->mci_direct_rx_arg); 1755 } 1756 1757 /* Create the Oth softrings which has to go through the DLS */ 1758 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1759 (type|ST_RING_OTH), pri, mcip, mac_srs, 1760 cpuid, rx_func, x_arg1, x_arg2); 1761 softring->s_ring_rx_arg2 = NULL; 1762 } 1763 1764 /* 1765 * This routine associates a CPU or a set of CPU to process incoming 1766 * traffic from a mac client. If multiple CPUs are specified, then 1767 * so many soft rings are created with each soft ring worker thread 1768 * bound to a CPU in the set. Each soft ring in turn will be 1769 * associated with an squeue and the squeue will be moved to the 1770 * same CPU as that of the soft ring's. 1771 */ 1772 static void 1773 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 1774 void *x_arg1, mac_resource_handle_t x_arg2, 1775 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1776 { 1777 mac_soft_ring_t *softring; 1778 uint32_t soft_ring_flag = 0; 1779 processorid_t cpuid = -1; 1780 int i, srings_present, new_fanout_cnt; 1781 mac_cpus_t *srs_cpu; 1782 1783 /* fanout state is REINIT. Set it back to INIT */ 1784 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1785 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1786 1787 /* how many are present right now */ 1788 srings_present = mac_rx_srs->srs_tcp_ring_count; 1789 /* new request */ 1790 srs_cpu = &mac_rx_srs->srs_cpu; 1791 new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1792 1793 mutex_enter(&mac_rx_srs->srs_lock); 1794 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1795 soft_ring_flag |= ST_RING_BW_CTL; 1796 mutex_exit(&mac_rx_srs->srs_lock); 1797 1798 if (new_fanout_cnt > srings_present) { 1799 /* soft rings increased */ 1800 mutex_enter(&mac_rx_srs->srs_lock); 1801 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1802 mutex_exit(&mac_rx_srs->srs_lock); 1803 1804 for (i = mac_rx_srs->srs_tcp_ring_count; 1805 i < new_fanout_cnt; i++) { 1806 /* 1807 * Create the protocol softrings and set the 1808 * DLS bypass where possible. 1809 */ 1810 mac_srs_create_proto_softrings(i, soft_ring_flag, 1811 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1812 rx_func, x_arg1, x_arg2, B_TRUE); 1813 } 1814 mac_srs_update_fanout_list(mac_rx_srs); 1815 } else if (new_fanout_cnt < srings_present) { 1816 /* soft rings decreased */ 1817 if (new_fanout_cnt == 1) { 1818 mutex_enter(&mac_rx_srs->srs_lock); 1819 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1820 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1821 mutex_exit(&mac_rx_srs->srs_lock); 1822 } 1823 /* Get rid of extra soft rings */ 1824 for (i = new_fanout_cnt; 1825 i < mac_rx_srs->srs_tcp_ring_count; i++) { 1826 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1827 if (softring->s_ring_rx_arg2 != NULL) { 1828 mcip->mci_resource_remove( 1829 (void *)mcip->mci_resource_arg, 1830 softring->s_ring_rx_arg2); 1831 } 1832 mac_soft_ring_remove(mac_rx_srs, 1833 mac_rx_srs->srs_tcp_soft_rings[i]); 1834 mac_soft_ring_remove(mac_rx_srs, 1835 mac_rx_srs->srs_udp_soft_rings[i]); 1836 mac_soft_ring_remove(mac_rx_srs, 1837 mac_rx_srs->srs_oth_soft_rings[i]); 1838 } 1839 mac_srs_update_fanout_list(mac_rx_srs); 1840 } 1841 1842 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1843 mutex_enter(&cpu_lock); 1844 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1845 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1846 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1847 cpuid); 1848 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1849 cpuid); 1850 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1851 cpuid); 1852 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1853 if (softring->s_ring_rx_arg2 != NULL) { 1854 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1855 softring->s_ring_rx_arg2, cpuid); 1856 } 1857 } 1858 1859 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1860 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1861 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1862 /* 1863 * Bind Tx srs and soft ring threads too. Let's bind tx 1864 * srs to the last cpu in mrp list. 1865 */ 1866 if (mac_tx_srs != NULL) { 1867 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1868 mac_tx_srs_retarget_intr(mac_tx_srs); 1869 } 1870 mutex_exit(&cpu_lock); 1871 } 1872 1873 /* 1874 * Bind SRS threads and soft rings to CPUs/create fanout list. 1875 */ 1876 void 1877 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 1878 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1879 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 1880 cpupart_t *cpupart) 1881 { 1882 int i; 1883 processorid_t cpuid; 1884 uint32_t soft_ring_flag = 0; 1885 int soft_ring_cnt; 1886 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1887 1888 /* 1889 * Remove the no soft ring flag and we will adjust it 1890 * appropriately further down. 1891 */ 1892 mutex_enter(&mac_rx_srs->srs_lock); 1893 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1894 mutex_exit(&mac_rx_srs->srs_lock); 1895 1896 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1897 1898 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1899 soft_ring_flag |= ST_RING_BW_CTL; 1900 1901 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1902 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1903 /* 1904 * Ring count can be 0 if no fanout is required and no cpu 1905 * were specified. Leave the SRS worker and poll thread 1906 * unbound 1907 */ 1908 ASSERT(mrp != NULL); 1909 soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1910 1911 /* Step 1: bind cpu contains cpu list where threads need to bind */ 1912 if (soft_ring_cnt > 0) { 1913 mutex_enter(&cpu_lock); 1914 for (i = 0; i < soft_ring_cnt; i++) { 1915 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1916 /* Create the protocol softrings */ 1917 mac_srs_create_proto_softrings(i, soft_ring_flag, 1918 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1919 rx_func, x_arg1, x_arg2, B_FALSE); 1920 } 1921 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1922 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1923 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1924 /* 1925 * Bind Tx srs and soft ring threads too. 1926 * Let's bind tx srs to the last cpu in 1927 * mrp list. 1928 */ 1929 if (mac_tx_srs == NULL) { 1930 mutex_exit(&cpu_lock); 1931 goto alldone; 1932 } 1933 1934 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1935 mac_tx_srs_retarget_intr(mac_tx_srs); 1936 mutex_exit(&cpu_lock); 1937 } else { 1938 mutex_enter(&cpu_lock); 1939 /* 1940 * For a subflow, mrp_workerid and mrp_pollid 1941 * is not set. 1942 */ 1943 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 1944 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1945 mutex_exit(&cpu_lock); 1946 goto no_softrings; 1947 } 1948 1949 alldone: 1950 if (soft_ring_cnt > 1) 1951 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1952 mac_srs_update_fanout_list(mac_rx_srs); 1953 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1954 return; 1955 1956 no_softrings: 1957 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1958 mutex_enter(&cpu_lock); 1959 cpuid = mac_next_bind_cpu(cpupart); 1960 /* Create the protocol softrings */ 1961 mac_srs_create_proto_softrings(0, soft_ring_flag, 1962 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1963 rx_func, x_arg1, x_arg2, B_FALSE); 1964 mutex_exit(&cpu_lock); 1965 } else { 1966 /* 1967 * This is the case when there is no fanout which is 1968 * true for subflows. 1969 */ 1970 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1971 } 1972 mac_srs_update_fanout_list(mac_rx_srs); 1973 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1974 } 1975 1976 /* 1977 * Calls mac_srs_fanout_init() or modify() depending upon whether 1978 * the SRS is getting initialized or re-initialized. 1979 */ 1980 void 1981 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1982 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1983 mac_resource_handle_t x_arg2, cpupart_t *cpupart) 1984 { 1985 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1986 int i, rx_srs_cnt; 1987 1988 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1989 1990 /* 1991 * Aggr ports do not have SRSes. This function should never be 1992 * called on an aggr port. 1993 */ 1994 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 1995 mac_rx_srs = flent->fe_rx_srs[0]; 1996 1997 /* 1998 * Set up the fanout on the tx side only once, with the 1999 * first rx SRS. The CPU binding, fanout, and bandwidth 2000 * criteria are common to both RX and TX, so 2001 * initializing them along side avoids redundant code. 2002 */ 2003 mac_tx_srs = flent->fe_tx_srs; 2004 rx_srs_cnt = flent->fe_rx_srs_cnt; 2005 2006 /* No fanout for subflows */ 2007 if (flent->fe_type & FLOW_USER) { 2008 mac_srs_fanout_init(mcip, mrp, rx_func, 2009 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2010 cpupart); 2011 return; 2012 } 2013 2014 if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 2015 mac_flow_user_cpu_init(flent, mrp); 2016 else 2017 mac_flow_cpu_init(flent, cpupart); 2018 2019 mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2020 2021 /* 2022 * Set up fanout for both SW (0th SRS) and HW classified 2023 * SRS (the rest of Rx SRSs in flent). 2024 */ 2025 for (i = 0; i < rx_srs_cnt; i++) { 2026 mac_rx_srs = flent->fe_rx_srs[i]; 2027 if (i != 0) 2028 mac_tx_srs = NULL; 2029 switch (mac_rx_srs->srs_fanout_state) { 2030 case SRS_FANOUT_UNINIT: 2031 mac_srs_fanout_init(mcip, mrp, rx_func, 2032 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2033 cpupart); 2034 break; 2035 case SRS_FANOUT_INIT: 2036 break; 2037 case SRS_FANOUT_REINIT: 2038 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 2039 mac_srs_fanout_modify(mcip, rx_func, x_arg1, 2040 x_arg2, mac_rx_srs, mac_tx_srs); 2041 mac_rx_srs_restart(mac_rx_srs); 2042 break; 2043 default: 2044 VERIFY(mac_rx_srs->srs_fanout_state <= 2045 SRS_FANOUT_REINIT); 2046 break; 2047 } 2048 } 2049 } 2050 2051 /* 2052 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2053 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2054 * processing is created. 2055 * 2056 * Details on Rx SRS: 2057 * Create a SRS and also add the necessary soft rings for TCP and 2058 * non-TCP based on fanout type and count specified. 2059 * 2060 * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2061 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2062 * to be heavily modified. 2063 * 2064 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2065 */ 2066 mac_soft_ring_set_t * 2067 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2068 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2069 mac_ring_t *ring) 2070 { 2071 mac_soft_ring_set_t *mac_srs; 2072 mac_srs_rx_t *srs_rx; 2073 mac_srs_tx_t *srs_tx; 2074 mac_bw_ctl_t *mac_bw; 2075 mac_resource_props_t *mrp; 2076 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2077 2078 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2079 bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2080 srs_rx = &mac_srs->srs_rx; 2081 srs_tx = &mac_srs->srs_tx; 2082 2083 mutex_enter(&flent->fe_lock); 2084 2085 /* 2086 * Get the bandwidth control structure from the flent. Get 2087 * rid of any residual values in the control structure for 2088 * the tx bw struct and also for the rx, if the rx srs is 2089 * the 1st one being brought up (the rx bw ctl struct may 2090 * be shared by multiple SRSs) 2091 */ 2092 if (is_tx_srs) { 2093 mac_srs->srs_bw = &flent->fe_tx_bw; 2094 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2095 flent->fe_tx_srs = mac_srs; 2096 } else { 2097 /* 2098 * The bw counter (stored in the flent) is shared 2099 * by SRS's within an rx group. 2100 */ 2101 mac_srs->srs_bw = &flent->fe_rx_bw; 2102 /* First rx SRS, clear the bw structure */ 2103 if (flent->fe_rx_srs_cnt == 0) 2104 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2105 2106 /* 2107 * It is better to panic here rather than just assert because 2108 * on a non-debug kernel we might end up courrupting memory 2109 * and making it difficult to debug. 2110 */ 2111 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 2112 panic("Array Overrun detected due to MAC client %p " 2113 " having more rings than %d", (void *)mcip, 2114 MAX_RINGS_PER_GROUP); 2115 } 2116 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2117 flent->fe_rx_srs_cnt++; 2118 } 2119 mac_srs->srs_flent = flent; 2120 mutex_exit(&flent->fe_lock); 2121 2122 mac_srs->srs_state = 0; 2123 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2124 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2125 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 2126 mac_srs->srs_mcip = mcip; 2127 mac_srs_fanout_list_alloc(mac_srs); 2128 2129 /* 2130 * For a flow we use the underlying MAC client's priority range with 2131 * the priority value to find an absolute priority value. For a MAC 2132 * client we use the MAC client's maximum priority as the value. 2133 */ 2134 mrp = &flent->fe_effective_props; 2135 if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2136 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2137 mcip->mci_max_pri, mrp->mrp_priority); 2138 } else { 2139 mac_srs->srs_pri = mcip->mci_max_pri; 2140 } 2141 /* 2142 * We need to insert the SRS in the global list before 2143 * binding the SRS and SR threads. Otherwise there is a 2144 * is a small window where the cpu reconfig callbacks 2145 * may miss the SRS in the list walk and DR could fail 2146 * as there are bound threads. 2147 */ 2148 mac_srs_add_glist(mac_srs); 2149 2150 /* Initialize bw limit */ 2151 if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2152 mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2153 2154 mac_bw = mac_srs->srs_bw; 2155 mutex_enter(&mac_bw->mac_bw_lock); 2156 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2157 2158 /* 2159 * Give twice the queuing capability before 2160 * dropping packets. The unit is bytes/tick. 2161 */ 2162 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2163 mutex_exit(&mac_bw->mac_bw_lock); 2164 mac_srs->srs_type |= SRST_BW_CONTROL; 2165 } else { 2166 mac_srs->srs_drain_func = mac_rx_srs_drain; 2167 } 2168 2169 /* 2170 * We use the following policy to control Receive 2171 * Side Dynamic Polling: 2172 * 1) We switch to poll mode anytime the processing thread causes 2173 * a backlog to build up in SRS and its associated Soft Rings 2174 * (sr_poll_pkt_cnt > 0). 2175 * 2) As long as the backlog stays under the low water mark 2176 * (sr_lowat), we poll the H/W for more packets. 2177 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2178 * stay in poll mode but don't poll the H/W for more packets. 2179 * 4) Anytime in polling mode, if we poll the H/W for packets and 2180 * find nothing plus we have an existing backlog 2181 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2182 * the H/W for packets anymore (let the polling thread go to sleep). 2183 * 5) Once the backlog is relieved (packets are processed) we reenable 2184 * polling (by signalling the poll thread) only when the backlog 2185 * dips below sr_poll_thres. 2186 * 6) sr_hiwat is used exclusively when we are not polling capable 2187 * and is used to decide when to drop packets so the SRS queue 2188 * length doesn't grow infinitely. 2189 */ 2190 if (!is_tx_srs) { 2191 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2192 /* Low water mark needs to be less than high water mark */ 2193 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2194 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2195 (mac_soft_ring_max_q_cnt >> 2); 2196 /* Poll threshold need to be half of low water mark or less */ 2197 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2198 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2199 (srs_rx->sr_lowat >> 1); 2200 if (mac_latency_optimize) 2201 mac_srs->srs_state |= SRS_LATENCY_OPT; 2202 else 2203 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2204 } 2205 2206 /* 2207 * Create the srs_worker with twice the stack of a normal kernel thread 2208 * to reduce the likelihood of stack overflows in receive-side 2209 * processing. (The larger stacks are not the only precaution taken 2210 * against stack overflows; see the use of mac_rx_srs_stack_needed 2211 * in mac_sched.c). 2212 */ 2213 mac_srs->srs_worker = thread_create(NULL, default_stksize << 1, 2214 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2215 2216 if (is_tx_srs) { 2217 /* Handle everything about Tx SRS and return */ 2218 mac_srs->srs_drain_func = mac_tx_srs_drain; 2219 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2220 srs_tx->st_hiwat = 2221 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2222 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2223 srs_tx->st_arg1 = x_arg1; 2224 srs_tx->st_arg2 = x_arg2; 2225 goto done; 2226 } 2227 2228 if ((srs_type & SRST_FLOW) != 0 || 2229 FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2230 srs_rx->sr_lower_proc = mac_rx_srs_process; 2231 else 2232 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2233 2234 srs_rx->sr_func = rx_func; 2235 srs_rx->sr_arg1 = x_arg1; 2236 srs_rx->sr_arg2 = x_arg2; 2237 2238 if (ring != NULL) { 2239 uint_t ring_info; 2240 2241 /* Is the mac_srs created over the RX default group? */ 2242 if (ring->mr_gh == (mac_group_handle_t) 2243 MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2244 mac_srs->srs_type |= SRST_DEFAULT_GRP; 2245 } 2246 mac_srs->srs_ring = ring; 2247 ring->mr_srs = mac_srs; 2248 ring->mr_classify_type = MAC_HW_CLASSIFIER; 2249 ring->mr_flag |= MR_INCIPIENT; 2250 2251 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 2252 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2253 mac_srs->srs_state |= SRS_POLLING_CAPAB; 2254 2255 mac_srs->srs_poll_thr = thread_create(NULL, 0, 2256 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2257 mac_srs->srs_pri); 2258 /* 2259 * Some drivers require serialization and don't send 2260 * packet chains in interrupt context. For such 2261 * drivers, we should always queue in the soft ring 2262 * so that we get a chance to switch into polling 2263 * mode under backlog. 2264 */ 2265 ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 2266 if (ring_info & MAC_RING_RX_ENQUEUE) 2267 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2268 } 2269 done: 2270 mac_srs_stat_create(mac_srs); 2271 return (mac_srs); 2272 } 2273 2274 /* 2275 * Figure out the number of soft rings required. Its dependant on 2276 * if protocol fanout is required (for LINKs), global settings 2277 * require us to do fanout for performance (based on mac_soft_ring_enable), 2278 * or user has specifically requested fanout. 2279 */ 2280 static uint32_t 2281 mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2282 { 2283 uint32_t fanout_type; 2284 mac_resource_props_t *mrp = &flent->fe_effective_props; 2285 2286 /* no fanout for subflows */ 2287 switch (link_type) { 2288 case SRST_FLOW: 2289 fanout_type = SRST_NO_SOFT_RINGS; 2290 break; 2291 case SRST_LINK: 2292 fanout_type = SRST_FANOUT_PROTO; 2293 break; 2294 } 2295 2296 /* A primary NIC/link is being plumbed */ 2297 if (flent->fe_type & FLOW_PRIMARY_MAC) { 2298 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2299 fanout_type |= SRST_FANOUT_SRC_IP; 2300 } 2301 } else if (flent->fe_type & FLOW_VNIC) { 2302 /* A VNIC is being created */ 2303 if (mrp != NULL && mrp->mrp_ncpus > 0) { 2304 fanout_type |= SRST_FANOUT_SRC_IP; 2305 } 2306 } 2307 2308 return (fanout_type); 2309 } 2310 2311 /* 2312 * Change a group from h/w to s/w classification. 2313 */ 2314 void 2315 mac_rx_switch_grp_to_sw(mac_group_t *group) 2316 { 2317 mac_ring_t *ring; 2318 mac_soft_ring_set_t *mac_srs; 2319 2320 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2321 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2322 /* 2323 * Remove the SRS associated with the HW ring. 2324 * As a result, polling will be disabled. 2325 */ 2326 mac_srs = ring->mr_srs; 2327 ASSERT(mac_srs != NULL); 2328 mac_rx_srs_remove(mac_srs); 2329 ring->mr_srs = NULL; 2330 } 2331 2332 if (ring->mr_state != MR_INUSE) 2333 (void) mac_start_ring(ring); 2334 2335 /* 2336 * We need to perform SW classification 2337 * for packets landing in these rings 2338 */ 2339 ring->mr_flag = 0; 2340 ring->mr_classify_type = MAC_SW_CLASSIFIER; 2341 } 2342 } 2343 2344 /* 2345 * Create the Rx SRS for S/W classifier and for each ring in the 2346 * group (if exclusive group). Also create the Tx SRS. 2347 */ 2348 void 2349 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2350 uint32_t link_type) 2351 { 2352 cpupart_t *cpupart; 2353 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2354 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2355 boolean_t use_default = B_FALSE; 2356 2357 mac_rx_srs_group_setup(mcip, flent, link_type); 2358 mac_tx_srs_group_setup(mcip, flent, link_type); 2359 2360 /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */ 2361 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 2362 return; 2363 2364 pool_lock(); 2365 cpupart = mac_pset_find(mrp, &use_default); 2366 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2367 mac_rx_deliver, mcip, NULL, cpupart); 2368 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 2369 pool_unlock(); 2370 } 2371 2372 /* 2373 * Set up the Rx SRSes. If there is no group associated with the 2374 * client, then only setup SW classification. If the client has 2375 * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an 2376 * SRS for each HW ring. If the client is sharing a group, then make 2377 * sure to teardown the HW SRSes. 2378 */ 2379 void 2380 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2381 uint32_t link_type) 2382 { 2383 mac_impl_t *mip = mcip->mci_mip; 2384 mac_soft_ring_set_t *mac_srs; 2385 mac_ring_t *ring; 2386 uint32_t fanout_type; 2387 mac_group_t *rx_group = flent->fe_rx_ring_group; 2388 boolean_t no_unicast; 2389 2390 /* 2391 * If this is an an aggr port, then don't setup Rx SRS and Rx 2392 * soft rings as they won't be used. However, we still need to 2393 * start the rings to receive data on them. 2394 */ 2395 if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) { 2396 if (rx_group == NULL) 2397 return; 2398 2399 for (ring = rx_group->mrg_rings; ring != NULL; 2400 ring = ring->mr_next) { 2401 if (ring->mr_state != MR_INUSE) 2402 (void) mac_start_ring(ring); 2403 } 2404 2405 return; 2406 } 2407 2408 /* 2409 * Aggr ports should never have SRSes. 2410 */ 2411 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2412 2413 fanout_type = mac_find_fanout(flent, link_type); 2414 no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0; 2415 2416 /* Create the SRS for SW classification if none exists */ 2417 if (flent->fe_rx_srs[0] == NULL) { 2418 ASSERT(flent->fe_rx_srs_cnt == 0); 2419 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2420 mac_rx_deliver, mcip, NULL, NULL); 2421 mutex_enter(&flent->fe_lock); 2422 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2423 flent->fe_cb_arg1 = (void *)mip; 2424 flent->fe_cb_arg2 = (void *)mac_srs; 2425 mutex_exit(&flent->fe_lock); 2426 } 2427 2428 if (rx_group == NULL) 2429 return; 2430 2431 /* 2432 * If the group is marked RESERVED then setup an SRS and 2433 * fanout for each HW ring. 2434 */ 2435 switch (rx_group->mrg_state) { 2436 case MAC_GROUP_STATE_RESERVED: 2437 for (ring = rx_group->mrg_rings; ring != NULL; 2438 ring = ring->mr_next) { 2439 uint16_t vid = i_mac_flow_vid(mcip->mci_flent); 2440 2441 switch (ring->mr_state) { 2442 case MR_INUSE: 2443 case MR_FREE: 2444 if (ring->mr_srs != NULL) 2445 break; 2446 if (ring->mr_state != MR_INUSE) 2447 (void) mac_start_ring(ring); 2448 2449 /* 2450 * If a client requires SW VLAN 2451 * filtering or has no unicast address 2452 * then we don't create any HW ring 2453 * SRSes. 2454 */ 2455 if ((!MAC_GROUP_HW_VLAN(rx_group) && 2456 vid != VLAN_ID_NONE) || no_unicast) 2457 break; 2458 2459 /* 2460 * When a client has exclusive use of 2461 * a group, and that group's traffic 2462 * is fully HW classified, we create 2463 * an SRS for each HW ring in order to 2464 * make use of dynamic polling of said 2465 * HW rings. 2466 */ 2467 mac_srs = mac_srs_create(mcip, flent, 2468 fanout_type | link_type, 2469 mac_rx_deliver, mcip, NULL, ring); 2470 break; 2471 default: 2472 cmn_err(CE_PANIC, 2473 "srs_setup: mcip = %p " 2474 "trying to add UNKNOWN ring = %p\n", 2475 (void *)mcip, (void *)ring); 2476 break; 2477 } 2478 } 2479 break; 2480 case MAC_GROUP_STATE_SHARED: 2481 /* 2482 * When a group is shared by multiple clients, we must 2483 * use SW classifiction to ensure packets are 2484 * delivered to the correct client. 2485 */ 2486 mac_rx_switch_grp_to_sw(rx_group); 2487 break; 2488 default: 2489 ASSERT(B_FALSE); 2490 break; 2491 } 2492 } 2493 2494 /* 2495 * Set up the TX SRS. 2496 */ 2497 void 2498 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2499 uint32_t link_type) 2500 { 2501 /* 2502 * If this is an exclusive client (e.g. an aggr port), then 2503 * don't setup Tx SRS and Tx soft rings as they won't be used. 2504 * However, we still need to start the rings to send data 2505 * across them. 2506 */ 2507 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 2508 mac_ring_t *ring; 2509 mac_group_t *grp; 2510 2511 grp = (mac_group_t *)flent->fe_tx_ring_group; 2512 2513 if (grp == NULL) 2514 return; 2515 2516 for (ring = grp->mrg_rings; ring != NULL; 2517 ring = ring->mr_next) { 2518 if (ring->mr_state != MR_INUSE) 2519 (void) mac_start_ring(ring); 2520 } 2521 2522 return; 2523 } 2524 2525 /* 2526 * Aggr ports should never have SRSes. 2527 */ 2528 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2529 2530 if (flent->fe_tx_srs == NULL) { 2531 (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 2532 NULL, mcip, NULL, NULL); 2533 } 2534 2535 mac_tx_srs_setup(mcip, flent); 2536 } 2537 2538 /* 2539 * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown 2540 * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set 2541 * when we wish to move a MAC client from one group to another. In 2542 * that case, we need to release the current HW SRSes but keep the SW 2543 * SRS for continued traffic classifiction. 2544 */ 2545 void 2546 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 2547 { 2548 mac_soft_ring_set_t *mac_srs; 2549 int i; 2550 int count = flent->fe_rx_srs_cnt; 2551 2552 for (i = 0; i < count; i++) { 2553 if (i == 0 && hwonly) 2554 continue; 2555 mac_srs = flent->fe_rx_srs[i]; 2556 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2557 mac_srs_free(mac_srs); 2558 flent->fe_rx_srs[i] = NULL; 2559 flent->fe_rx_srs_cnt--; 2560 } 2561 2562 /* 2563 * If we are only tearing down the HW SRSes then there must be 2564 * one SRS left for SW classification. Otherwise we are tearing 2565 * down both HW and SW and there should be no SRSes left. 2566 */ 2567 if (hwonly) 2568 VERIFY3S(flent->fe_rx_srs_cnt, ==, 1); 2569 else 2570 VERIFY3S(flent->fe_rx_srs_cnt, ==, 0); 2571 } 2572 2573 /* 2574 * Remove the TX SRS. 2575 */ 2576 void 2577 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2578 uint32_t link_type) 2579 { 2580 mac_soft_ring_set_t *tx_srs; 2581 mac_srs_tx_t *tx; 2582 2583 if ((tx_srs = flent->fe_tx_srs) == NULL) 2584 return; 2585 2586 tx = &tx_srs->srs_tx; 2587 switch (link_type) { 2588 case SRST_FLOW: 2589 /* 2590 * For flows, we need to work with passed 2591 * flent to find the Rx/Tx SRS. 2592 */ 2593 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2594 break; 2595 case SRST_LINK: 2596 mac_tx_client_condemn((mac_client_handle_t)mcip); 2597 if (tx->st_arg2 != NULL) { 2598 ASSERT(tx_srs->srs_type & SRST_TX); 2599 /* 2600 * The ring itself will be stopped when 2601 * we release the group or in the 2602 * mac_datapath_teardown (for the default 2603 * group) 2604 */ 2605 tx->st_arg2 = NULL; 2606 } 2607 break; 2608 default: 2609 ASSERT(B_FALSE); 2610 break; 2611 } 2612 mac_srs_free(tx_srs); 2613 flent->fe_tx_srs = NULL; 2614 } 2615 2616 /* 2617 * This is the group state machine. 2618 * 2619 * The state of an Rx group is given by 2620 * the following table. The default group and its rings are started in 2621 * mac_start itself and the default group stays in SHARED state until 2622 * mac_stop at which time the group and rings are stopped and and it 2623 * reverts to the Registered state. 2624 * 2625 * Typically this function is called on a group after adding or removing a 2626 * client from it, to find out what should be the new state of the group. 2627 * If the new state is RESERVED, then the client that owns this group 2628 * exclusively is also returned. Note that adding or removing a client from 2629 * a group could also impact the default group and the caller needs to 2630 * evaluate the effect on the default group. 2631 * 2632 * Group type # of clients mi_nactiveclients Group State 2633 * in the group 2634 * 2635 * Non-default 0 N.A. REGISTERED 2636 * Non-default 1 N.A. RESERVED 2637 * 2638 * Default 0 N.A. SHARED 2639 * Default 1 1 RESERVED 2640 * Default 1 > 1 SHARED 2641 * Default > 1 N.A. SHARED 2642 * 2643 * For a TX group, the following is the state table. 2644 * 2645 * Group type # of clients Group State 2646 * in the group 2647 * 2648 * Non-default 0 REGISTERED 2649 * Non-default 1 RESERVED 2650 * 2651 * Default 0 REGISTERED 2652 * Default 1 RESERVED 2653 * Default > 1 SHARED 2654 */ 2655 mac_group_state_t 2656 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 2657 mac_group_t *defgrp, boolean_t rx_group) 2658 { 2659 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2660 2661 *group_only_mcip = NULL; 2662 2663 /* Non-default group */ 2664 2665 if (grp != defgrp) { 2666 if (MAC_GROUP_NO_CLIENT(grp)) 2667 return (MAC_GROUP_STATE_REGISTERED); 2668 2669 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2670 if (*group_only_mcip != NULL) 2671 return (MAC_GROUP_STATE_RESERVED); 2672 2673 return (MAC_GROUP_STATE_SHARED); 2674 } 2675 2676 /* Default group */ 2677 2678 if (MAC_GROUP_NO_CLIENT(grp)) { 2679 if (rx_group) 2680 return (MAC_GROUP_STATE_SHARED); 2681 else 2682 return (MAC_GROUP_STATE_REGISTERED); 2683 } 2684 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2685 if (*group_only_mcip == NULL) 2686 return (MAC_GROUP_STATE_SHARED); 2687 2688 if (rx_group && mip->mi_nactiveclients != 1) 2689 return (MAC_GROUP_STATE_SHARED); 2690 2691 ASSERT(*group_only_mcip != NULL); 2692 return (MAC_GROUP_STATE_RESERVED); 2693 } 2694 2695 /* 2696 * OVERVIEW NOTES FOR DATAPATH 2697 * =========================== 2698 * 2699 * Create an SRS and setup the corresponding flow function and args. 2700 * Add a classification rule for the flow specified by 'flent' and program 2701 * the hardware classifier when applicable. 2702 * 2703 * Rx ring assignment, SRS, polling and B/W enforcement 2704 * ---------------------------------------------------- 2705 * 2706 * We try to use H/W classification on NIC and assign traffic to a 2707 * MAC address to a particular Rx ring. There is a 1-1 mapping 2708 * between a SRS and a Rx ring. The SRS (short for soft ring set) 2709 * dynamically switches the underlying Rx ring between interrupt 2710 * and polling mode and enforces any specified B/W control. 2711 * 2712 * There is always a SRS created and tied to each H/W and S/W rule. 2713 * Whenever we create a H/W rule, we always add the the same rule to 2714 * S/W classifier and tie a SRS to it. 2715 * 2716 * In case a B/W control is specified, its broken into bytes 2717 * per ticks and as soon as the quota for a tick is exhausted, 2718 * the underlying Rx ring is forced into poll mode for remianing 2719 * tick. The SRS poll thread only polls for bytes that are 2720 * allowed to come in the SRS. We typically let 4x the configured 2721 * B/W worth of packets to come in the SRS (to prevent unnecessary 2722 * drops due to bursts) but only process the specified amount. 2723 * 2724 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2725 * Rx rings (and corresponding SRSs) assigned to it. The SRS 2726 * in turn can have softrings to do protocol level fanout or 2727 * softrings to do S/W based fanout or both. In case the NIC 2728 * has no Rx rings, we do S/W classification to respective SRS. 2729 * The S/W classification rule is always setup and ready. This 2730 * allows the MAC layer to reassign Rx rings whenever needed 2731 * but packets still continue to flow via the default path and 2732 * getting S/W classified to correct SRS. 2733 * 2734 * In other cases where a NIC or VNIC is plumbed, our goal is use 2735 * H/W classifier and get two Rx ring assigned for the Link. One 2736 * for TCP and one for UDP|SCTP. The respective SRS still do the 2737 * polling on the Rx ring. For Link that is plumbed for IP, there 2738 * is a TCP squeue which also does polling and can control the 2739 * the Rx ring directly (where SRS is just pass through). For 2740 * the following cases, the SRS does the polling underneath. 2741 * 1) non IP based Links (Links which are not plumbed via ifconfig) 2742 * and paths which have no IP squeues (UDP & SCTP) 2743 * 2) If B/W control is specified on the Link 2744 * 3) If S/W fanout is secified 2745 * 2746 * Note1: As of current implementation, we try to assign only 1 Rx 2747 * ring per Link and more than 1 Rx ring for primary Link for 2748 * H/W based fanout. We always create following softrings per SRS: 2749 * 1) TCP softring which is polled by TCP squeue where possible 2750 * (and also bypasses DLS) 2751 * 2) UDP/SCTP based which bypasses DLS 2752 * 3) OTH softring which goes via DLS (currently deal with IPv6 2753 * and non TCP/UDP/SCTP for IPv4 packets). 2754 * 2755 * It is necessary to create 3 softrings since SRS has to poll 2756 * the single Rx ring underneath and enforce any link level B/W 2757 * control (we can't switch the Rx ring in poll mode just based 2758 * on TCP squeue if the same Rx ring is sharing UDP and other 2759 * traffic as well). Once polling is done and any Link level B/W 2760 * control is specified, the packets are assigned to respective 2761 * softring based on protocol. Since TCP has IP based squeue 2762 * which benefits by polling, we separate TCP packets into 2763 * its own softring which can be polled by IP squeue. We need 2764 * to separate out UDP/SCTP to UDP softring since it can bypass 2765 * the DLS layer which has heavy performance advanatges and we 2766 * need a softring (OTH) for rest. 2767 * 2768 * ToDo: The 3 softrings for protocol are needed only till we can 2769 * get rid of DLS from datapath, make IPv4 and IPv6 paths 2770 * symmetric (deal with mac_header_info for v6 and polling for 2771 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2772 * are generic), and bring SAP based classification to MAC layer 2773 * 2774 * H/W and S/W based fanout and multiple Rx rings per Link 2775 * ------------------------------------------------------- 2776 * 2777 * In case, fanout is requested (or determined automatically based 2778 * on Link speed and processor speed), we try to assign multiple 2779 * Rx rings per Link with their respective SRS. In this case 2780 * the NIC should be capable of fanning out incoming packets between 2781 * the assigned Rx rings (H/W based fanout). All the SRS 2782 * individually switch their Rx ring between interrupt and polling 2783 * mode but share a common B/W control counter in case of Link 2784 * level B/W is specified. 2785 * 2786 * If S/W based fanout is specified in lieu of H/W based fanout, 2787 * the Link SRS creates the specified number of softrings for 2788 * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2789 * out to the correct softring based on their protocol and 2790 * protocol specific hash function. 2791 * 2792 * Primary and non primary MAC clients 2793 * ----------------------------------- 2794 * 2795 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2796 * and are a Layer 2 construct. 2797 * 2798 * Primary NIC: 2799 * The Link that owns the primary MAC address and typically 2800 * is used as the data NIC in non virtualized cases. As such 2801 * H/W resources are preferntially given to primary NIC. As 2802 * far as code is concerned, there is no difference in the 2803 * primary NIC vs VNICs. They are all treated as Links. 2804 * At the very first call to mac_unicast_add() we program the S/W 2805 * classifier for the primary MAC address, get a soft ring set 2806 * (and soft rings based on 'ip_soft_ring_cnt') 2807 * and a Rx ring assigned for polling to get enabled. 2808 * When IP get plumbed and negotiates polling, we can 2809 * let squeue do the polling on TCP softring. 2810 * 2811 * VNICs: 2812 * Same as any other Link. As long as the H/W resource assignments 2813 * are equal, the data path and setup for all Links is same. 2814 * 2815 * Flows: 2816 * Can be configured on Links. They have their own SRS and the 2817 * S/W classifier is programmed appropriately based on the flow. 2818 * The flows typically deal with layer 3 and above and 2819 * creates a soft ring set specific to the flow. The receive 2820 * side function is switched from mac_rx_srs_process to 2821 * mac_rx_srs_subflow_process which first tries to assign the 2822 * packet to appropriate flow SRS and failing which assigns it 2823 * to link SRS. This allows us to avoid the layered approach 2824 * which gets complex. 2825 * 2826 * By the time mac_datapath_setup() completes, we already have the 2827 * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2828 * and S/W classifiers programmed. IP is not plumbed yet (and might 2829 * never be for Virtual Machines guest OS path). When IP is plumbed 2830 * (for both NIC and VNIC), we do a capability negotiation for polling 2831 * and upcall functions etc. 2832 * 2833 * Rx ring Assignement NOTES 2834 * ------------------------- 2835 * 2836 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2837 * as NIC with a single default ring), we assign the only ring to 2838 * primary Link. The primary Link SRS can do polling on it as long as 2839 * it is the only link in use and we compare the MAC address for unicast 2840 * packets before accepting an incoming packet (there is no need for S/W 2841 * classification in this case). We disable polling on the only ring the 2842 * moment 2nd link gets created (the polling remains enabled even though 2843 * there are broadcast and * multicast flows created). 2844 * 2845 * If the NIC has more than 1 Rx ring, we assign the default ring (the 2846 * 1st ring) to deal with broadcast, multicast and traffic for other 2847 * NICs which needs S/W classification. We assign the primary mac 2848 * addresses to another ring by specifiying a classification rule for 2849 * primary unicast MAC address to the selected ring. The primary Link 2850 * (and its SRS) can continue to poll the assigned Rx ring at all times 2851 * independantly. 2852 * 2853 * Note: In future, if no fanout is specified, we try to assign 2 Rx 2854 * rings for the primary Link with the primary MAC address + TCP going 2855 * to one ring and primary MAC address + UDP|SCTP going to other ring. 2856 * Any remaining traffic for primary MAC address can go to the default 2857 * Rx ring and get S/W classified. This way the respective SRSs don't 2858 * need to do proto fanout and don't need to have softrings at all and 2859 * can poll their respective Rx rings. 2860 * 2861 * As an optimization, when a new NIC or VNIC is created, we can get 2862 * only one Rx ring and make it a TCP specific Rx ring and use the 2863 * H/W default Rx ring for the rest (this Rx ring is never polled). 2864 * 2865 * For clients that don't have MAC address, but want to receive and 2866 * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 2867 * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 2868 * always give the default group and use software classification (i.e. 2869 * even if this is the only client in the default group, we will 2870 * leave group as shared). 2871 */ 2872 2873 int 2874 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2875 uint32_t link_type) 2876 { 2877 mac_impl_t *mip = mcip->mci_mip; 2878 mac_group_t *rgroup = NULL; 2879 mac_group_t *tgroup = NULL; 2880 mac_group_t *default_rgroup; 2881 mac_group_t *default_tgroup; 2882 int err; 2883 uint16_t vid; 2884 uint8_t *mac_addr; 2885 mac_group_state_t next_state; 2886 mac_client_impl_t *group_only_mcip; 2887 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2888 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2889 boolean_t rxhw; 2890 boolean_t txhw; 2891 boolean_t use_default = B_FALSE; 2892 cpupart_t *cpupart; 2893 boolean_t no_unicast; 2894 boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 2895 mac_client_impl_t *reloc_pmcip = NULL; 2896 boolean_t use_hw; 2897 2898 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2899 2900 switch (link_type) { 2901 case SRST_FLOW: 2902 mac_srs_group_setup(mcip, flent, link_type); 2903 return (0); 2904 2905 case SRST_LINK: 2906 no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2907 mac_addr = flent->fe_flow_desc.fd_dst_mac; 2908 2909 /* Default RX group */ 2910 default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2911 2912 /* Default TX group */ 2913 default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 2914 2915 if (no_unicast) { 2916 rgroup = default_rgroup; 2917 tgroup = default_tgroup; 2918 goto grp_found; 2919 } 2920 rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 2921 (mrp->mrp_nrxrings > 0 || 2922 (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 2923 txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 2924 (mrp->mrp_ntxrings > 0 || 2925 (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 2926 2927 /* 2928 * All the rings initially belong to the default group 2929 * under dynamic grouping. The primary client uses the 2930 * default group when it is the only client. The 2931 * default group is also used as the destination for 2932 * all multicast and broadcast traffic of all clients. 2933 * Therefore, the primary client loses its ability to 2934 * poll the softrings on addition of a second client. 2935 * To avoid a performance penalty, MAC will move the 2936 * primary client to a dedicated group when it can. 2937 * 2938 * When using static grouping, the primary client 2939 * begins life on a non-default group. There is 2940 * no moving needed upon addition of a second client. 2941 */ 2942 if (!isprimary && mip->mi_nactiveclients == 2 && 2943 (group_only_mcip = mac_primary_client_handle(mip)) != 2944 NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 2945 reloc_pmcip = mac_check_primary_relocation( 2946 group_only_mcip, rxhw); 2947 } 2948 2949 /* 2950 * Check to see if we can get an exclusive group for 2951 * this mac address or if there already exists a 2952 * group that has this mac address (case of VLANs). 2953 * If no groups are available, use the default group. 2954 */ 2955 rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 2956 if (rgroup == NULL && rxhw) { 2957 err = ENOSPC; 2958 goto setup_failed; 2959 } else if (rgroup == NULL) { 2960 rgroup = default_rgroup; 2961 } 2962 2963 /* 2964 * If we are adding a second client to a 2965 * non-default group then we need to move the 2966 * existing client to the default group and 2967 * add the new client to the default group as 2968 * well. 2969 */ 2970 if (rgroup != default_rgroup && 2971 rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) { 2972 group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup); 2973 err = mac_rx_switch_group(group_only_mcip, rgroup, 2974 default_rgroup); 2975 2976 if (err != 0) 2977 goto setup_failed; 2978 2979 rgroup = default_rgroup; 2980 } 2981 2982 /* 2983 * Check to see if we can get an exclusive group for 2984 * this mac client. If no groups are available, use 2985 * the default group. 2986 */ 2987 tgroup = mac_reserve_tx_group(mcip, B_FALSE); 2988 if (tgroup == NULL && txhw) { 2989 if (rgroup != NULL && rgroup != default_rgroup) 2990 mac_release_rx_group(mcip, rgroup); 2991 err = ENOSPC; 2992 goto setup_failed; 2993 } else if (tgroup == NULL) { 2994 tgroup = default_tgroup; 2995 } 2996 2997 /* 2998 * Some NICs don't support any Rx rings, so there may not 2999 * even be a default group. 3000 */ 3001 grp_found: 3002 if (rgroup != NULL) { 3003 if (rgroup != default_rgroup && 3004 MAC_GROUP_NO_CLIENT(rgroup) && 3005 (rxhw || mcip->mci_share != 0)) { 3006 MAC_RX_GRP_RESERVED(mip); 3007 if (mip->mi_rx_group_type == 3008 MAC_GROUP_TYPE_DYNAMIC) { 3009 MAC_RX_RING_RESERVED(mip, 3010 rgroup->mrg_cur_count); 3011 } 3012 } 3013 3014 flent->fe_rx_ring_group = rgroup; 3015 /* 3016 * Add the client to the group and update the 3017 * group's state. If rgroup != default_group 3018 * then the rgroup should only ever have one 3019 * client and be in the RESERVED state. But no 3020 * matter what, the default_rgroup will enter 3021 * the SHARED state since it has to receive 3022 * all broadcast and multicast traffic. This 3023 * case is handled later in the function. 3024 */ 3025 mac_group_add_client(rgroup, mcip); 3026 next_state = mac_group_next_state(rgroup, 3027 &group_only_mcip, default_rgroup, B_TRUE); 3028 mac_set_group_state(rgroup, next_state); 3029 } 3030 3031 if (tgroup != NULL) { 3032 if (tgroup != default_tgroup && 3033 MAC_GROUP_NO_CLIENT(tgroup) && 3034 (txhw || mcip->mci_share != 0)) { 3035 MAC_TX_GRP_RESERVED(mip); 3036 if (mip->mi_tx_group_type == 3037 MAC_GROUP_TYPE_DYNAMIC) { 3038 MAC_TX_RING_RESERVED(mip, 3039 tgroup->mrg_cur_count); 3040 } 3041 } 3042 flent->fe_tx_ring_group = tgroup; 3043 mac_group_add_client(tgroup, mcip); 3044 next_state = mac_group_next_state(tgroup, 3045 &group_only_mcip, default_tgroup, B_FALSE); 3046 tgroup->mrg_state = next_state; 3047 } 3048 3049 /* We are setting up minimal datapath only */ 3050 if (no_unicast) { 3051 mac_srs_group_setup(mcip, flent, link_type); 3052 break; 3053 } 3054 3055 /* Program software classification. */ 3056 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3057 goto setup_failed; 3058 3059 /* Program hardware classification. */ 3060 vid = i_mac_flow_vid(flent); 3061 use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0; 3062 err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw); 3063 3064 if (err != 0) 3065 goto setup_failed; 3066 3067 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 3068 VERIFY3P(mcip->mci_unicast, !=, NULL); 3069 3070 /* 3071 * Setup the Rx and Tx SRSes. If the client has a 3072 * reserved group, then mac_srs_group_setup() creates 3073 * the required SRSes for the HW rings. If we have a 3074 * shared group, mac_srs_group_setup() dismantles the 3075 * HW SRSes of the previously exclusive group. 3076 */ 3077 mac_srs_group_setup(mcip, flent, link_type); 3078 3079 /* (Re)init the v6 token & local addr used by link protection */ 3080 mac_protect_update_mac_token(mcip); 3081 break; 3082 3083 default: 3084 ASSERT(B_FALSE); 3085 break; 3086 } 3087 3088 /* 3089 * All broadcast and multicast traffic is received only on the default 3090 * group. If we have setup the datapath for a non-default group above 3091 * then move the default group to shared state to allow distribution of 3092 * incoming broadcast traffic to the other groups and dismantle the 3093 * SRSes over the default group. 3094 */ 3095 if (rgroup != NULL) { 3096 if (rgroup != default_rgroup) { 3097 if (default_rgroup->mrg_state == 3098 MAC_GROUP_STATE_RESERVED) { 3099 group_only_mcip = MAC_GROUP_ONLY_CLIENT( 3100 default_rgroup); 3101 ASSERT(group_only_mcip != NULL && 3102 mip->mi_nactiveclients > 1); 3103 3104 mac_set_group_state(default_rgroup, 3105 MAC_GROUP_STATE_SHARED); 3106 mac_rx_srs_group_setup(group_only_mcip, 3107 group_only_mcip->mci_flent, SRST_LINK); 3108 pool_lock(); 3109 cpupart = mac_pset_find(mrp, &use_default); 3110 mac_fanout_setup(group_only_mcip, 3111 group_only_mcip->mci_flent, 3112 MCIP_RESOURCE_PROPS(group_only_mcip), 3113 mac_rx_deliver, group_only_mcip, NULL, 3114 cpupart); 3115 mac_set_pool_effective(use_default, cpupart, 3116 mrp, emrp); 3117 pool_unlock(); 3118 } 3119 ASSERT(default_rgroup->mrg_state == 3120 MAC_GROUP_STATE_SHARED); 3121 } 3122 3123 /* 3124 * A VLAN MAC client on a reserved group still 3125 * requires SW classification if the MAC doesn't 3126 * provide VLAN HW filtering. 3127 * 3128 * Clients with no unicast address also require SW 3129 * classification. 3130 */ 3131 if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 3132 ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) || 3133 no_unicast)) { 3134 mac_rx_switch_grp_to_sw(rgroup); 3135 } 3136 3137 } 3138 3139 mac_set_rings_effective(mcip); 3140 return (0); 3141 3142 setup_failed: 3143 /* Switch the primary back to default group */ 3144 if (reloc_pmcip != NULL) { 3145 (void) mac_rx_switch_group(reloc_pmcip, 3146 reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 3147 } 3148 mac_datapath_teardown(mcip, flent, link_type); 3149 return (err); 3150 } 3151 3152 void 3153 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3154 uint32_t link_type) 3155 { 3156 mac_impl_t *mip = mcip->mci_mip; 3157 mac_group_t *group = NULL; 3158 mac_client_impl_t *grp_only_mcip; 3159 flow_entry_t *group_only_flent; 3160 mac_group_t *default_group; 3161 boolean_t check_default_group = B_FALSE; 3162 mac_group_state_t next_state; 3163 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 3164 uint16_t vid; 3165 3166 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3167 3168 switch (link_type) { 3169 case SRST_FLOW: 3170 mac_rx_srs_group_teardown(flent, B_FALSE); 3171 mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3172 return; 3173 3174 case SRST_LINK: 3175 /* Stop sending packets */ 3176 mac_tx_client_block(mcip); 3177 group = flent->fe_rx_ring_group; 3178 vid = i_mac_flow_vid(flent); 3179 3180 /* 3181 * Stop the packet flow from the hardware by disabling 3182 * any hardware filters assigned to this client. 3183 */ 3184 if (mcip->mci_unicast != NULL) { 3185 int err; 3186 3187 err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid); 3188 3189 if (err != 0) { 3190 cmn_err(CE_WARN, "%s: failed to remove a MAC HW" 3191 " filters because of error 0x%x", 3192 mip->mi_name, err); 3193 } 3194 3195 mcip->mci_unicast = NULL; 3196 } 3197 3198 /* Stop the packets coming from the S/W classifier */ 3199 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3200 mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3201 3202 /* Quiesce and destroy all the SRSes. */ 3203 mac_rx_srs_group_teardown(flent, B_FALSE); 3204 mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 3205 3206 ASSERT3P(mcip->mci_flent, ==, flent); 3207 ASSERT3P(flent->fe_next, ==, NULL); 3208 3209 /* 3210 * Release our hold on the group as well. We need 3211 * to check if the shared group has only one client 3212 * left who can use it exclusively. Also, if we 3213 * were the last client, release the group. 3214 */ 3215 default_group = MAC_DEFAULT_RX_GROUP(mip); 3216 if (group != NULL) { 3217 mac_group_remove_client(group, mcip); 3218 next_state = mac_group_next_state(group, 3219 &grp_only_mcip, default_group, B_TRUE); 3220 3221 if (next_state == MAC_GROUP_STATE_RESERVED) { 3222 /* 3223 * Only one client left on this RX group. 3224 */ 3225 VERIFY3P(grp_only_mcip, !=, NULL); 3226 mac_set_group_state(group, 3227 MAC_GROUP_STATE_RESERVED); 3228 group_only_flent = grp_only_mcip->mci_flent; 3229 3230 /* 3231 * The only remaining client has exclusive 3232 * access on the group. Allow it to 3233 * dynamically poll the H/W rings etc. 3234 */ 3235 mac_rx_srs_group_setup(grp_only_mcip, 3236 group_only_flent, SRST_LINK); 3237 mac_fanout_setup(grp_only_mcip, 3238 group_only_flent, 3239 MCIP_RESOURCE_PROPS(grp_only_mcip), 3240 mac_rx_deliver, grp_only_mcip, NULL, NULL); 3241 mac_rx_group_unmark(group, MR_INCIPIENT); 3242 mac_set_rings_effective(grp_only_mcip); 3243 } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3244 /* 3245 * This is a non-default group being freed up. 3246 * We need to reevaluate the default group 3247 * to see if the primary client can get 3248 * exclusive access to the default group. 3249 */ 3250 VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip)); 3251 if (mrp->mrp_mask & MRP_RX_RINGS) { 3252 MAC_RX_GRP_RELEASED(mip); 3253 if (mip->mi_rx_group_type == 3254 MAC_GROUP_TYPE_DYNAMIC) { 3255 MAC_RX_RING_RELEASED(mip, 3256 group->mrg_cur_count); 3257 } 3258 } 3259 mac_release_rx_group(mcip, group); 3260 mac_set_group_state(group, 3261 MAC_GROUP_STATE_REGISTERED); 3262 check_default_group = B_TRUE; 3263 } else { 3264 VERIFY3S(next_state, ==, 3265 MAC_GROUP_STATE_SHARED); 3266 mac_set_group_state(group, 3267 MAC_GROUP_STATE_SHARED); 3268 mac_rx_group_unmark(group, MR_CONDEMNED); 3269 } 3270 flent->fe_rx_ring_group = NULL; 3271 } 3272 /* 3273 * Remove the client from the TX group. Additionally, if 3274 * this a non-default group, then we also need to release 3275 * the group. 3276 */ 3277 group = flent->fe_tx_ring_group; 3278 default_group = MAC_DEFAULT_TX_GROUP(mip); 3279 if (group != NULL) { 3280 mac_group_remove_client(group, mcip); 3281 next_state = mac_group_next_state(group, 3282 &grp_only_mcip, default_group, B_FALSE); 3283 if (next_state == MAC_GROUP_STATE_REGISTERED) { 3284 if (group != default_group) { 3285 if (mrp->mrp_mask & MRP_TX_RINGS) { 3286 MAC_TX_GRP_RELEASED(mip); 3287 if (mip->mi_tx_group_type == 3288 MAC_GROUP_TYPE_DYNAMIC) { 3289 MAC_TX_RING_RELEASED( 3290 mip, group-> 3291 mrg_cur_count); 3292 } 3293 } 3294 mac_release_tx_group(mcip, group); 3295 /* 3296 * If the default group is reserved, 3297 * then we need to set the effective 3298 * rings as we would have given 3299 * back some rings when the group 3300 * was released 3301 */ 3302 if (mip->mi_tx_group_type == 3303 MAC_GROUP_TYPE_DYNAMIC && 3304 default_group->mrg_state == 3305 MAC_GROUP_STATE_RESERVED) { 3306 grp_only_mcip = 3307 MAC_GROUP_ONLY_CLIENT 3308 (default_group); 3309 mac_set_rings_effective( 3310 grp_only_mcip); 3311 } 3312 } else { 3313 mac_ring_t *ring; 3314 int cnt; 3315 int ringcnt; 3316 3317 /* 3318 * Stop all the rings except the 3319 * default ring. 3320 */ 3321 ringcnt = group->mrg_cur_count; 3322 ring = group->mrg_rings; 3323 for (cnt = 0; cnt < ringcnt; cnt++) { 3324 if (ring->mr_state == 3325 MR_INUSE && ring != 3326 (mac_ring_t *) 3327 mip->mi_default_tx_ring) { 3328 mac_stop_ring(ring); 3329 ring->mr_flag = 0; 3330 } 3331 ring = ring->mr_next; 3332 } 3333 } 3334 } else if (next_state == MAC_GROUP_STATE_RESERVED) { 3335 mac_set_rings_effective(grp_only_mcip); 3336 } 3337 flent->fe_tx_ring_group = NULL; 3338 group->mrg_state = next_state; 3339 } 3340 break; 3341 default: 3342 ASSERT(B_FALSE); 3343 break; 3344 } 3345 3346 /* 3347 * The mac client using the default group gets exclusive access to the 3348 * default group if and only if it is the sole client on the entire 3349 * mip. If so set the group state to reserved, and set up the SRSes 3350 * over the default group. 3351 */ 3352 if (check_default_group) { 3353 default_group = MAC_DEFAULT_RX_GROUP(mip); 3354 VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED); 3355 next_state = mac_group_next_state(default_group, 3356 &grp_only_mcip, default_group, B_TRUE); 3357 if (next_state == MAC_GROUP_STATE_RESERVED) { 3358 VERIFY3P(grp_only_mcip, !=, NULL); 3359 VERIFY3U(mip->mi_nactiveclients, ==, 1); 3360 mac_set_group_state(default_group, 3361 MAC_GROUP_STATE_RESERVED); 3362 mac_rx_srs_group_setup(grp_only_mcip, 3363 grp_only_mcip->mci_flent, SRST_LINK); 3364 mac_fanout_setup(grp_only_mcip, 3365 grp_only_mcip->mci_flent, 3366 MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 3367 grp_only_mcip, NULL, NULL); 3368 mac_rx_group_unmark(default_group, MR_INCIPIENT); 3369 mac_set_rings_effective(grp_only_mcip); 3370 } 3371 } 3372 3373 /* 3374 * If the primary is the only one left and the MAC supports 3375 * dynamic grouping, we need to see if the primary needs to 3376 * be moved to the default group so that it can use all the 3377 * H/W rings. 3378 */ 3379 if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 3380 mip->mi_nactiveclients == 1 && 3381 mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 3382 default_group = MAC_DEFAULT_RX_GROUP(mip); 3383 grp_only_mcip = mac_primary_client_handle(mip); 3384 if (grp_only_mcip == NULL) 3385 return; 3386 group_only_flent = grp_only_mcip->mci_flent; 3387 mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 3388 /* 3389 * If the primary has an explicit property set, leave it 3390 * alone. 3391 */ 3392 if (mrp->mrp_mask & MRP_RX_RINGS) 3393 return; 3394 /* 3395 * Switch the primary to the default group. 3396 */ 3397 (void) mac_rx_switch_group(grp_only_mcip, 3398 group_only_flent->fe_rx_ring_group, default_group); 3399 } 3400 } 3401 3402 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3403 3404 static void 3405 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3406 { 3407 if (mac_srs->srs_type & SRST_TX) { 3408 mac_srs_tx_t *tx; 3409 3410 ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 3411 ASSERT(mac_srs->srs_udp_soft_rings == NULL); 3412 ASSERT(mac_srs->srs_oth_soft_rings == NULL); 3413 ASSERT(mac_srs->srs_tx_soft_rings != NULL); 3414 kmem_free(mac_srs->srs_tx_soft_rings, 3415 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3416 mac_srs->srs_tx_soft_rings = NULL; 3417 tx = &mac_srs->srs_tx; 3418 if (tx->st_soft_rings != NULL) { 3419 kmem_free(tx->st_soft_rings, 3420 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3421 } 3422 } else { 3423 ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3424 ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3425 kmem_free(mac_srs->srs_tcp_soft_rings, 3426 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3427 mac_srs->srs_tcp_soft_rings = NULL; 3428 ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3429 kmem_free(mac_srs->srs_udp_soft_rings, 3430 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3431 mac_srs->srs_udp_soft_rings = NULL; 3432 ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3433 kmem_free(mac_srs->srs_oth_soft_rings, 3434 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3435 mac_srs->srs_oth_soft_rings = NULL; 3436 } 3437 } 3438 3439 /* 3440 * An RX SRS is attached to at most one mac_ring. 3441 * A TX SRS has no rings. 3442 */ 3443 static void 3444 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3445 { 3446 mac_client_impl_t *mcip; 3447 mac_ring_t *ring; 3448 flow_entry_t *flent; 3449 3450 ring = mac_srs->srs_ring; 3451 if (mac_srs->srs_type & SRST_TX) { 3452 ASSERT(ring == NULL); 3453 return; 3454 } 3455 3456 if (ring == NULL) 3457 return; 3458 3459 /* 3460 * Broadcast flows don't have a client impl association, but they 3461 * use only soft rings. 3462 */ 3463 flent = mac_srs->srs_flent; 3464 mcip = flent->fe_mcip; 3465 ASSERT(mcip != NULL); 3466 3467 ring->mr_classify_type = MAC_NO_CLASSIFIER; 3468 ring->mr_srs = NULL; 3469 } 3470 3471 /* 3472 * Physical unlink and free of the data structures happen below. This is 3473 * driven from mac_flow_destroy(), on the last refrele of a flow. 3474 * 3475 * Assumes Rx srs is 1-1 mapped with an ring. 3476 */ 3477 void 3478 mac_srs_free(mac_soft_ring_set_t *mac_srs) 3479 { 3480 ASSERT(mac_srs->srs_mcip == NULL || 3481 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3482 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3483 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3484 3485 mac_drop_chain(mac_srs->srs_first, "SRS free"); 3486 mac_srs_ring_free(mac_srs); 3487 mac_srs_soft_rings_free(mac_srs); 3488 mac_srs_fanout_list_free(mac_srs); 3489 3490 mac_srs->srs_bw = NULL; 3491 mac_srs_stat_delete(mac_srs); 3492 kmem_cache_free(mac_srs_cache, mac_srs); 3493 } 3494 3495 static void 3496 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3497 { 3498 mac_soft_ring_t *softring; 3499 3500 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3501 3502 mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3503 if (s_ring_flag == S_RING_CONDEMNED) { 3504 while (mac_srs->srs_soft_ring_condemned_count != 3505 mac_srs->srs_soft_ring_count) 3506 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3507 } else { 3508 while (mac_srs->srs_soft_ring_quiesced_count != 3509 mac_srs->srs_soft_ring_count) 3510 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3511 } 3512 mutex_exit(&mac_srs->srs_lock); 3513 3514 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3515 softring = softring->s_ring_next) { 3516 (void) untimeout(softring->s_ring_tid); 3517 softring->s_ring_tid = NULL; 3518 } 3519 3520 (void) untimeout(mac_srs->srs_tid); 3521 mac_srs->srs_tid = NULL; 3522 3523 mutex_enter(&mac_srs->srs_lock); 3524 } 3525 3526 /* 3527 * The block comment above mac_rx_classify_flow_state_change explains the 3528 * background. At this point upcalls from the driver (both hardware classified 3529 * and software classified) have been cut off. We now need to quiesce the 3530 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3531 * the master controller. The steps involved are described below in the function 3532 */ 3533 void 3534 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3535 { 3536 uint_t s_ring_flag; 3537 uint_t srs_poll_wait_flag; 3538 3539 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3540 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3541 3542 if (mac_srs->srs_state & SRS_CONDEMNED) { 3543 s_ring_flag = S_RING_CONDEMNED; 3544 srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3545 } else { 3546 s_ring_flag = S_RING_QUIESCE; 3547 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3548 } 3549 3550 /* 3551 * In the case of Rx SRS wait till the poll thread is done. 3552 */ 3553 if ((mac_srs->srs_type & SRST_TX) == 0 && 3554 mac_srs->srs_poll_thr != NULL) { 3555 while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3556 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3557 3558 /* 3559 * Turn off polling as part of the quiesce operation. 3560 */ 3561 MAC_SRS_POLLING_OFF(mac_srs); 3562 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3563 } 3564 3565 /* 3566 * Then signal the soft ring worker threads to quiesce or quit 3567 * as needed and then wait till that happens. 3568 */ 3569 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3570 3571 if (mac_srs->srs_state & SRS_CONDEMNED) 3572 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3573 else 3574 mac_srs->srs_state |= SRS_QUIESCE_DONE; 3575 cv_signal(&mac_srs->srs_quiesce_done_cv); 3576 } 3577 3578 /* 3579 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3580 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3581 * or poll thread. This function is internal to the quiescing logic and is 3582 * called internally from the SRS quiesce or flow quiesce or client quiesce 3583 * higher level functions. 3584 */ 3585 void 3586 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3587 { 3588 mac_ring_t *ring; 3589 3590 ring = mac_srs->srs_ring; 3591 ASSERT(ring == NULL || ring->mr_refcnt == 0); 3592 3593 if (srs_flag == SRS_CONDEMNED) { 3594 /* 3595 * The SRS is going away. We need to unbind the SRS and SR 3596 * threads before removing from the global SRS list. Otherwise 3597 * there is a small window where the cpu reconfig callbacks 3598 * may miss the SRS in the list walk and DR could fail since 3599 * there are still bound threads. 3600 */ 3601 mac_srs_threads_unbind(mac_srs); 3602 mac_srs_remove_glist(mac_srs); 3603 } 3604 /* 3605 * Wakeup the SRS worker and poll threads. 3606 */ 3607 mutex_enter(&mac_srs->srs_lock); 3608 mac_srs->srs_state |= srs_flag; 3609 cv_signal(&mac_srs->srs_async); 3610 cv_signal(&mac_srs->srs_cv); 3611 mutex_exit(&mac_srs->srs_lock); 3612 } 3613 3614 /* 3615 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3616 * from the driver are done, then the Rx SRS is quiesced and only then can 3617 * we signal the soft rings. Thus this function can't be called arbitrarily 3618 * without satisfying the prerequisites. On the Tx side, the threads from 3619 * top need to quiesced, then the Tx SRS and only then can we signal the 3620 * Tx soft rings. 3621 */ 3622 static void 3623 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3624 { 3625 mac_soft_ring_t *softring; 3626 3627 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3628 softring = softring->s_ring_next) 3629 mac_soft_ring_signal(softring, sr_flag); 3630 } 3631 3632 /* 3633 * The block comment above mac_rx_classify_flow_state_change explains the 3634 * background. At this point the SRS is quiesced and we need to restart the 3635 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3636 * the master controller. The steps involved are described below in the function 3637 */ 3638 void 3639 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3640 { 3641 boolean_t iam_rx_srs; 3642 mac_soft_ring_t *softring; 3643 3644 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3645 if ((mac_srs->srs_type & SRST_TX) != 0) { 3646 iam_rx_srs = B_FALSE; 3647 ASSERT((mac_srs->srs_state & 3648 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3649 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3650 } else { 3651 iam_rx_srs = B_TRUE; 3652 ASSERT((mac_srs->srs_state & 3653 (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3654 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3655 if (mac_srs->srs_poll_thr != NULL) { 3656 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3657 SRS_POLL_THR_QUIESCED); 3658 } 3659 } 3660 3661 /* 3662 * Signal any quiesced soft ring workers to restart and wait for the 3663 * soft ring down count to come down to zero. 3664 */ 3665 if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3666 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3667 softring = softring->s_ring_next) { 3668 if (!(softring->s_ring_state & S_RING_QUIESCE)) 3669 continue; 3670 mac_soft_ring_signal(softring, S_RING_RESTART); 3671 } 3672 while (mac_srs->srs_soft_ring_quiesced_count != 0) 3673 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3674 } 3675 3676 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3677 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3678 /* 3679 * Signal the poll thread and ask it to restart. Wait till it 3680 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3681 * cleared. 3682 */ 3683 mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3684 cv_signal(&mac_srs->srs_cv); 3685 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3686 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3687 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3688 } 3689 /* Wake up any waiter waiting for the restart to complete */ 3690 mac_srs->srs_state |= SRS_RESTART_DONE; 3691 cv_signal(&mac_srs->srs_quiesce_done_cv); 3692 } 3693 3694 static void 3695 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3696 { 3697 mutex_enter(&mac_srs->srs_lock); 3698 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3699 ASSERT(mac_srs->srs_worker_cpuid == -1); 3700 mutex_exit(&mac_srs->srs_lock); 3701 return; 3702 } 3703 3704 mac_srs->srs_worker_cpuid = -1; 3705 mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3706 thread_affinity_clear(mac_srs->srs_worker); 3707 mutex_exit(&mac_srs->srs_lock); 3708 } 3709 3710 static void 3711 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3712 { 3713 mutex_enter(&mac_srs->srs_lock); 3714 if (mac_srs->srs_poll_thr == NULL || 3715 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3716 ASSERT(mac_srs->srs_poll_cpuid == -1); 3717 mutex_exit(&mac_srs->srs_lock); 3718 return; 3719 } 3720 3721 mac_srs->srs_poll_cpuid = -1; 3722 mac_srs->srs_state &= ~SRS_POLL_BOUND; 3723 thread_affinity_clear(mac_srs->srs_poll_thr); 3724 mutex_exit(&mac_srs->srs_lock); 3725 } 3726 3727 static void 3728 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3729 { 3730 mac_soft_ring_t *soft_ring; 3731 3732 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3733 3734 mutex_enter(&cpu_lock); 3735 mac_srs_worker_unbind(mac_srs); 3736 if (!(mac_srs->srs_type & SRST_TX)) 3737 mac_srs_poll_unbind(mac_srs); 3738 3739 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3740 soft_ring = soft_ring->s_ring_next) { 3741 mac_soft_ring_unbind(soft_ring); 3742 } 3743 mutex_exit(&cpu_lock); 3744 } 3745 3746 /* 3747 * When a CPU is going away, unbind all MAC threads which are bound 3748 * to that CPU. The affinity of the thread to the CPU is saved to allow 3749 * the thread to be rebound to the CPU if it comes back online. 3750 */ 3751 static void 3752 mac_walk_srs_and_unbind(int cpuid) 3753 { 3754 mac_soft_ring_set_t *mac_srs; 3755 mac_soft_ring_t *soft_ring; 3756 3757 rw_enter(&mac_srs_g_lock, RW_READER); 3758 3759 if ((mac_srs = mac_srs_g_list) == NULL) 3760 goto done; 3761 3762 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3763 if (mac_srs->srs_worker_cpuid == cpuid) { 3764 mac_srs->srs_worker_cpuid_save = cpuid; 3765 mac_srs_worker_unbind(mac_srs); 3766 } 3767 3768 if (!(mac_srs->srs_type & SRST_TX)) { 3769 if (mac_srs->srs_poll_cpuid == cpuid) { 3770 mac_srs->srs_poll_cpuid_save = cpuid; 3771 mac_srs_poll_unbind(mac_srs); 3772 } 3773 } 3774 3775 /* Next tackle the soft rings associated with the srs */ 3776 mutex_enter(&mac_srs->srs_lock); 3777 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3778 soft_ring = soft_ring->s_ring_next) { 3779 if (soft_ring->s_ring_cpuid == cpuid) { 3780 soft_ring->s_ring_cpuid_save = cpuid; 3781 mac_soft_ring_unbind(soft_ring); 3782 } 3783 } 3784 mutex_exit(&mac_srs->srs_lock); 3785 } 3786 done: 3787 rw_exit(&mac_srs_g_lock); 3788 } 3789 3790 /* TX SETUP and TEARDOWN ROUTINES */ 3791 3792 /* 3793 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3794 * handle the case where the number of rings is one. I.e. there is 3795 * a ring pointed to by mac_srs->srs_tx_arg2. 3796 */ 3797 void 3798 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3799 { 3800 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3801 mac_soft_ring_t *soft_ring; 3802 int count = mac_srs->srs_tx_ring_count; 3803 uint32_t soft_ring_type = ST_RING_TX; 3804 uint_t ring_info; 3805 3806 ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3807 ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 3808 if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 3809 soft_ring_type |= ST_RING_WORKER_ONLY; 3810 soft_ring = mac_soft_ring_create(count, 0, 3811 soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3812 NULL, mcip, (mac_resource_handle_t)tx_ring); 3813 mac_srs->srs_tx_ring_count++; 3814 mac_srs_update_fanout_list(mac_srs); 3815 /* 3816 * put this soft ring in quiesce mode too so when we restart 3817 * all soft rings in the srs are in the same state. 3818 */ 3819 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3820 } 3821 3822 static void 3823 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3824 { 3825 int sringcnt; 3826 3827 mutex_enter(&mac_srs->srs_lock); 3828 sringcnt = mac_srs->srs_soft_ring_count; 3829 ASSERT(sringcnt > 0); 3830 mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3831 3832 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3833 while (mac_srs->srs_soft_ring_condemned_count != 1) 3834 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3835 3836 if (softring == mac_srs->srs_soft_ring_head) { 3837 mac_srs->srs_soft_ring_head = softring->s_ring_next; 3838 if (mac_srs->srs_soft_ring_head != NULL) { 3839 mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3840 } else { 3841 mac_srs->srs_soft_ring_tail = NULL; 3842 } 3843 } else { 3844 softring->s_ring_prev->s_ring_next = 3845 softring->s_ring_next; 3846 if (softring->s_ring_next != NULL) { 3847 softring->s_ring_next->s_ring_prev = 3848 softring->s_ring_prev; 3849 } else { 3850 mac_srs->srs_soft_ring_tail = 3851 softring->s_ring_prev; 3852 } 3853 } 3854 mac_srs->srs_soft_ring_count--; 3855 3856 mac_srs->srs_soft_ring_condemned_count--; 3857 mutex_exit(&mac_srs->srs_lock); 3858 3859 mac_soft_ring_free(softring); 3860 } 3861 3862 void 3863 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3864 { 3865 int i; 3866 mac_soft_ring_t *soft_ring, *remove_sring; 3867 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3868 3869 mutex_enter(&mac_srs->srs_lock); 3870 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 3871 soft_ring = mac_srs->srs_tx_soft_rings[i]; 3872 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3873 break; 3874 } 3875 mutex_exit(&mac_srs->srs_lock); 3876 ASSERT(i < mac_srs->srs_tx_ring_count); 3877 remove_sring = soft_ring; 3878 /* 3879 * In the case of aggr, the soft ring associated with a Tx ring 3880 * is also stored in st_soft_rings[] array. That entry should 3881 * be removed. 3882 */ 3883 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 3884 mac_srs_tx_t *tx = &mac_srs->srs_tx; 3885 3886 ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 3887 tx->st_soft_rings[tx_ring->mr_index] = NULL; 3888 } 3889 mac_soft_ring_remove(mac_srs, remove_sring); 3890 mac_srs_update_fanout_list(mac_srs); 3891 } 3892 3893 /* 3894 * mac_tx_srs_setup(): 3895 * Used to setup Tx rings. If no free Tx ring is available, then default 3896 * Tx ring is used. 3897 */ 3898 void 3899 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3900 { 3901 mac_impl_t *mip = mcip->mci_mip; 3902 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 3903 int i; 3904 int tx_ring_count = 0; 3905 uint32_t soft_ring_type; 3906 mac_group_t *grp = NULL; 3907 mac_ring_t *ring; 3908 mac_srs_tx_t *tx = &tx_srs->srs_tx; 3909 boolean_t is_aggr; 3910 uint_t ring_info = 0; 3911 3912 is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0; 3913 grp = flent->fe_tx_ring_group; 3914 if (grp == NULL) { 3915 ring = (mac_ring_t *)mip->mi_default_tx_ring; 3916 goto no_group; 3917 } 3918 tx_ring_count = grp->mrg_cur_count; 3919 ring = grp->mrg_rings; 3920 /* 3921 * An attempt is made to reserve 'tx_ring_count' number 3922 * of Tx rings. If tx_ring_count is 0, default Tx ring 3923 * is used. If it is 1, an attempt is made to reserve one 3924 * Tx ring. In both the cases, the ring information is 3925 * stored in Tx SRS. If multiple Tx rings are specified, 3926 * then each Tx ring will have a Tx-side soft ring. All 3927 * these soft rings will be hang off Tx SRS. 3928 */ 3929 switch (grp->mrg_state) { 3930 case MAC_GROUP_STATE_SHARED: 3931 case MAC_GROUP_STATE_RESERVED: 3932 if (tx_ring_count <= 1 && !is_aggr) { 3933 no_group: 3934 if (ring != NULL && 3935 ring->mr_state != MR_INUSE) { 3936 (void) mac_start_ring(ring); 3937 ring_info = mac_hwring_getinfo( 3938 (mac_ring_handle_t)ring); 3939 } 3940 tx->st_arg2 = (void *)ring; 3941 mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 3942 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3943 tx->st_mode = SRS_TX_BW; 3944 } else if (mac_tx_serialize || 3945 (ring_info & MAC_RING_TX_SERIALIZE)) { 3946 tx->st_mode = SRS_TX_SERIALIZE; 3947 } else { 3948 tx->st_mode = SRS_TX_DEFAULT; 3949 } 3950 break; 3951 } 3952 soft_ring_type = ST_RING_TX; 3953 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3954 tx->st_mode = is_aggr ? 3955 SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3956 } else { 3957 tx->st_mode = is_aggr ? SRS_TX_AGGR : 3958 SRS_TX_FANOUT; 3959 } 3960 for (i = 0; i < tx_ring_count; i++) { 3961 ASSERT(ring != NULL); 3962 switch (ring->mr_state) { 3963 case MR_INUSE: 3964 case MR_FREE: 3965 ASSERT(ring->mr_srs == NULL); 3966 3967 if (ring->mr_state != MR_INUSE) 3968 (void) mac_start_ring(ring); 3969 ring_info = mac_hwring_getinfo( 3970 (mac_ring_handle_t)ring); 3971 if (mac_tx_serialize || (ring_info & 3972 MAC_RING_TX_SERIALIZE)) { 3973 soft_ring_type |= 3974 ST_RING_WORKER_ONLY; 3975 } 3976 (void) mac_soft_ring_create(i, 0, 3977 soft_ring_type, maxclsyspri, 3978 mcip, tx_srs, -1, NULL, mcip, 3979 (mac_resource_handle_t)ring); 3980 break; 3981 default: 3982 cmn_err(CE_PANIC, 3983 "srs_setup: mcip = %p " 3984 "trying to add UNKNOWN ring = %p\n", 3985 (void *)mcip, (void *)ring); 3986 break; 3987 } 3988 ring = ring->mr_next; 3989 } 3990 mac_srs_update_fanout_list(tx_srs); 3991 break; 3992 default: 3993 ASSERT(B_FALSE); 3994 break; 3995 } 3996 tx->st_func = mac_tx_get_func(tx->st_mode); 3997 if (is_aggr) { 3998 VERIFY(i_mac_capab_get((mac_handle_t)mip, 3999 MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 4000 } 4001 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 4002 int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 4003 } 4004 4005 /* 4006 * Update the fanout of a client if its recorded link speed doesn't match 4007 * its current link speed. 4008 */ 4009 void 4010 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 4011 { 4012 uint64_t link_speed; 4013 mac_resource_props_t *mcip_mrp; 4014 flow_entry_t *flent = mcip->mci_flent; 4015 mac_soft_ring_set_t *rx_srs; 4016 mac_cpus_t *srs_cpu; 4017 int soft_ring_count, maxcpus; 4018 4019 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 4020 4021 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 4022 MAC_STAT_IFSPEED); 4023 4024 if ((link_speed != 0) && 4025 (link_speed != mcip->mci_flent->fe_nic_speed)) { 4026 mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 4027 /* 4028 * Before calling mac_fanout_setup(), check to see if 4029 * the SRSes already have the right number of soft 4030 * rings. mac_fanout_setup() is a heavy duty operation 4031 * where new cpu bindings are done for SRS and soft 4032 * ring threads and interrupts re-targeted. 4033 */ 4034 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 4035 soft_ring_count = mac_compute_soft_ring_count(flent, 4036 flent->fe_rx_srs_cnt - 1, maxcpus); 4037 /* 4038 * If soft_ring_count returned by 4039 * mac_compute_soft_ring_count() is 0, bump it 4040 * up by 1 because we always have atleast one 4041 * TCP, UDP, and OTH soft ring associated with 4042 * an SRS. 4043 */ 4044 soft_ring_count = (soft_ring_count == 0) ? 4045 1 : soft_ring_count; 4046 rx_srs = flent->fe_rx_srs[0]; 4047 srs_cpu = &rx_srs->srs_cpu; 4048 if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 4049 mac_fanout_setup(mcip, flent, mcip_mrp, 4050 mac_rx_deliver, mcip, NULL, cpupart); 4051 } 4052 } 4053 } 4054 4055 /* 4056 * Walk through the list of MAC clients for the MAC. 4057 * For each active MAC client, recompute the number of soft rings 4058 * associated with every client, only if current speed is different 4059 * from the speed that was previously used for soft ring computation. 4060 * If the cable is disconnected whlie the NIC is started, we would get 4061 * notification with speed set to 0. We do not recompute in that case. 4062 */ 4063 void 4064 mac_fanout_recompute(mac_impl_t *mip) 4065 { 4066 mac_client_impl_t *mcip; 4067 cpupart_t *cpupart; 4068 boolean_t use_default; 4069 mac_resource_props_t *mrp, *emrp; 4070 4071 i_mac_perim_enter(mip); 4072 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 4073 mip->mi_linkstate != LINK_STATE_UP) { 4074 i_mac_perim_exit(mip); 4075 return; 4076 } 4077 4078 for (mcip = mip->mi_clients_list; mcip != NULL; 4079 mcip = mcip->mci_client_next) { 4080 /* Aggr port clients don't have SRSes. */ 4081 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 4082 continue; 4083 4084 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 4085 !MCIP_DATAPATH_SETUP(mcip)) 4086 continue; 4087 mrp = MCIP_RESOURCE_PROPS(mcip); 4088 emrp = MCIP_EFFECTIVE_PROPS(mcip); 4089 use_default = B_FALSE; 4090 pool_lock(); 4091 cpupart = mac_pset_find(mrp, &use_default); 4092 mac_fanout_recompute_client(mcip, cpupart); 4093 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 4094 pool_unlock(); 4095 } 4096 4097 i_mac_perim_exit(mip); 4098 } 4099 4100 /* 4101 * Given a MAC, change the polling state for all its MAC clients. 'enable' is 4102 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 4103 * default. 4104 */ 4105 void 4106 mac_poll_state_change(mac_handle_t mh, boolean_t enable) 4107 { 4108 mac_impl_t *mip = (mac_impl_t *)mh; 4109 mac_client_impl_t *mcip; 4110 4111 i_mac_perim_enter(mip); 4112 if (enable) 4113 mip->mi_state_flags &= ~MIS_POLL_DISABLE; 4114 else 4115 mip->mi_state_flags |= MIS_POLL_DISABLE; 4116 for (mcip = mip->mi_clients_list; mcip != NULL; 4117 mcip = mcip->mci_client_next) 4118 mac_client_update_classifier(mcip, B_TRUE); 4119 i_mac_perim_exit(mip); 4120 } 4121