1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/types.h> 26 #include <sys/callb.h> 27 #include <sys/cpupart.h> 28 #include <sys/pool.h> 29 #include <sys/pool_pset.h> 30 #include <sys/sdt.h> 31 #include <sys/strsubr.h> 32 #include <sys/strsun.h> 33 #include <sys/vlan.h> 34 #include <inet/ipsec_impl.h> 35 #include <inet/ip_impl.h> 36 #include <inet/sadb.h> 37 #include <inet/ipsecesp.h> 38 #include <inet/ipsecah.h> 39 40 #include <sys/mac_impl.h> 41 #include <sys/mac_client_impl.h> 42 #include <sys/mac_client_priv.h> 43 #include <sys/mac_soft_ring.h> 44 #include <sys/mac_flow_impl.h> 45 #include <sys/mac_stat.h> 46 47 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 48 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 49 static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 50 static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 51 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 52 53 static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 54 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 55 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 56 static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 57 static void mac_srs_add_glist(mac_soft_ring_set_t *); 58 static void mac_srs_remove_glist(mac_soft_ring_set_t *); 59 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 60 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 61 62 static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 63 static void mac_walk_srs_and_bind(int); 64 static void mac_walk_srs_and_unbind(int); 65 66 extern boolean_t mac_latency_optimize; 67 68 static kmem_cache_t *mac_srs_cache; 69 kmem_cache_t *mac_soft_ring_cache; 70 71 /* 72 * The duration in msec we wait before signalling the soft ring 73 * worker thread in case packets get queued. 74 */ 75 uint32_t mac_soft_ring_worker_wait = 0; 76 77 /* 78 * A global tunable for turning polling on/off. By default, dynamic 79 * polling is always on and is always very beneficial. It should be 80 * turned off with absolute care and for the rare workload (very 81 * low latency sensitive traffic). 82 */ 83 int mac_poll_enable = B_TRUE; 84 85 /* 86 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 87 * Large values could end up in consuming lot of system memory and cause 88 * system hang. 89 */ 90 int mac_soft_ring_max_q_cnt = 1024; 91 int mac_soft_ring_min_q_cnt = 256; 92 int mac_soft_ring_poll_thres = 16; 93 94 boolean_t mac_tx_serialize = B_FALSE; 95 96 /* 97 * mac_tx_srs_hiwat is the queue depth threshold at which callers of 98 * mac_tx() will be notified of flow control condition. 99 * 100 * TCP does not honour flow control condition sent up by mac_tx(). 101 * Thus provision is made for TCP to allow more packets to be queued 102 * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 103 * 104 * Note that mac_tx_srs_hiwat is always be lesser than 105 * mac_tx_srs_max_q_cnt. 106 */ 107 uint32_t mac_tx_srs_max_q_cnt = 100000; 108 uint32_t mac_tx_srs_hiwat = 1000; 109 110 /* 111 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 112 * 113 * Global tunables that determines the number of soft rings to be used for 114 * fanning out incoming traffic on a link. These count will be used only 115 * when no explicit set of CPUs was assigned to the data-links. 116 * 117 * mac_rx_soft_ring_count tunable will come into effect only if 118 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 119 * default only for sun4v platforms. 120 * 121 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 122 * 10Gbps link and is not dependent upon mac_soft_ring_enable. 123 * 124 * The number of soft rings for fanout for a link or a flow is determined 125 * by mac_compute_soft_ring_count() routine. This routine will take into 126 * account mac_soft_ring_enable, mac_rx_soft_ring_count and 127 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 128 * 129 * If a bandwidth is specified, the determination of the number of soft 130 * rings is based on specified bandwidth, CPU speed and number of CPUs in 131 * the system. 132 */ 133 uint_t mac_rx_soft_ring_count = 8; 134 uint_t mac_rx_soft_ring_10gig_count = 8; 135 136 /* 137 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 138 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 139 * list is used to walk the list of all MAC threads when a CPU is 140 * coming online or going offline. 141 */ 142 static mac_soft_ring_set_t *mac_srs_g_list = NULL; 143 static krwlock_t mac_srs_g_lock; 144 145 /* 146 * Whether the SRS threads should be bound, or not. 147 */ 148 boolean_t mac_srs_thread_bind = B_TRUE; 149 150 /* 151 * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 152 * dladm command would override this. 153 */ 154 boolean_t mac_tx_intr_retarget = B_FALSE; 155 boolean_t mac_rx_intr_retarget = B_FALSE; 156 157 /* 158 * If cpu bindings are specified by user, then Tx SRS and its soft 159 * rings should also be bound to the CPUs specified by user. The 160 * CPUs for Tx bindings are at the end of the cpu list provided by 161 * the user. If enough CPUs are not available (for Tx and Rx 162 * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 163 */ 164 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 165 processorid_t cpuid; \ 166 int i; \ 167 mac_soft_ring_t *softring; \ 168 mac_cpus_t *srs_cpu; \ 169 \ 170 srs_cpu = &mac_tx_srs->srs_cpu; \ 171 cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 172 mac_srs_worker_bind(mac_tx_srs, cpuid); \ 173 if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 174 for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 175 cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 176 softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 177 if (cpuid != -1) { \ 178 (void) mac_soft_ring_bind(softring, \ 179 cpuid); \ 180 } \ 181 } \ 182 } \ 183 } 184 185 /* 186 * Re-targeting is allowed only for exclusive group or for primary. 187 */ 188 #define RETARGETABLE_CLIENT(group, mcip) \ 189 ((((group) != NULL) && \ 190 ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 191 mac_is_primary_client(mcip)) 192 193 #define MAC_RING_RETARGETABLE(ring) \ 194 (((ring) != NULL) && \ 195 ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 196 !((ring)->mr_info.mri_intr.mi_ddi_shared)) 197 198 199 /* INIT and FINI ROUTINES */ 200 201 void 202 mac_soft_ring_init(void) 203 { 204 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 205 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 206 207 mac_srs_cache = kmem_cache_create("mac_srs_cache", 208 sizeof (mac_soft_ring_set_t), 209 64, NULL, NULL, NULL, NULL, NULL, 0); 210 211 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 212 mutex_enter(&cpu_lock); 213 register_cpu_setup_func(mac_srs_cpu_setup, NULL); 214 mutex_exit(&cpu_lock); 215 } 216 217 void 218 mac_soft_ring_finish(void) 219 { 220 mutex_enter(&cpu_lock); 221 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 222 mutex_exit(&cpu_lock); 223 rw_destroy(&mac_srs_g_lock); 224 kmem_cache_destroy(mac_soft_ring_cache); 225 kmem_cache_destroy(mac_srs_cache); 226 } 227 228 static void 229 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 230 { 231 mac_soft_ring_t *softring, *next, *head; 232 233 /* 234 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 235 * DR. The callbacks from DR are called with cpu_lock held, and hence 236 * can't wait to grab the mac perimeter. The soft ring list is hence 237 * protected for read access by srs_lock. Changing the soft ring list 238 * needs the mac perimeter and the srs_lock. 239 */ 240 mutex_enter(&mac_srs->srs_lock); 241 242 head = mac_srs->srs_soft_ring_head; 243 mac_srs->srs_soft_ring_head = NULL; 244 mac_srs->srs_soft_ring_tail = NULL; 245 mac_srs->srs_soft_ring_count = 0; 246 247 mutex_exit(&mac_srs->srs_lock); 248 249 for (softring = head; softring != NULL; softring = next) { 250 next = softring->s_ring_next; 251 mac_soft_ring_free(softring); 252 } 253 } 254 255 static void 256 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 257 { 258 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 259 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 260 261 rw_enter(&mac_srs_g_lock, RW_WRITER); 262 mutex_enter(&mac_srs->srs_lock); 263 264 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 265 266 if (mac_srs_g_list == NULL) { 267 mac_srs_g_list = mac_srs; 268 } else { 269 mac_srs->srs_next = mac_srs_g_list; 270 mac_srs_g_list->srs_prev = mac_srs; 271 mac_srs->srs_prev = NULL; 272 mac_srs_g_list = mac_srs; 273 } 274 mac_srs->srs_state |= SRS_IN_GLIST; 275 276 mutex_exit(&mac_srs->srs_lock); 277 rw_exit(&mac_srs_g_lock); 278 } 279 280 static void 281 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 282 { 283 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 284 285 rw_enter(&mac_srs_g_lock, RW_WRITER); 286 mutex_enter(&mac_srs->srs_lock); 287 288 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 289 290 if (mac_srs == mac_srs_g_list) { 291 mac_srs_g_list = mac_srs->srs_next; 292 if (mac_srs_g_list != NULL) 293 mac_srs_g_list->srs_prev = NULL; 294 } else { 295 mac_srs->srs_prev->srs_next = mac_srs->srs_next; 296 if (mac_srs->srs_next != NULL) 297 mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 298 } 299 mac_srs->srs_state &= ~SRS_IN_GLIST; 300 301 mutex_exit(&mac_srs->srs_lock); 302 rw_exit(&mac_srs_g_lock); 303 } 304 305 /* POLLING SETUP AND TEAR DOWN ROUTINES */ 306 307 /* 308 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 309 * 310 * These routines are used to call back into the upper layer 311 * (primarily TCP squeue) to stop polling the soft rings or 312 * restart polling. 313 */ 314 void 315 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 316 mac_soft_ring_set_t *mac_srs) 317 { 318 mac_soft_ring_t *softring; 319 320 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 321 322 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 323 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 324 return; 325 } 326 327 for (softring = mac_srs->srs_soft_ring_head; 328 softring != NULL; softring = softring->s_ring_next) { 329 if ((softring->s_ring_type & ST_RING_TCP) && 330 (softring->s_ring_rx_arg2 != NULL)) { 331 mcip->mci_resource_quiesce(mcip->mci_resource_arg, 332 softring->s_ring_rx_arg2); 333 } 334 } 335 } 336 337 void 338 mac_srs_client_poll_restart(mac_client_impl_t *mcip, 339 mac_soft_ring_set_t *mac_srs) 340 { 341 mac_soft_ring_t *softring; 342 343 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 344 345 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 346 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 347 return; 348 } 349 350 for (softring = mac_srs->srs_soft_ring_head; 351 softring != NULL; softring = softring->s_ring_next) { 352 if ((softring->s_ring_type & ST_RING_TCP) && 353 (softring->s_ring_rx_arg2 != NULL)) { 354 mcip->mci_resource_restart(mcip->mci_resource_arg, 355 softring->s_ring_rx_arg2); 356 } 357 } 358 } 359 360 /* 361 * Register the given SRS and associated soft rings with the consumer and 362 * enable the polling interface used by the consumer.(i.e IP) over this 363 * SRS and associated soft rings. 364 */ 365 void 366 mac_srs_client_poll_enable(mac_client_impl_t *mcip, 367 mac_soft_ring_set_t *mac_srs) 368 { 369 mac_rx_fifo_t mrf; 370 mac_soft_ring_t *softring; 371 372 ASSERT(mac_srs->srs_mcip == mcip); 373 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 374 375 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 376 return; 377 378 bzero(&mrf, sizeof (mac_rx_fifo_t)); 379 mrf.mrf_type = MAC_RX_FIFO; 380 381 /* 382 * A SRS is capable of acting as a soft ring for cases 383 * where no fanout is needed. This is the case for userland 384 * flows. 385 */ 386 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 387 return; 388 389 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 390 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 391 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 392 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 393 394 softring = mac_srs->srs_soft_ring_head; 395 while (softring != NULL) { 396 if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 397 /* 398 * TCP and UDP support DLS bypass. Squeue polling 399 * support implies DLS bypass since the squeue poll 400 * path does not have DLS processing. 401 */ 402 mac_soft_ring_dls_bypass(softring, 403 mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 404 } 405 /* 406 * Non-TCP protocols don't support squeues. Hence we don't 407 * make any ring addition callbacks for non-TCP rings 408 */ 409 if (!(softring->s_ring_type & ST_RING_TCP)) { 410 softring->s_ring_rx_arg2 = NULL; 411 softring = softring->s_ring_next; 412 continue; 413 } 414 mrf.mrf_rx_arg = softring; 415 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 416 mrf.mrf_cpu_id = softring->s_ring_cpuid; 417 mrf.mrf_flow_priority = mac_srs->srs_pri; 418 419 softring->s_ring_rx_arg2 = mcip->mci_resource_add( 420 mcip->mci_resource_arg, (mac_resource_t *)&mrf); 421 422 softring = softring->s_ring_next; 423 } 424 } 425 426 /* 427 * Unregister the given SRS and associated soft rings with the consumer and 428 * disable the polling interface used by the consumer.(i.e IP) over this 429 * SRS and associated soft rings. 430 */ 431 void 432 mac_srs_client_poll_disable(mac_client_impl_t *mcip, 433 mac_soft_ring_set_t *mac_srs) 434 { 435 mac_soft_ring_t *softring; 436 437 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 438 439 /* 440 * A SRS is capable of acting as a soft ring for cases 441 * where no protocol fanout is needed. This is the case 442 * for userland flows. Nothing to do here. 443 */ 444 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 445 return; 446 447 mutex_enter(&mac_srs->srs_lock); 448 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 449 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 450 mutex_exit(&mac_srs->srs_lock); 451 return; 452 } 453 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 454 mutex_exit(&mac_srs->srs_lock); 455 456 /* 457 * DLS bypass is now disabled in the case of both TCP and UDP. 458 * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 459 * callback. In addition, in the case of TCP, invoke IP's callback 460 * for ring removal. 461 */ 462 for (softring = mac_srs->srs_soft_ring_head; 463 softring != NULL; softring = softring->s_ring_next) { 464 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 465 continue; 466 467 if ((softring->s_ring_type & ST_RING_TCP) && 468 softring->s_ring_rx_arg2 != NULL) { 469 mcip->mci_resource_remove(mcip->mci_resource_arg, 470 softring->s_ring_rx_arg2); 471 } 472 473 mutex_enter(&softring->s_ring_lock); 474 while (softring->s_ring_state & S_RING_PROC) { 475 softring->s_ring_state |= S_RING_CLIENT_WAIT; 476 cv_wait(&softring->s_ring_client_cv, 477 &softring->s_ring_lock); 478 } 479 softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 480 softring->s_ring_rx_arg2 = NULL; 481 softring->s_ring_rx_func = mac_rx_deliver; 482 softring->s_ring_rx_arg1 = mcip; 483 mutex_exit(&softring->s_ring_lock); 484 } 485 } 486 487 /* 488 * Enable or disable poll capability of the SRS on the underlying Rx ring. 489 * 490 * There is a need to enable or disable the poll capability of an SRS over an 491 * Rx ring depending on the number of mac clients sharing the ring and also 492 * whether user flows are configured on it. However the poll state is actively 493 * manipulated by the SRS worker and poll threads and uncoordinated changes by 494 * yet another thread to the underlying capability can surprise them leading 495 * to assert failures. Instead we quiesce the SRS, make the changes and then 496 * restart the SRS. 497 */ 498 static void 499 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 500 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 501 { 502 boolean_t need_restart = B_FALSE; 503 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 504 mac_ring_t *ring; 505 506 if (!SRS_QUIESCED(mac_srs)) { 507 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 508 need_restart = B_TRUE; 509 } 510 511 ring = mac_srs->srs_ring; 512 if ((ring != NULL) && 513 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 514 if (turn_off_poll_capab) 515 mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 516 else if (mac_poll_enable) 517 mac_srs->srs_state |= SRS_POLLING_CAPAB; 518 } 519 srs_rx->sr_lower_proc = rx_func; 520 521 if (need_restart) 522 mac_rx_srs_restart(mac_srs); 523 } 524 525 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 526 527 /* 528 * Return the next CPU to be used to bind a MAC kernel thread. 529 * If a cpupart is specified, the cpu chosen must be from that 530 * cpu partition. 531 */ 532 static processorid_t 533 mac_next_bind_cpu(cpupart_t *cpupart) 534 { 535 static cpu_t *cp = NULL; 536 cpu_t *cp_start; 537 538 ASSERT(MUTEX_HELD(&cpu_lock)); 539 540 if (cp == NULL) 541 cp = cpu_list; 542 543 cp = cp->cpu_next_onln; 544 cp_start = cp; 545 546 do { 547 if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 548 return (cp->cpu_id); 549 550 } while ((cp = cp->cpu_next_onln) != cp_start); 551 552 return (NULL); 553 } 554 555 /* ARGSUSED */ 556 static int 557 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 558 { 559 ASSERT(MUTEX_HELD(&cpu_lock)); 560 switch (what) { 561 case CPU_CONFIG: 562 case CPU_ON: 563 case CPU_CPUPART_IN: 564 mac_walk_srs_and_bind(id); 565 break; 566 567 case CPU_UNCONFIG: 568 case CPU_OFF: 569 case CPU_CPUPART_OUT: 570 mac_walk_srs_and_unbind(id); 571 break; 572 573 default: 574 break; 575 } 576 return (0); 577 } 578 579 /* 580 * mac_compute_soft_ring_count(): 581 * 582 * This routine computes the number of soft rings needed to handle incoming 583 * load given a flow_entry. 584 * 585 * The routine does the following: 586 * 1) soft rings will be created if mac_soft_ring_enable is set. 587 * 2) If the underlying link is a 10Gbps link, then soft rings will be 588 * created even if mac_soft_ring_enable is not set. The number of soft 589 * rings, so created, will equal mac_rx_soft_ring_10gig_count. 590 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 591 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 592 * 10Gbps link. 593 * 594 * If a bandwidth limit is specified, the number that gets computed is 595 * dependent upon CPU speed, the number of Rx rings configured, and 596 * the bandwidth limit. 597 * If more Rx rings are available, less number of soft rings is needed. 598 * 599 * mac_use_bw_heuristic is another "hidden" variable that can be used to 600 * override the default use of soft ring count computation. Depending upon 601 * the usefulness of it, mac_use_bw_heuristic can later be made into a 602 * data-link property or removed altogether. 603 * 604 * TODO: Cleanup and tighten some of the assumptions. 605 */ 606 boolean_t mac_use_bw_heuristic = B_TRUE; 607 static int 608 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 609 { 610 uint64_t cpu_speed, bw = 0; 611 int srings = 0; 612 boolean_t bw_enabled = B_FALSE; 613 614 ASSERT(!(flent->fe_type & FLOW_USER)); 615 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 616 mac_use_bw_heuristic) { 617 /* bandwidth enabled */ 618 bw_enabled = B_TRUE; 619 bw = flent->fe_resource_props.mrp_maxbw; 620 } 621 if (!bw_enabled) { 622 /* No bandwidth enabled */ 623 if (mac_soft_ring_enable) 624 srings = mac_rx_soft_ring_count; 625 626 /* Is this a 10Gig link? */ 627 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 628 MAC_STAT_IFSPEED); 629 /* convert to Mbps */ 630 if (((flent->fe_nic_speed)/1000000) > 1000 && 631 mac_rx_soft_ring_10gig_count > 0) { 632 /* This is a 10Gig link */ 633 srings = mac_rx_soft_ring_10gig_count; 634 /* 635 * Use 2 times mac_rx_soft_ring_10gig_count for 636 * sun4v systems. 637 */ 638 if (mac_soft_ring_enable) 639 srings = srings * 2; 640 } 641 } else { 642 /* 643 * Soft ring computation using CPU speed and specified 644 * bandwidth limit. 645 */ 646 /* Assumption: all CPUs have the same frequency */ 647 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 648 649 /* cpu_speed is in MHz; make bw in units of Mbps. */ 650 bw = bw/1000000; 651 652 if (bw >= 1000) { 653 /* 654 * bw is greater than or equal to 1Gbps. 655 * The number of soft rings required is a function 656 * of bandwidth and CPU speed. To keep this simple, 657 * let's use this rule: 1GHz CPU can handle 1Gbps. 658 * If bw is less than 1 Gbps, then there is no need 659 * for soft rings. Assumption is that CPU speeds 660 * (on modern systems) are at least 1GHz. 661 */ 662 srings = bw/cpu_speed; 663 if (srings <= 1 && mac_soft_ring_enable) { 664 /* 665 * Give at least 2 soft rings 666 * for sun4v systems 667 */ 668 srings = 2; 669 } 670 } 671 } 672 /* 673 * If the flent has multiple Rx SRSs, then each SRS need not 674 * have that many soft rings on top of it. The number of 675 * soft rings for each Rx SRS is found by dividing srings by 676 * rx_srs_cnt. 677 */ 678 if (rx_srs_cnt > 1) { 679 int remainder; 680 681 remainder = srings%rx_srs_cnt; 682 srings = srings/rx_srs_cnt; 683 if (remainder != 0) 684 srings++; 685 /* 686 * Fanning out to 1 soft ring is not very useful. 687 * Set it as well to 0 and mac_srs_fanout_init() 688 * will take care of creating a single soft ring 689 * for proto fanout. 690 */ 691 if (srings == 1) 692 srings = 0; 693 } 694 /* Do some more massaging */ 695 srings = min(srings, maxcpus); 696 srings = min(srings, MAX_SR_FANOUT); 697 return (srings); 698 } 699 700 /* 701 * mac_tx_cpu_init: 702 * set up CPUs for Tx interrupt re-targeting and Tx worker 703 * thread binding 704 */ 705 static void 706 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 707 cpupart_t *cpupart) 708 { 709 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 710 mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 711 mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 712 mac_soft_ring_t *sringp; 713 mac_ring_t *ring; 714 processorid_t worker_cpuid; 715 boolean_t retargetable_client = B_FALSE; 716 int i, j; 717 718 if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 719 flent->fe_mcip)) { 720 retargetable_client = B_TRUE; 721 } 722 723 if (MAC_TX_SOFT_RINGS(tx_srs)) { 724 if (mrp != NULL) 725 j = mrp->mrp_ncpus - 1; 726 for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 727 if (mrp != NULL) { 728 if (j < 0) 729 j = mrp->mrp_ncpus - 1; 730 worker_cpuid = mrp->mrp_cpu[j]; 731 } else { 732 /* 733 * Bind interrupt to the next CPU available 734 * and leave the worker unbound. 735 */ 736 worker_cpuid = -1; 737 } 738 sringp = tx_srs->srs_tx_soft_rings[i]; 739 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 740 srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 741 if (MAC_RING_RETARGETABLE(ring) && 742 retargetable_client) { 743 mutex_enter(&cpu_lock); 744 srs_cpu->mc_tx_intr_cpu[i] = 745 (mrp != NULL) ? mrp->mrp_cpu[j] : 746 (mac_tx_intr_retarget ? 747 mac_next_bind_cpu(cpupart) : -1); 748 mutex_exit(&cpu_lock); 749 } else { 750 srs_cpu->mc_tx_intr_cpu[i] = -1; 751 } 752 if (mrp != NULL) 753 j--; 754 } 755 } else { 756 /* Tx mac_ring_handle_t is stored in st_arg2 */ 757 srs_cpu->mc_tx_fanout_cpus[0] = 758 (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 759 ring = (mac_ring_t *)srs_tx->st_arg2; 760 if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 761 mutex_enter(&cpu_lock); 762 srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 763 mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 764 (mac_tx_intr_retarget ? 765 mac_next_bind_cpu(cpupart) : -1); 766 mutex_exit(&cpu_lock); 767 } else { 768 srs_cpu->mc_tx_intr_cpu[0] = -1; 769 } 770 } 771 } 772 773 /* 774 * Assignment of user specified CPUs to a link. 775 * 776 * Minimum CPUs required to get an optimal assignmet: 777 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 778 * flag is set -- one for polling, one for fanout soft ring. 779 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 780 * for polling, one for SRS worker thread and one for fanout soft ring. 781 * 782 * The CPUs needed for Tx side is equal to the number of Tx rings 783 * the link is using. 784 * 785 * mac_flow_user_cpu_init() categorizes the CPU assignment depending 786 * upon the number of CPUs in 3 different buckets. 787 * 788 * In the first bucket, the most optimal case is handled. The user has 789 * passed enough number of CPUs and every thread gets its own CPU. 790 * 791 * The second and third are the sub-optimal cases. Enough CPUs are not 792 * available. 793 * 794 * The second bucket handles the case where atleast one distinct CPU is 795 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 796 * SRS or soft rings). 797 * 798 * In the third case (worst case scenario), specified CPU count is less 799 * than the Rx rings configured for the link. In this case, we round 800 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 801 */ 802 static void 803 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 804 { 805 mac_soft_ring_set_t *rx_srs, *tx_srs; 806 int i, srs_cnt; 807 mac_cpus_t *srs_cpu; 808 int no_of_cpus, cpu_cnt; 809 int rx_srs_cnt, reqd_rx_cpu_cnt; 810 int fanout_cpu_cnt, reqd_tx_cpu_cnt; 811 int reqd_poll_worker_cnt, fanout_cnt_per_srs; 812 mac_resource_props_t *emrp = &flent->fe_effective_props; 813 814 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 815 /* 816 * The check for nbc_ncpus to be within limits for 817 * the user specified case was done earlier and if 818 * not within limits, an error would have been 819 * returned to the user. 820 */ 821 ASSERT(mrp->mrp_ncpus > 0); 822 823 no_of_cpus = mrp->mrp_ncpus; 824 825 if (mrp->mrp_rx_intr_cpu != -1) { 826 /* 827 * interrupt has been re-targetted. Poll 828 * thread needs to be bound to interrupt 829 * CPU. 830 * 831 * Find where in the list is the intr 832 * CPU and swap it with the first one. 833 * We will be using the first CPU in the 834 * list for poll. 835 */ 836 for (i = 0; i < no_of_cpus; i++) { 837 if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 838 break; 839 } 840 mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 841 mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 842 } 843 844 /* 845 * Requirements: 846 * The number of CPUs that each Rx ring needs is dependent 847 * upon mac_latency_optimize flag. 848 * 1) If set, atleast 2 CPUs are needed -- one for 849 * polling, one for fanout soft ring. 850 * 2) If not set, then atleast 3 CPUs are needed -- one 851 * for polling, one for srs worker thread, and one for 852 * fanout soft ring. 853 */ 854 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 855 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 856 reqd_rx_cpu_cnt = mac_latency_optimize ? 857 (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 858 859 /* How many CPUs are needed for Tx side? */ 860 tx_srs = flent->fe_tx_srs; 861 reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 862 tx_srs->srs_tx_ring_count : 1; 863 864 /* CPUs needed for Rx SRSes poll and worker threads */ 865 reqd_poll_worker_cnt = mac_latency_optimize ? 866 rx_srs_cnt : rx_srs_cnt * 2; 867 868 /* Has the user provided enough CPUs? */ 869 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 870 /* 871 * Best case scenario. There is enough CPUs. All 872 * Rx rings will get their own set of CPUs plus 873 * Tx soft rings will get their own. 874 */ 875 /* 876 * fanout_cpu_cnt is the number of CPUs available 877 * for Rx side fanout soft rings. 878 */ 879 fanout_cpu_cnt = no_of_cpus - 880 reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 881 882 /* 883 * Divide fanout_cpu_cnt by rx_srs_cnt to find 884 * out how many fanout soft rings each Rx SRS 885 * can have. 886 */ 887 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 888 889 /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 890 fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 891 892 /* Do the assignment for the default Rx ring */ 893 cpu_cnt = 0; 894 rx_srs = flent->fe_rx_srs[0]; 895 ASSERT(rx_srs->srs_ring == NULL); 896 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 897 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 898 srs_cpu = &rx_srs->srs_cpu; 899 srs_cpu->mc_ncpus = no_of_cpus; 900 bcopy(mrp->mrp_cpu, 901 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 902 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 903 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 904 /* Retarget the interrupt to the same CPU as the poll */ 905 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 906 srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 907 srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 908 for (i = 0; i < fanout_cnt_per_srs; i++) 909 srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 910 911 /* Do the assignment for h/w Rx SRSes */ 912 if (flent->fe_rx_srs_cnt > 1) { 913 cpu_cnt = 0; 914 for (srs_cnt = 1; 915 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 916 rx_srs = flent->fe_rx_srs[srs_cnt]; 917 ASSERT(rx_srs->srs_ring != NULL); 918 if (rx_srs->srs_fanout_state == 919 SRS_FANOUT_INIT) { 920 rx_srs->srs_fanout_state = 921 SRS_FANOUT_REINIT; 922 } 923 srs_cpu = &rx_srs->srs_cpu; 924 srs_cpu->mc_ncpus = no_of_cpus; 925 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 926 sizeof (srs_cpu->mc_cpus)); 927 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 928 /* The first CPU in the list is the intr CPU */ 929 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 930 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 931 srs_cpu->mc_rx_workerid = 932 (mac_latency_optimize ? 933 srs_cpu->mc_rx_pollid : 934 mrp->mrp_cpu[cpu_cnt++]); 935 for (i = 0; i < fanout_cnt_per_srs; i++) { 936 srs_cpu->mc_rx_fanout_cpus[i] = 937 mrp->mrp_cpu[cpu_cnt++]; 938 } 939 ASSERT(cpu_cnt <= no_of_cpus); 940 } 941 } 942 goto tx_cpu_init; 943 } 944 945 /* 946 * Sub-optimal case. 947 * We have the following information: 948 * no_of_cpus - no. of cpus that user passed. 949 * rx_srs_cnt - no. of rx rings. 950 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 951 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 952 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 953 */ 954 /* 955 * If we bind the Rx fanout soft rings to the same CPUs 956 * as poll/worker, would that be enough? 957 */ 958 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 959 boolean_t worker_assign = B_FALSE; 960 961 /* 962 * If mac_latency_optimize is not set, are there 963 * enough CPUs to assign a CPU for worker also? 964 */ 965 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 966 worker_assign = B_TRUE; 967 /* 968 * Zero'th Rx SRS is the default Rx ring. It is not 969 * associated with h/w Rx ring. 970 */ 971 rx_srs = flent->fe_rx_srs[0]; 972 ASSERT(rx_srs->srs_ring == NULL); 973 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 974 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 975 cpu_cnt = 0; 976 srs_cpu = &rx_srs->srs_cpu; 977 srs_cpu->mc_ncpus = no_of_cpus; 978 bcopy(mrp->mrp_cpu, 979 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 980 srs_cpu->mc_rx_fanout_cnt = 1; 981 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 982 /* Retarget the interrupt to the same CPU as the poll */ 983 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 984 srs_cpu->mc_rx_workerid = 985 ((!mac_latency_optimize && worker_assign) ? 986 mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 987 988 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 989 990 /* Do CPU bindings for SRSes having h/w Rx rings */ 991 if (flent->fe_rx_srs_cnt > 1) { 992 cpu_cnt = 0; 993 for (srs_cnt = 1; 994 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 995 rx_srs = flent->fe_rx_srs[srs_cnt]; 996 ASSERT(rx_srs->srs_ring != NULL); 997 if (rx_srs->srs_fanout_state == 998 SRS_FANOUT_INIT) { 999 rx_srs->srs_fanout_state = 1000 SRS_FANOUT_REINIT; 1001 } 1002 srs_cpu = &rx_srs->srs_cpu; 1003 srs_cpu->mc_ncpus = no_of_cpus; 1004 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1005 sizeof (srs_cpu->mc_cpus)); 1006 srs_cpu->mc_rx_pollid = 1007 mrp->mrp_cpu[cpu_cnt]; 1008 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1009 srs_cpu->mc_rx_workerid = 1010 ((!mac_latency_optimize && worker_assign) ? 1011 mrp->mrp_cpu[++cpu_cnt] : 1012 srs_cpu->mc_rx_pollid); 1013 srs_cpu->mc_rx_fanout_cnt = 1; 1014 srs_cpu->mc_rx_fanout_cpus[0] = 1015 mrp->mrp_cpu[cpu_cnt]; 1016 cpu_cnt++; 1017 ASSERT(cpu_cnt <= no_of_cpus); 1018 } 1019 } 1020 goto tx_cpu_init; 1021 } 1022 1023 /* 1024 * Real sub-optimal case. Not enough CPUs for poll and 1025 * Tx soft rings. Do a round robin assignment where 1026 * each Rx SRS will get the same CPU for poll, worker 1027 * and fanout soft ring. 1028 */ 1029 cpu_cnt = 0; 1030 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1031 rx_srs = flent->fe_rx_srs[srs_cnt]; 1032 srs_cpu = &rx_srs->srs_cpu; 1033 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1034 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1035 srs_cpu->mc_ncpus = no_of_cpus; 1036 bcopy(mrp->mrp_cpu, 1037 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 1038 srs_cpu->mc_rx_fanout_cnt = 1; 1039 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 1040 /* Retarget the interrupt to the same CPU as the poll */ 1041 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1042 srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 1043 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1044 if (++cpu_cnt >= no_of_cpus) 1045 cpu_cnt = 0; 1046 } 1047 1048 tx_cpu_init: 1049 mac_tx_cpu_init(flent, mrp, NULL); 1050 1051 /* 1052 * Copy the user specified CPUs to the effective CPUs 1053 */ 1054 for (i = 0; i < mrp->mrp_ncpus; i++) { 1055 emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 1056 } 1057 emrp->mrp_ncpus = mrp->mrp_ncpus; 1058 emrp->mrp_mask = mrp->mrp_mask; 1059 bzero(emrp->mrp_pool, MAXPATHLEN); 1060 } 1061 1062 /* 1063 * mac_flow_cpu_init(): 1064 * 1065 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1066 * the CPU binding information in srs_cpu for all Rx SRSes associated 1067 * with a flent. 1068 */ 1069 static void 1070 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1071 { 1072 mac_soft_ring_set_t *rx_srs; 1073 processorid_t cpuid; 1074 int i, j, k, srs_cnt, nscpus, maxcpus, soft_ring_cnt = 0; 1075 mac_cpus_t *srs_cpu; 1076 mac_resource_props_t *emrp = &flent->fe_effective_props; 1077 uint32_t cpus[MRP_NCPUS]; 1078 1079 /* 1080 * The maximum number of CPUs available can either be 1081 * the number of CPUs in the pool or the number of CPUs 1082 * in the system. 1083 */ 1084 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 1085 1086 /* 1087 * Compute the number of soft rings needed on top for each Rx 1088 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1089 * associated with h/w Rx rings. Soft ring count needed for 1090 * each h/w Rx SRS is computed and the same is applied to 1091 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1092 * is the software classified Rx SRS. 1093 */ 1094 soft_ring_cnt = mac_compute_soft_ring_count(flent, 1095 flent->fe_rx_srs_cnt - 1, maxcpus); 1096 if (soft_ring_cnt == 0) { 1097 /* 1098 * Even when soft_ring_cnt is 0, we still need 1099 * to create a soft ring for TCP, UDP and 1100 * OTHER. So set it to 1. 1101 */ 1102 soft_ring_cnt = 1; 1103 } 1104 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1105 rx_srs = flent->fe_rx_srs[srs_cnt]; 1106 srs_cpu = &rx_srs->srs_cpu; 1107 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1108 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1109 srs_cpu->mc_ncpus = soft_ring_cnt; 1110 srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1111 mutex_enter(&cpu_lock); 1112 for (j = 0; j < soft_ring_cnt; j++) { 1113 cpuid = mac_next_bind_cpu(cpupart); 1114 srs_cpu->mc_cpus[j] = cpuid; 1115 srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1116 } 1117 cpuid = mac_next_bind_cpu(cpupart); 1118 srs_cpu->mc_rx_pollid = cpuid; 1119 srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 1120 srs_cpu->mc_rx_pollid : -1); 1121 /* increment ncpus to account for polling cpu */ 1122 srs_cpu->mc_ncpus++; 1123 srs_cpu->mc_cpus[j++] = cpuid; 1124 if (!mac_latency_optimize) { 1125 cpuid = mac_next_bind_cpu(cpupart); 1126 srs_cpu->mc_ncpus++; 1127 srs_cpu->mc_cpus[j++] = cpuid; 1128 } 1129 srs_cpu->mc_rx_workerid = cpuid; 1130 mutex_exit(&cpu_lock); 1131 } 1132 1133 nscpus = 0; 1134 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1135 rx_srs = flent->fe_rx_srs[srs_cnt]; 1136 srs_cpu = &rx_srs->srs_cpu; 1137 for (j = 0; j < srs_cpu->mc_ncpus; j++) { 1138 cpus[nscpus++] = srs_cpu->mc_cpus[j]; 1139 } 1140 } 1141 1142 1143 /* 1144 * Copy cpu list to fe_effective_props 1145 * without duplicates. 1146 */ 1147 k = 0; 1148 for (i = 0; i < nscpus; i++) { 1149 for (j = 0; j < k; j++) { 1150 if (emrp->mrp_cpu[j] == cpus[i]) 1151 break; 1152 } 1153 if (j == k) 1154 emrp->mrp_cpu[k++] = cpus[i]; 1155 } 1156 emrp->mrp_ncpus = k; 1157 1158 mac_tx_cpu_init(flent, NULL, cpupart); 1159 } 1160 1161 /* 1162 * DATAPATH SETUP ROUTINES 1163 * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1164 */ 1165 1166 /* 1167 * mac_srs_fanout_list_alloc: 1168 * 1169 * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 1170 * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 1171 * array space is needed to store Tx soft rings. Thus we allocate so 1172 * much array space for srs_tx_soft_rings. 1173 * 1174 * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 1175 * of space to st_soft_rings. This array is used for quick access to 1176 * soft ring associated with a pseudo Tx ring based on the pseudo 1177 * ring's index (mr_index). 1178 */ 1179 static void 1180 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1181 { 1182 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1183 1184 if (mac_srs->srs_type & SRST_TX) { 1185 mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 1186 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1187 MAX_RINGS_PER_GROUP, KM_SLEEP); 1188 if (mcip->mci_state_flags & MCIS_IS_AGGR) { 1189 mac_srs_tx_t *tx = &mac_srs->srs_tx; 1190 1191 tx->st_soft_rings = (mac_soft_ring_t **) 1192 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1193 MAX_RINGS_PER_GROUP, KM_SLEEP); 1194 } 1195 } else { 1196 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1197 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1198 KM_SLEEP); 1199 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1200 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1201 KM_SLEEP); 1202 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1203 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1204 KM_SLEEP); 1205 } 1206 } 1207 1208 static void 1209 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1210 { 1211 cpu_t *cp; 1212 boolean_t clear = B_FALSE; 1213 1214 ASSERT(MUTEX_HELD(&cpu_lock)); 1215 1216 if (!mac_srs_thread_bind) 1217 return; 1218 1219 cp = cpu_get(cpuid); 1220 if (cp == NULL || !cpu_is_online(cp)) 1221 return; 1222 1223 mutex_enter(&mac_srs->srs_lock); 1224 mac_srs->srs_state |= SRS_WORKER_BOUND; 1225 if (mac_srs->srs_worker_cpuid != -1) 1226 clear = B_TRUE; 1227 mac_srs->srs_worker_cpuid = cpuid; 1228 mutex_exit(&mac_srs->srs_lock); 1229 1230 if (clear) 1231 thread_affinity_clear(mac_srs->srs_worker); 1232 1233 thread_affinity_set(mac_srs->srs_worker, cpuid); 1234 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1235 } 1236 1237 static void 1238 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1239 { 1240 cpu_t *cp; 1241 boolean_t clear = B_FALSE; 1242 1243 ASSERT(MUTEX_HELD(&cpu_lock)); 1244 1245 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1246 return; 1247 1248 cp = cpu_get(cpuid); 1249 if (cp == NULL || !cpu_is_online(cp)) 1250 return; 1251 1252 mutex_enter(&mac_srs->srs_lock); 1253 mac_srs->srs_state |= SRS_POLL_BOUND; 1254 if (mac_srs->srs_poll_cpuid != -1) 1255 clear = B_TRUE; 1256 mac_srs->srs_poll_cpuid = cpuid; 1257 mutex_exit(&mac_srs->srs_lock); 1258 1259 if (clear) 1260 thread_affinity_clear(mac_srs->srs_poll_thr); 1261 1262 thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1263 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1264 } 1265 1266 /* 1267 * Re-target interrupt to the passed CPU. If re-target is successful, 1268 * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 1269 */ 1270 void 1271 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1272 { 1273 cpu_t *cp; 1274 mac_ring_t *ring = mac_srs->srs_ring; 1275 mac_intr_t *mintr = &ring->mr_info.mri_intr; 1276 flow_entry_t *flent = mac_srs->srs_flent; 1277 boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 1278 1279 ASSERT(MUTEX_HELD(&cpu_lock)); 1280 1281 /* 1282 * Don't re-target the interrupt for these cases: 1283 * 1) ring is NULL 1284 * 2) the interrupt is shared (mi_ddi_shared) 1285 * 3) ddi_handle is NULL and !primary 1286 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 1287 * Case 3 & 4 are because of mac_client_intr_cpu() routine. 1288 * This routine will re-target fixed interrupt for primary 1289 * mac client if the client has only one ring. In that 1290 * case, mc_rx_intr_cpu will already have the correct value. 1291 */ 1292 if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 1293 (mintr->mi_ddi_handle == NULL && !primary) || (primary && 1294 mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 1295 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1296 return; 1297 } 1298 1299 if (mintr->mi_ddi_handle == NULL) 1300 return; 1301 1302 cp = cpu_get(cpuid); 1303 if (cp == NULL || !cpu_is_online(cp)) 1304 return; 1305 1306 /* Drop the cpu_lock as ddi_intr_set_affinity() holds it */ 1307 mutex_exit(&cpu_lock); 1308 if (ddi_intr_set_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 1309 mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 1310 else 1311 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1312 mutex_enter(&cpu_lock); 1313 } 1314 1315 /* 1316 * Re-target Tx interrupts 1317 */ 1318 void 1319 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 1320 { 1321 cpu_t *cp; 1322 mac_ring_t *ring; 1323 mac_intr_t *mintr; 1324 mac_soft_ring_t *sringp; 1325 mac_srs_tx_t *srs_tx; 1326 mac_cpus_t *srs_cpu; 1327 processorid_t cpuid; 1328 int i; 1329 1330 ASSERT(MUTEX_HELD(&cpu_lock)); 1331 1332 srs_cpu = &mac_srs->srs_cpu; 1333 if (MAC_TX_SOFT_RINGS(mac_srs)) { 1334 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 1335 sringp = mac_srs->srs_tx_soft_rings[i]; 1336 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 1337 cpuid = srs_cpu->mc_tx_intr_cpu[i]; 1338 cp = cpu_get(cpuid); 1339 if (cp == NULL || !cpu_is_online(cp) || 1340 !MAC_RING_RETARGETABLE(ring)) { 1341 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1342 continue; 1343 } 1344 mintr = &ring->mr_info.mri_intr; 1345 /* 1346 * Drop the cpu_lock as ddi_intr_set_affinity() 1347 * holds it 1348 */ 1349 mutex_exit(&cpu_lock); 1350 if (ddi_intr_set_affinity(mintr->mi_ddi_handle, 1351 cpuid) == DDI_SUCCESS) { 1352 srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 1353 } else { 1354 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1355 } 1356 mutex_enter(&cpu_lock); 1357 } 1358 } else { 1359 cpuid = srs_cpu->mc_tx_intr_cpu[0]; 1360 cp = cpu_get(cpuid); 1361 if (cp == NULL || !cpu_is_online(cp)) { 1362 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1363 return; 1364 } 1365 srs_tx = &mac_srs->srs_tx; 1366 ring = (mac_ring_t *)srs_tx->st_arg2; 1367 if (MAC_RING_RETARGETABLE(ring)) { 1368 mintr = &ring->mr_info.mri_intr; 1369 mutex_exit(&cpu_lock); 1370 if ((ddi_intr_set_affinity(mintr->mi_ddi_handle, 1371 cpuid) == DDI_SUCCESS)) { 1372 srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 1373 } else { 1374 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1375 } 1376 mutex_enter(&cpu_lock); 1377 } 1378 } 1379 } 1380 1381 /* 1382 * When a CPU comes back online, bind the MAC kernel threads which 1383 * were previously bound to that CPU, and had to be unbound because 1384 * the CPU was going away. 1385 * 1386 * These functions are called with cpu_lock held and hence we can't 1387 * cv_wait to grab the mac perimeter. Since these functions walk the soft 1388 * ring list of an SRS without being in the perimeter, the list itself 1389 * is protected by the SRS lock. 1390 */ 1391 static void 1392 mac_walk_srs_and_bind(int cpuid) 1393 { 1394 mac_soft_ring_set_t *mac_srs; 1395 mac_soft_ring_t *soft_ring; 1396 1397 rw_enter(&mac_srs_g_lock, RW_READER); 1398 1399 if ((mac_srs = mac_srs_g_list) == NULL) 1400 goto done; 1401 1402 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1403 if (mac_srs->srs_worker_cpuid == -1 && 1404 mac_srs->srs_worker_cpuid_save == cpuid) { 1405 mac_srs->srs_worker_cpuid_save = -1; 1406 mac_srs_worker_bind(mac_srs, cpuid); 1407 } 1408 1409 if (!(mac_srs->srs_type & SRST_TX)) { 1410 if (mac_srs->srs_poll_cpuid == -1 && 1411 mac_srs->srs_poll_cpuid_save == cpuid) { 1412 mac_srs->srs_poll_cpuid_save = -1; 1413 mac_srs_poll_bind(mac_srs, cpuid); 1414 } 1415 } 1416 1417 /* Next tackle the soft rings associated with the srs */ 1418 mutex_enter(&mac_srs->srs_lock); 1419 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1420 soft_ring = soft_ring->s_ring_next) { 1421 if (soft_ring->s_ring_cpuid == -1 && 1422 soft_ring->s_ring_cpuid_save == cpuid) { 1423 soft_ring->s_ring_cpuid_save = -1; 1424 (void) mac_soft_ring_bind(soft_ring, cpuid); 1425 } 1426 } 1427 mutex_exit(&mac_srs->srs_lock); 1428 } 1429 done: 1430 rw_exit(&mac_srs_g_lock); 1431 } 1432 1433 /* 1434 * Change the priority of the SRS's poll and worker thread. Additionally, 1435 * update the priority of the worker threads for the SRS's soft rings. 1436 * Need to modify any associated squeue threads. 1437 */ 1438 void 1439 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1440 { 1441 mac_soft_ring_t *ringp; 1442 1443 mac_srs->srs_pri = prival; 1444 thread_lock(mac_srs->srs_worker); 1445 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1446 thread_unlock(mac_srs->srs_worker); 1447 if (mac_srs->srs_poll_thr != NULL) { 1448 thread_lock(mac_srs->srs_poll_thr); 1449 (void) thread_change_pri(mac_srs->srs_poll_thr, 1450 mac_srs->srs_pri, 0); 1451 thread_unlock(mac_srs->srs_poll_thr); 1452 } 1453 if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1454 return; 1455 while (ringp != mac_srs->srs_soft_ring_tail) { 1456 thread_lock(ringp->s_ring_worker); 1457 (void) thread_change_pri(ringp->s_ring_worker, 1458 mac_srs->srs_pri, 0); 1459 thread_unlock(ringp->s_ring_worker); 1460 ringp = ringp->s_ring_next; 1461 } 1462 ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1463 thread_lock(ringp->s_ring_worker); 1464 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1465 thread_unlock(ringp->s_ring_worker); 1466 } 1467 1468 /* 1469 * Change the receive bandwidth limit. 1470 */ 1471 static void 1472 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1473 { 1474 mac_soft_ring_t *softring; 1475 1476 mutex_enter(&srs->srs_lock); 1477 mutex_enter(&srs->srs_bw->mac_bw_lock); 1478 1479 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1480 /* Reset bandwidth limit */ 1481 if (srs->srs_type & SRST_BW_CONTROL) { 1482 softring = srs->srs_soft_ring_head; 1483 while (softring != NULL) { 1484 softring->s_ring_type &= ~ST_RING_BW_CTL; 1485 softring = softring->s_ring_next; 1486 } 1487 srs->srs_type &= ~SRST_BW_CONTROL; 1488 srs->srs_drain_func = mac_rx_srs_drain; 1489 } 1490 } else { 1491 /* Set/Modify bandwidth limit */ 1492 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1493 /* 1494 * Give twice the queuing capability before 1495 * dropping packets. The unit is bytes/tick. 1496 */ 1497 srs->srs_bw->mac_bw_drop_threshold = 1498 srs->srs_bw->mac_bw_limit << 1; 1499 if (!(srs->srs_type & SRST_BW_CONTROL)) { 1500 softring = srs->srs_soft_ring_head; 1501 while (softring != NULL) { 1502 softring->s_ring_type |= ST_RING_BW_CTL; 1503 softring = softring->s_ring_next; 1504 } 1505 srs->srs_type |= SRST_BW_CONTROL; 1506 srs->srs_drain_func = mac_rx_srs_drain_bw; 1507 } 1508 } 1509 done: 1510 mutex_exit(&srs->srs_bw->mac_bw_lock); 1511 mutex_exit(&srs->srs_lock); 1512 } 1513 1514 /* Change the transmit bandwidth limit */ 1515 static void 1516 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1517 { 1518 uint32_t tx_mode, ring_info = 0; 1519 mac_srs_tx_t *srs_tx = &srs->srs_tx; 1520 mac_client_impl_t *mcip = srs->srs_mcip; 1521 1522 /* 1523 * We need to quiesce/restart the client here because mac_tx() and 1524 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1525 * st_mode and related fields, which are modified by the code below. 1526 */ 1527 mac_tx_client_quiesce((mac_client_handle_t)mcip); 1528 1529 mutex_enter(&srs->srs_lock); 1530 mutex_enter(&srs->srs_bw->mac_bw_lock); 1531 1532 tx_mode = srs_tx->st_mode; 1533 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1534 /* Reset bandwidth limit */ 1535 if (tx_mode == SRS_TX_BW) { 1536 if (srs_tx->st_arg2 != NULL) 1537 ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1538 if (mac_tx_serialize || 1539 (ring_info & MAC_RING_TX_SERIALIZE)) { 1540 srs_tx->st_mode = SRS_TX_SERIALIZE; 1541 } else { 1542 srs_tx->st_mode = SRS_TX_DEFAULT; 1543 } 1544 } else if (tx_mode == SRS_TX_BW_FANOUT) { 1545 srs_tx->st_mode = SRS_TX_FANOUT; 1546 } else if (tx_mode == SRS_TX_BW_AGGR) { 1547 srs_tx->st_mode = SRS_TX_AGGR; 1548 } 1549 srs->srs_type &= ~SRST_BW_CONTROL; 1550 } else { 1551 /* Set/Modify bandwidth limit */ 1552 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1553 /* 1554 * Give twice the queuing capability before 1555 * dropping packets. The unit is bytes/tick. 1556 */ 1557 srs->srs_bw->mac_bw_drop_threshold = 1558 srs->srs_bw->mac_bw_limit << 1; 1559 srs->srs_type |= SRST_BW_CONTROL; 1560 if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 1561 tx_mode != SRS_TX_BW_AGGR) { 1562 if (tx_mode == SRS_TX_SERIALIZE || 1563 tx_mode == SRS_TX_DEFAULT) { 1564 srs_tx->st_mode = SRS_TX_BW; 1565 } else if (tx_mode == SRS_TX_FANOUT) { 1566 srs_tx->st_mode = SRS_TX_BW_FANOUT; 1567 } else if (tx_mode == SRS_TX_AGGR) { 1568 srs_tx->st_mode = SRS_TX_BW_AGGR; 1569 } else { 1570 ASSERT(0); 1571 } 1572 } 1573 } 1574 done: 1575 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1576 mutex_exit(&srs->srs_bw->mac_bw_lock); 1577 mutex_exit(&srs->srs_lock); 1578 1579 mac_tx_client_restart((mac_client_handle_t)mcip); 1580 } 1581 1582 /* 1583 * The uber function that deals with any update to bandwidth limits. 1584 */ 1585 void 1586 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1587 { 1588 int count; 1589 1590 for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1591 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1592 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1593 } 1594 1595 void 1596 mac_srs_change_upcall(void *arg, mac_direct_rx_t rx_func, void *rx_arg1) 1597 { 1598 mac_soft_ring_set_t *mac_srs = arg; 1599 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 1600 mac_soft_ring_t *softring; 1601 1602 mutex_enter(&mac_srs->srs_lock); 1603 ASSERT((mac_srs->srs_type & SRST_TX) == 0); 1604 srs_rx->sr_func = rx_func; 1605 srs_rx->sr_arg1 = rx_arg1; 1606 1607 softring = mac_srs->srs_soft_ring_head; 1608 while (softring != NULL) { 1609 mutex_enter(&softring->s_ring_lock); 1610 softring->s_ring_rx_func = rx_func; 1611 softring->s_ring_rx_arg1 = rx_arg1; 1612 mutex_exit(&softring->s_ring_lock); 1613 softring = softring->s_ring_next; 1614 } 1615 1616 mutex_exit(&mac_srs->srs_lock); 1617 } 1618 1619 /* 1620 * When the first sub-flow is added to a link, we disable polling on the 1621 * link and also modify the entry point to mac_rx_srs_subflow_process. 1622 * (polling is disabled because with the subflow added, accounting 1623 * for polling needs additional logic, it is assumed that when a subflow is 1624 * added, we can take some hit as a result of disabling polling rather than 1625 * adding more complexity - if this becomes a perf. issue we need to 1626 * re-rvaluate this logic). When the last subflow is removed, we turn back 1627 * polling and also reset the entry point to mac_rx_srs_process. 1628 * 1629 * In the future if there are multiple SRS, we can simply 1630 * take one and give it to the flow rather than disabling polling and 1631 * resetting the entry point. 1632 */ 1633 void 1634 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1635 { 1636 flow_entry_t *flent = mcip->mci_flent; 1637 int i; 1638 mac_impl_t *mip = mcip->mci_mip; 1639 mac_rx_func_t rx_func; 1640 uint_t rx_srs_cnt; 1641 boolean_t enable_classifier; 1642 1643 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1644 1645 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1646 1647 rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1648 mac_rx_srs_process; 1649 1650 /* Tell mac_srs_poll_state_change to disable polling if necessary */ 1651 if (mip->mi_state_flags & MIS_POLL_DISABLE) 1652 enable_classifier = B_TRUE; 1653 1654 /* 1655 * If receive function has already been configured correctly for 1656 * current subflow configuration, do nothing. 1657 */ 1658 if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1659 return; 1660 1661 rx_srs_cnt = flent->fe_rx_srs_cnt; 1662 for (i = 0; i < rx_srs_cnt; i++) { 1663 ASSERT(flent->fe_rx_srs[i] != NULL); 1664 mac_srs_poll_state_change(flent->fe_rx_srs[i], 1665 enable_classifier, rx_func); 1666 } 1667 1668 /* 1669 * Change the S/W classifier so that we can land in the 1670 * correct processing function with correct argument. 1671 * If all subflows have been removed we can revert to 1672 * mac_rx_srsprocess, else we need mac_rx_srs_subflow_process. 1673 */ 1674 mutex_enter(&flent->fe_lock); 1675 flent->fe_cb_fn = (flow_fn_t)rx_func; 1676 flent->fe_cb_arg1 = (void *)mip; 1677 flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1678 mutex_exit(&flent->fe_lock); 1679 } 1680 1681 static void 1682 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1683 { 1684 int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1685 mac_soft_ring_t *softring; 1686 1687 softring = mac_srs->srs_soft_ring_head; 1688 if (softring == NULL) { 1689 ASSERT(mac_srs->srs_soft_ring_count == 0); 1690 mac_srs->srs_tcp_ring_count = 0; 1691 mac_srs->srs_udp_ring_count = 0; 1692 mac_srs->srs_oth_ring_count = 0; 1693 mac_srs->srs_tx_ring_count = 0; 1694 return; 1695 } 1696 1697 while (softring != NULL) { 1698 if (softring->s_ring_type & ST_RING_TCP) { 1699 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1700 } else if (softring->s_ring_type & ST_RING_UDP) { 1701 mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1702 } else if (softring->s_ring_type & ST_RING_OTH) { 1703 mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1704 } else { 1705 ASSERT(softring->s_ring_type & ST_RING_TX); 1706 mac_srs->srs_tx_soft_rings[tx_count++] = softring; 1707 } 1708 softring = softring->s_ring_next; 1709 } 1710 1711 ASSERT(mac_srs->srs_soft_ring_count == 1712 (tcp_count + udp_count + oth_count + tx_count)); 1713 mac_srs->srs_tcp_ring_count = tcp_count; 1714 mac_srs->srs_udp_ring_count = udp_count; 1715 mac_srs->srs_oth_ring_count = oth_count; 1716 mac_srs->srs_tx_ring_count = tx_count; 1717 } 1718 1719 void 1720 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 1721 mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1722 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1723 mac_resource_handle_t x_arg2, boolean_t set_bypass) 1724 { 1725 mac_soft_ring_t *softring; 1726 mac_rx_fifo_t mrf; 1727 1728 bzero(&mrf, sizeof (mac_rx_fifo_t)); 1729 mrf.mrf_type = MAC_RX_FIFO; 1730 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1731 mrf.mrf_intr_enable = 1732 (mac_intr_enable_t)mac_soft_ring_intr_enable; 1733 mrf.mrf_intr_disable = 1734 (mac_intr_disable_t)mac_soft_ring_intr_disable; 1735 mrf.mrf_flow_priority = pri; 1736 1737 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1738 (type|ST_RING_TCP), pri, mcip, mac_srs, 1739 cpuid, rx_func, x_arg1, x_arg2); 1740 softring->s_ring_rx_arg2 = NULL; 1741 1742 /* 1743 * TCP and UDP support DLS bypass. In addition TCP 1744 * squeue can also poll their corresponding soft rings. 1745 */ 1746 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1747 mac_soft_ring_dls_bypass(softring, 1748 mcip->mci_direct_rx_fn, 1749 mcip->mci_direct_rx_arg); 1750 1751 mrf.mrf_rx_arg = softring; 1752 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1753 1754 /* 1755 * Make a call in IP to get a TCP squeue assigned to 1756 * this softring to maintain full CPU locality through 1757 * the stack and allow the squeue to be able to poll 1758 * the softring so the flow control can be pushed 1759 * all the way to H/W. 1760 */ 1761 softring->s_ring_rx_arg2 = 1762 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1763 (mac_resource_t *)&mrf); 1764 } 1765 1766 /* 1767 * Non-TCP protocols don't support squeues. Hence we 1768 * don't make any ring addition callbacks for non-TCP 1769 * rings. Now create the UDP softring and allow it to 1770 * bypass the DLS layer. 1771 */ 1772 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1773 (type|ST_RING_UDP), pri, mcip, mac_srs, 1774 cpuid, rx_func, x_arg1, x_arg2); 1775 softring->s_ring_rx_arg2 = NULL; 1776 1777 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1778 mac_soft_ring_dls_bypass(softring, 1779 mcip->mci_direct_rx_fn, 1780 mcip->mci_direct_rx_arg); 1781 } 1782 1783 /* Create the Oth softrings which has to go through the DLS */ 1784 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1785 (type|ST_RING_OTH), pri, mcip, mac_srs, 1786 cpuid, rx_func, x_arg1, x_arg2); 1787 softring->s_ring_rx_arg2 = NULL; 1788 } 1789 1790 /* 1791 * This routine associates a CPU or a set of CPU to process incoming 1792 * traffic from a mac client. If multiple CPUs are specified, then 1793 * so many soft rings are created with each soft ring worker thread 1794 * bound to a CPU in the set. Each soft ring in turn will be 1795 * associated with an squeue and the squeue will be moved to the 1796 * same CPU as that of the soft ring's. 1797 */ 1798 static void 1799 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 1800 void *x_arg1, mac_resource_handle_t x_arg2, 1801 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1802 { 1803 mac_soft_ring_t *softring; 1804 uint32_t soft_ring_flag = 0; 1805 processorid_t cpuid = -1; 1806 int i, srings_present, new_fanout_cnt; 1807 mac_cpus_t *srs_cpu; 1808 1809 /* fanout state is REINIT. Set it back to INIT */ 1810 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1811 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1812 1813 /* how many are present right now */ 1814 srings_present = mac_rx_srs->srs_tcp_ring_count; 1815 /* new request */ 1816 srs_cpu = &mac_rx_srs->srs_cpu; 1817 new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1818 1819 mutex_enter(&mac_rx_srs->srs_lock); 1820 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1821 soft_ring_flag |= ST_RING_BW_CTL; 1822 mutex_exit(&mac_rx_srs->srs_lock); 1823 1824 if (new_fanout_cnt > srings_present) { 1825 /* soft rings increased */ 1826 mutex_enter(&mac_rx_srs->srs_lock); 1827 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1828 mutex_exit(&mac_rx_srs->srs_lock); 1829 1830 for (i = mac_rx_srs->srs_tcp_ring_count; 1831 i < new_fanout_cnt; i++) { 1832 /* 1833 * Create the protocol softrings and set the 1834 * DLS bypass where possible. 1835 */ 1836 mac_srs_create_proto_softrings(i, soft_ring_flag, 1837 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1838 rx_func, x_arg1, x_arg2, B_TRUE); 1839 } 1840 mac_srs_update_fanout_list(mac_rx_srs); 1841 } else if (new_fanout_cnt < srings_present) { 1842 /* soft rings decreased */ 1843 if (new_fanout_cnt == 1) { 1844 mutex_enter(&mac_rx_srs->srs_lock); 1845 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1846 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1847 mutex_exit(&mac_rx_srs->srs_lock); 1848 } 1849 /* Get rid of extra soft rings */ 1850 for (i = new_fanout_cnt; 1851 i < mac_rx_srs->srs_tcp_ring_count; i++) { 1852 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1853 if (softring->s_ring_rx_arg2 != NULL) { 1854 mcip->mci_resource_remove( 1855 (void *)mcip->mci_resource_arg, 1856 softring->s_ring_rx_arg2); 1857 } 1858 mac_soft_ring_remove(mac_rx_srs, 1859 mac_rx_srs->srs_tcp_soft_rings[i]); 1860 mac_soft_ring_remove(mac_rx_srs, 1861 mac_rx_srs->srs_udp_soft_rings[i]); 1862 mac_soft_ring_remove(mac_rx_srs, 1863 mac_rx_srs->srs_oth_soft_rings[i]); 1864 } 1865 mac_srs_update_fanout_list(mac_rx_srs); 1866 } 1867 1868 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1869 mutex_enter(&cpu_lock); 1870 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1871 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1872 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1873 cpuid); 1874 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1875 cpuid); 1876 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1877 cpuid); 1878 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1879 if (softring->s_ring_rx_arg2 != NULL) { 1880 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1881 softring->s_ring_rx_arg2, cpuid); 1882 } 1883 } 1884 1885 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1886 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1887 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1888 /* 1889 * Bind Tx srs and soft ring threads too. Let's bind tx 1890 * srs to the last cpu in mrp list. 1891 */ 1892 if (mac_tx_srs != NULL) { 1893 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1894 mac_tx_srs_retarget_intr(mac_tx_srs); 1895 } 1896 mutex_exit(&cpu_lock); 1897 } 1898 1899 /* 1900 * Bind SRS threads and soft rings to CPUs/create fanout list. 1901 */ 1902 void 1903 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 1904 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1905 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 1906 cpupart_t *cpupart) 1907 { 1908 int i; 1909 processorid_t cpuid; 1910 uint32_t soft_ring_flag = 0; 1911 int soft_ring_cnt; 1912 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1913 1914 /* 1915 * Remove the no soft ring flag and we will adjust it 1916 * appropriately further down. 1917 */ 1918 mutex_enter(&mac_rx_srs->srs_lock); 1919 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1920 mutex_exit(&mac_rx_srs->srs_lock); 1921 1922 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1923 1924 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1925 soft_ring_flag |= ST_RING_BW_CTL; 1926 1927 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1928 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1929 /* 1930 * Ring count can be 0 if no fanout is required and no cpu 1931 * were specified. Leave the SRS worker and poll thread 1932 * unbound 1933 */ 1934 ASSERT(mrp != NULL); 1935 soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1936 1937 /* Step 1: bind cpu contains cpu list where threads need to bind */ 1938 if (soft_ring_cnt > 0) { 1939 mutex_enter(&cpu_lock); 1940 for (i = 0; i < soft_ring_cnt; i++) { 1941 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1942 /* Create the protocol softrings */ 1943 mac_srs_create_proto_softrings(i, soft_ring_flag, 1944 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1945 rx_func, x_arg1, x_arg2, B_FALSE); 1946 } 1947 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1948 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1949 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1950 /* 1951 * Bind Tx srs and soft ring threads too. 1952 * Let's bind tx srs to the last cpu in 1953 * mrp list. 1954 */ 1955 if (mac_tx_srs == NULL) { 1956 mutex_exit(&cpu_lock); 1957 goto alldone; 1958 } 1959 1960 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1961 mac_tx_srs_retarget_intr(mac_tx_srs); 1962 mutex_exit(&cpu_lock); 1963 } else { 1964 mutex_enter(&cpu_lock); 1965 /* 1966 * For a subflow, mrp_workerid and mrp_pollid 1967 * is not set. 1968 */ 1969 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 1970 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1971 mutex_exit(&cpu_lock); 1972 goto no_softrings; 1973 } 1974 1975 alldone: 1976 if (soft_ring_cnt > 1) 1977 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1978 mac_srs_update_fanout_list(mac_rx_srs); 1979 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1980 return; 1981 1982 no_softrings: 1983 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1984 mutex_enter(&cpu_lock); 1985 cpuid = mac_next_bind_cpu(cpupart); 1986 /* Create the protocol softrings */ 1987 mac_srs_create_proto_softrings(0, soft_ring_flag, 1988 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1989 rx_func, x_arg1, x_arg2, B_FALSE); 1990 mutex_exit(&cpu_lock); 1991 } else { 1992 /* 1993 * This is the case when there is no fanout which is 1994 * true for subflows. 1995 */ 1996 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1997 } 1998 mac_srs_update_fanout_list(mac_rx_srs); 1999 mac_srs_client_poll_enable(mcip, mac_rx_srs); 2000 } 2001 2002 /* 2003 * mac_fanout_setup: 2004 * 2005 * Calls mac_srs_fanout_init() or modify() depending upon whether 2006 * the SRS is getting initialized or re-initialized. 2007 */ 2008 void 2009 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2010 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 2011 mac_resource_handle_t x_arg2, cpupart_t *cpupart) 2012 { 2013 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 2014 int i, rx_srs_cnt; 2015 2016 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 2017 /* 2018 * This is an aggregation port. Fanout will be setup 2019 * over the aggregation itself. 2020 */ 2021 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) 2022 return; 2023 2024 mac_rx_srs = flent->fe_rx_srs[0]; 2025 /* 2026 * Set up the fanout on the tx side only once, with the 2027 * first rx SRS. The CPU binding, fanout, and bandwidth 2028 * criteria are common to both RX and TX, so 2029 * initializing them along side avoids redundant code. 2030 */ 2031 mac_tx_srs = flent->fe_tx_srs; 2032 rx_srs_cnt = flent->fe_rx_srs_cnt; 2033 2034 /* No fanout for subflows */ 2035 if (flent->fe_type & FLOW_USER) { 2036 mac_srs_fanout_init(mcip, mrp, rx_func, 2037 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2038 cpupart); 2039 return; 2040 } 2041 2042 if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 2043 mac_flow_user_cpu_init(flent, mrp); 2044 else 2045 mac_flow_cpu_init(flent, cpupart); 2046 2047 mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2048 2049 /* 2050 * Set up fanout for both SW (0th SRS) and HW classified 2051 * SRS (the rest of Rx SRSs in flent). 2052 */ 2053 for (i = 0; i < rx_srs_cnt; i++) { 2054 mac_rx_srs = flent->fe_rx_srs[i]; 2055 if (i != 0) 2056 mac_tx_srs = NULL; 2057 switch (mac_rx_srs->srs_fanout_state) { 2058 case SRS_FANOUT_UNINIT: 2059 mac_srs_fanout_init(mcip, mrp, rx_func, 2060 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2061 cpupart); 2062 break; 2063 case SRS_FANOUT_INIT: 2064 break; 2065 case SRS_FANOUT_REINIT: 2066 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 2067 mac_srs_fanout_modify(mcip, rx_func, x_arg1, 2068 x_arg2, mac_rx_srs, mac_tx_srs); 2069 mac_rx_srs_restart(mac_rx_srs); 2070 break; 2071 default: 2072 VERIFY(mac_rx_srs->srs_fanout_state <= 2073 SRS_FANOUT_REINIT); 2074 break; 2075 } 2076 } 2077 } 2078 2079 /* 2080 * mac_srs_create: 2081 * 2082 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2083 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2084 * processing is created. 2085 * 2086 * Details on Rx SRS: 2087 * Create a SRS and also add the necessary soft rings for TCP and 2088 * non-TCP based on fanout type and count specified. 2089 * 2090 * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2091 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2092 * to be heavily modified. 2093 * 2094 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2095 */ 2096 mac_soft_ring_set_t * 2097 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2098 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2099 mac_ring_t *ring) 2100 { 2101 mac_soft_ring_set_t *mac_srs; 2102 mac_srs_rx_t *srs_rx; 2103 mac_srs_tx_t *srs_tx; 2104 mac_bw_ctl_t *mac_bw; 2105 mac_resource_props_t *mrp; 2106 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2107 2108 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2109 bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2110 srs_rx = &mac_srs->srs_rx; 2111 srs_tx = &mac_srs->srs_tx; 2112 2113 mutex_enter(&flent->fe_lock); 2114 2115 /* 2116 * Get the bandwidth control structure from the flent. Get 2117 * rid of any residual values in the control structure for 2118 * the tx bw struct and also for the rx, if the rx srs is 2119 * the 1st one being brought up (the rx bw ctl struct may 2120 * be shared by multiple SRSs) 2121 */ 2122 if (is_tx_srs) { 2123 mac_srs->srs_bw = &flent->fe_tx_bw; 2124 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2125 flent->fe_tx_srs = mac_srs; 2126 } else { 2127 /* 2128 * The bw counter (stored in the flent) is shared 2129 * by SRS's within an rx group. 2130 */ 2131 mac_srs->srs_bw = &flent->fe_rx_bw; 2132 /* First rx SRS, clear the bw structure */ 2133 if (flent->fe_rx_srs_cnt == 0) 2134 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2135 2136 /* 2137 * It is better to panic here rather than just assert because 2138 * on a non-debug kernel we might end up courrupting memory 2139 * and making it difficult to debug. 2140 */ 2141 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 2142 panic("Array Overrun detected due to MAC client %p " 2143 " having more rings than %d", (void *)mcip, 2144 MAX_RINGS_PER_GROUP); 2145 } 2146 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2147 flent->fe_rx_srs_cnt++; 2148 } 2149 mac_srs->srs_flent = flent; 2150 mutex_exit(&flent->fe_lock); 2151 2152 mac_srs->srs_state = 0; 2153 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2154 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2155 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 2156 mac_srs->srs_mcip = mcip; 2157 mac_srs_fanout_list_alloc(mac_srs); 2158 2159 /* 2160 * For a flow we use the underlying MAC client's priority range with 2161 * the priority value to find an absolute priority value. For a MAC 2162 * client we use the MAC client's maximum priority as the value. 2163 */ 2164 mrp = &flent->fe_effective_props; 2165 if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2166 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2167 mcip->mci_max_pri, mrp->mrp_priority); 2168 } else { 2169 mac_srs->srs_pri = mcip->mci_max_pri; 2170 } 2171 /* 2172 * We need to insert the SRS in the global list before 2173 * binding the SRS and SR threads. Otherwise there is a 2174 * is a small window where the cpu reconfig callbacks 2175 * may miss the SRS in the list walk and DR could fail 2176 * as there are bound threads. 2177 */ 2178 mac_srs_add_glist(mac_srs); 2179 2180 /* Initialize bw limit */ 2181 if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2182 mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2183 2184 mac_bw = mac_srs->srs_bw; 2185 mutex_enter(&mac_bw->mac_bw_lock); 2186 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2187 2188 /* 2189 * Give twice the queuing capability before 2190 * dropping packets. The unit is bytes/tick. 2191 */ 2192 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2193 mutex_exit(&mac_bw->mac_bw_lock); 2194 mac_srs->srs_type |= SRST_BW_CONTROL; 2195 } else { 2196 mac_srs->srs_drain_func = mac_rx_srs_drain; 2197 } 2198 2199 /* 2200 * We use the following policy to control Receive 2201 * Side Dynamic Polling: 2202 * 1) We switch to poll mode anytime the processing thread causes 2203 * a backlog to build up in SRS and its associated Soft Rings 2204 * (sr_poll_pkt_cnt > 0). 2205 * 2) As long as the backlog stays under the low water mark 2206 * (sr_lowat), we poll the H/W for more packets. 2207 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2208 * stay in poll mode but don't poll the H/W for more packets. 2209 * 4) Anytime in polling mode, if we poll the H/W for packets and 2210 * find nothing plus we have an existing backlog 2211 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2212 * the H/W for packets anymore (let the polling thread go to sleep). 2213 * 5) Once the backlog is relived (packets are processed) we reenable 2214 * polling (by signalling the poll thread) only when the backlog 2215 * dips below sr_poll_thres. 2216 * 6) sr_hiwat is used exclusively when we are not polling capable 2217 * and is used to decide when to drop packets so the SRS queue 2218 * length doesn't grow infinitely. 2219 */ 2220 if (!is_tx_srs) { 2221 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2222 /* Low water mark needs to be less than high water mark */ 2223 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2224 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2225 (mac_soft_ring_max_q_cnt >> 2); 2226 /* Poll threshold need to be half of low water mark or less */ 2227 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2228 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2229 (srs_rx->sr_lowat >> 1); 2230 if (mac_latency_optimize) 2231 mac_srs->srs_state |= SRS_LATENCY_OPT; 2232 else 2233 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2234 } 2235 2236 mac_srs->srs_worker = thread_create(NULL, 0, 2237 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2238 2239 if (is_tx_srs) { 2240 /* Handle everything about Tx SRS and return */ 2241 mac_srs->srs_drain_func = mac_tx_srs_drain; 2242 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2243 srs_tx->st_hiwat = 2244 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2245 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2246 srs_tx->st_arg1 = x_arg1; 2247 srs_tx->st_arg2 = x_arg2; 2248 goto done; 2249 } 2250 2251 if ((srs_type & SRST_FLOW) != 0 || 2252 FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2253 srs_rx->sr_lower_proc = mac_rx_srs_process; 2254 else 2255 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2256 2257 srs_rx->sr_func = rx_func; 2258 srs_rx->sr_arg1 = x_arg1; 2259 srs_rx->sr_arg2 = x_arg2; 2260 2261 if (ring != NULL) { 2262 uint_t ring_info; 2263 2264 /* Is the mac_srs created over the RX default group? */ 2265 if (ring->mr_gh == (mac_group_handle_t) 2266 MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2267 mac_srs->srs_type |= SRST_DEFAULT_GRP; 2268 } 2269 mac_srs->srs_ring = ring; 2270 ring->mr_srs = mac_srs; 2271 ring->mr_classify_type = MAC_HW_CLASSIFIER; 2272 ring->mr_flag |= MR_INCIPIENT; 2273 2274 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 2275 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2276 mac_srs->srs_state |= SRS_POLLING_CAPAB; 2277 2278 mac_srs->srs_poll_thr = thread_create(NULL, 0, 2279 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2280 mac_srs->srs_pri); 2281 /* 2282 * Some drivers require serialization and don't send 2283 * packet chains in interrupt context. For such 2284 * drivers, we should always queue in soft ring 2285 * so that we get a chance to switch into a polling 2286 * mode under backlog. 2287 */ 2288 ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 2289 if (ring_info & MAC_RING_RX_ENQUEUE) 2290 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2291 } 2292 done: 2293 mac_srs_stat_create(mac_srs); 2294 return (mac_srs); 2295 } 2296 2297 /* 2298 * Figure out the number of soft rings required. Its dependant on 2299 * if protocol fanout is required (for LINKs), global settings 2300 * require us to do fanout for performance (based on mac_soft_ring_enable), 2301 * or user has specifically requested fanout. 2302 */ 2303 static uint32_t 2304 mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2305 { 2306 uint32_t fanout_type; 2307 mac_resource_props_t *mrp = &flent->fe_effective_props; 2308 2309 /* no fanout for subflows */ 2310 switch (link_type) { 2311 case SRST_FLOW: 2312 fanout_type = SRST_NO_SOFT_RINGS; 2313 break; 2314 case SRST_LINK: 2315 fanout_type = SRST_FANOUT_PROTO; 2316 break; 2317 } 2318 2319 /* A primary NIC/link is being plumbed */ 2320 if (flent->fe_type & FLOW_PRIMARY_MAC) { 2321 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2322 fanout_type |= SRST_FANOUT_SRC_IP; 2323 } 2324 } else if (flent->fe_type & FLOW_VNIC) { 2325 /* A VNIC is being created */ 2326 if (mrp != NULL && mrp->mrp_ncpus > 0) { 2327 fanout_type |= SRST_FANOUT_SRC_IP; 2328 } 2329 } 2330 2331 return (fanout_type); 2332 } 2333 2334 /* 2335 * Change a group from h/w to s/w classification. 2336 */ 2337 void 2338 mac_rx_switch_grp_to_sw(mac_group_t *group) 2339 { 2340 mac_ring_t *ring; 2341 mac_soft_ring_set_t *mac_srs; 2342 2343 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2344 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2345 /* 2346 * Remove the SRS associated with the HW ring. 2347 * As a result, polling will be disabled. 2348 */ 2349 mac_srs = ring->mr_srs; 2350 ASSERT(mac_srs != NULL); 2351 mac_rx_srs_remove(mac_srs); 2352 ring->mr_srs = NULL; 2353 } 2354 2355 if (ring->mr_state != MR_INUSE) 2356 (void) mac_start_ring(ring); 2357 2358 /* 2359 * We need to perform SW classification 2360 * for packets landing in these rings 2361 */ 2362 ring->mr_flag = 0; 2363 ring->mr_classify_type = MAC_SW_CLASSIFIER; 2364 } 2365 } 2366 2367 /* 2368 * Create the Rx SRS for S/W classifier and for each ring in the 2369 * group (if exclusive group). Also create the Tx SRS. 2370 */ 2371 void 2372 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2373 uint32_t link_type) 2374 { 2375 cpupart_t *cpupart; 2376 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2377 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2378 boolean_t use_default = B_FALSE; 2379 2380 mac_rx_srs_group_setup(mcip, flent, link_type); 2381 mac_tx_srs_group_setup(mcip, flent, link_type); 2382 2383 pool_lock(); 2384 cpupart = mac_pset_find(mrp, &use_default); 2385 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2386 mac_rx_deliver, mcip, NULL, cpupart); 2387 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 2388 pool_unlock(); 2389 } 2390 2391 /* 2392 * Set up the RX SRSs. If the S/W SRS is not set, set it up, if there 2393 * is a group associated with this MAC client, set up SRSs for individual 2394 * h/w rings. 2395 */ 2396 void 2397 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2398 uint32_t link_type) 2399 { 2400 mac_impl_t *mip = mcip->mci_mip; 2401 mac_soft_ring_set_t *mac_srs; 2402 mac_ring_t *ring; 2403 uint32_t fanout_type; 2404 mac_group_t *rx_group = flent->fe_rx_ring_group; 2405 2406 fanout_type = mac_find_fanout(flent, link_type); 2407 2408 /* Create the SRS for S/W classification if none exists */ 2409 if (flent->fe_rx_srs[0] == NULL) { 2410 ASSERT(flent->fe_rx_srs_cnt == 0); 2411 /* Setup the Rx SRS */ 2412 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2413 mac_rx_deliver, mcip, NULL, NULL); 2414 mutex_enter(&flent->fe_lock); 2415 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2416 flent->fe_cb_arg1 = (void *)mip; 2417 flent->fe_cb_arg2 = (void *)mac_srs; 2418 mutex_exit(&flent->fe_lock); 2419 } 2420 2421 if (rx_group == NULL) 2422 return; 2423 /* 2424 * fanout for default SRS is done when default SRS are created 2425 * above. As each ring is added to the group, we setup the 2426 * SRS and fanout to it. 2427 */ 2428 switch (rx_group->mrg_state) { 2429 case MAC_GROUP_STATE_RESERVED: 2430 for (ring = rx_group->mrg_rings; ring != NULL; 2431 ring = ring->mr_next) { 2432 switch (ring->mr_state) { 2433 case MR_INUSE: 2434 case MR_FREE: 2435 if (ring->mr_srs != NULL) 2436 break; 2437 if (ring->mr_state != MR_INUSE) 2438 (void) mac_start_ring(ring); 2439 2440 /* 2441 * Since the group is exclusively ours create 2442 * an SRS for this ring to allow the 2443 * individual SRS to dynamically poll the 2444 * ring. Do this only if the client is not 2445 * a VLAN MAC client, since for VLAN we do 2446 * s/w classification for the VID check, and 2447 * if it has a unicast address. 2448 */ 2449 if ((mcip->mci_state_flags & 2450 MCIS_NO_UNICAST_ADDR) || 2451 i_mac_flow_vid(mcip->mci_flent) != 2452 VLAN_ID_NONE) { 2453 break; 2454 } 2455 mac_srs = mac_srs_create(mcip, flent, 2456 fanout_type | link_type, 2457 mac_rx_deliver, mcip, NULL, ring); 2458 break; 2459 default: 2460 cmn_err(CE_PANIC, 2461 "srs_setup: mcip = %p " 2462 "trying to add UNKNOWN ring = %p\n", 2463 (void *)mcip, (void *)ring); 2464 break; 2465 } 2466 } 2467 break; 2468 case MAC_GROUP_STATE_SHARED: 2469 /* 2470 * Set all rings of this group to software classified. 2471 * 2472 * If the group is current RESERVED, the existing mac 2473 * client (the only client on this group) is using 2474 * this group exclusively. In that case we need to 2475 * disable polling on the rings of the group (if it 2476 * was enabled), and free the SRS associated with the 2477 * rings. 2478 */ 2479 mac_rx_switch_grp_to_sw(rx_group); 2480 break; 2481 default: 2482 ASSERT(B_FALSE); 2483 break; 2484 } 2485 } 2486 2487 /* 2488 * Set up the TX SRS. 2489 */ 2490 void 2491 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2492 uint32_t link_type) 2493 { 2494 int cnt; 2495 int ringcnt; 2496 mac_ring_t *ring; 2497 mac_group_t *grp; 2498 2499 /* 2500 * If we are opened exclusively (like aggr does for aggr_ports), 2501 * don't set up Tx SRS and Tx soft rings as they won't be used. 2502 * The same thing has to be done for Rx side also. See bug: 2503 * 6880080 2504 */ 2505 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 2506 /* 2507 * If we have rings, start them here. 2508 */ 2509 if (flent->fe_tx_ring_group == NULL) 2510 return; 2511 grp = (mac_group_t *)flent->fe_tx_ring_group; 2512 ringcnt = grp->mrg_cur_count; 2513 ring = grp->mrg_rings; 2514 for (cnt = 0; cnt < ringcnt; cnt++) { 2515 if (ring->mr_state != MR_INUSE) { 2516 (void) mac_start_ring(ring); 2517 } 2518 ring = ring->mr_next; 2519 } 2520 return; 2521 } 2522 if (flent->fe_tx_srs == NULL) { 2523 (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 2524 NULL, mcip, NULL, NULL); 2525 } 2526 mac_tx_srs_setup(mcip, flent); 2527 } 2528 2529 /* 2530 * Remove all the RX SRSs. If we want to remove only the SRSs associated 2531 * with h/w rings, leave the S/W SRS alone. This is used when we want to 2532 * move the MAC client from one group to another, so we need to teardown 2533 * on the h/w SRSs. 2534 */ 2535 void 2536 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 2537 { 2538 mac_soft_ring_set_t *mac_srs; 2539 int i; 2540 int count = flent->fe_rx_srs_cnt; 2541 2542 for (i = 0; i < count; i++) { 2543 if (i == 0 && hwonly) 2544 continue; 2545 mac_srs = flent->fe_rx_srs[i]; 2546 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2547 mac_srs_free(mac_srs); 2548 flent->fe_rx_srs[i] = NULL; 2549 flent->fe_rx_srs_cnt--; 2550 } 2551 ASSERT(!hwonly || flent->fe_rx_srs_cnt == 1); 2552 ASSERT(hwonly || flent->fe_rx_srs_cnt == 0); 2553 } 2554 2555 /* 2556 * Remove the TX SRS. 2557 */ 2558 void 2559 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2560 uint32_t link_type) 2561 { 2562 mac_soft_ring_set_t *tx_srs; 2563 mac_srs_tx_t *tx; 2564 2565 if ((tx_srs = flent->fe_tx_srs) == NULL) 2566 return; 2567 2568 tx = &tx_srs->srs_tx; 2569 switch (link_type) { 2570 case SRST_FLOW: 2571 /* 2572 * For flows, we need to work with passed 2573 * flent to find the Rx/Tx SRS. 2574 */ 2575 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2576 break; 2577 case SRST_LINK: 2578 mac_tx_client_condemn((mac_client_handle_t)mcip); 2579 if (tx->st_arg2 != NULL) { 2580 ASSERT(tx_srs->srs_type & SRST_TX); 2581 /* 2582 * The ring itself will be stopped when 2583 * we release the group or in the 2584 * mac_datapath_teardown (for the default 2585 * group) 2586 */ 2587 tx->st_arg2 = NULL; 2588 } 2589 break; 2590 default: 2591 ASSERT(B_FALSE); 2592 break; 2593 } 2594 mac_srs_free(tx_srs); 2595 flent->fe_tx_srs = NULL; 2596 } 2597 2598 /* 2599 * This is the group state machine. 2600 * 2601 * The state of an Rx group is given by 2602 * the following table. The default group and its rings are started in 2603 * mac_start itself and the default group stays in SHARED state until 2604 * mac_stop at which time the group and rings are stopped and and it 2605 * reverts to the Registered state. 2606 * 2607 * Typically this function is called on a group after adding or removing a 2608 * client from it, to find out what should be the new state of the group. 2609 * If the new state is RESERVED, then the client that owns this group 2610 * exclusively is also returned. Note that adding or removing a client from 2611 * a group could also impact the default group and the caller needs to 2612 * evaluate the effect on the default group. 2613 * 2614 * Group type # of clients mi_nactiveclients Group State 2615 * in the group 2616 * 2617 * Non-default 0 N.A. REGISTERED 2618 * Non-default 1 N.A. RESERVED 2619 * 2620 * Default 0 N.A. SHARED 2621 * Default 1 1 RESERVED 2622 * Default 1 > 1 SHARED 2623 * Default > 1 N.A. SHARED 2624 * 2625 * For a TX group, the following is the state table. 2626 * 2627 * Group type # of clients Group State 2628 * in the group 2629 * 2630 * Non-default 0 REGISTERED 2631 * Non-default 1 RESERVED 2632 * 2633 * Default 0 REGISTERED 2634 * Default 1 RESERVED 2635 * Default > 1 SHARED 2636 */ 2637 mac_group_state_t 2638 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 2639 mac_group_t *defgrp, boolean_t rx_group) 2640 { 2641 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2642 2643 *group_only_mcip = NULL; 2644 2645 /* Non-default group */ 2646 2647 if (grp != defgrp) { 2648 if (MAC_GROUP_NO_CLIENT(grp)) 2649 return (MAC_GROUP_STATE_REGISTERED); 2650 2651 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2652 if (*group_only_mcip != NULL) 2653 return (MAC_GROUP_STATE_RESERVED); 2654 2655 return (MAC_GROUP_STATE_SHARED); 2656 } 2657 2658 /* Default group */ 2659 2660 if (MAC_GROUP_NO_CLIENT(grp)) { 2661 if (rx_group) 2662 return (MAC_GROUP_STATE_SHARED); 2663 else 2664 return (MAC_GROUP_STATE_REGISTERED); 2665 } 2666 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2667 if (*group_only_mcip == NULL) 2668 return (MAC_GROUP_STATE_SHARED); 2669 2670 if (rx_group && mip->mi_nactiveclients != 1) 2671 return (MAC_GROUP_STATE_SHARED); 2672 2673 ASSERT(*group_only_mcip != NULL); 2674 return (MAC_GROUP_STATE_RESERVED); 2675 } 2676 2677 /* 2678 * OVERVIEW NOTES FOR DATAPATH 2679 * =========================== 2680 * 2681 * Create an SRS and setup the corresponding flow function and args. 2682 * Add a classification rule for the flow specified by 'flent' and program 2683 * the hardware classifier when applicable. 2684 * 2685 * Rx ring assignment, SRS, polling and B/W enforcement 2686 * ---------------------------------------------------- 2687 * 2688 * We try to use H/W classification on NIC and assign traffic to a 2689 * MAC address to a particular Rx ring. There is a 1-1 mapping 2690 * between a SRS and a Rx ring. The SRS (short for soft ring set) 2691 * dynamically switches the underlying Rx ring between interrupt 2692 * and polling mode and enforces any specified B/W control. 2693 * 2694 * There is always a SRS created and tied to each H/W and S/W rule. 2695 * Whenever we create a H/W rule, we always add the the same rule to 2696 * S/W classifier and tie a SRS to it. 2697 * 2698 * In case a B/W control is specified, its broken into bytes 2699 * per ticks and as soon as the quota for a tick is exhausted, 2700 * the underlying Rx ring is forced into poll mode for remianing 2701 * tick. The SRS poll thread only polls for bytes that are 2702 * allowed to come in the SRS. We typically let 4x the configured 2703 * B/W worth of packets to come in the SRS (to prevent unnecessary 2704 * drops due to bursts) but only process the specified amount. 2705 * 2706 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2707 * Rx rings (and corresponding SRSs) assigned to it. The SRS 2708 * in turn can have softrings to do protocol level fanout or 2709 * softrings to do S/W based fanout or both. In case the NIC 2710 * has no Rx rings, we do S/W classification to respective SRS. 2711 * The S/W classification rule is always setup and ready. This 2712 * allows the MAC layer to reassign Rx rings whenever needed 2713 * but packets still continue to flow via the default path and 2714 * getting S/W classified to correct SRS. 2715 * 2716 * In other cases where a NIC or VNIC is plumbed, our goal is use 2717 * H/W classifier and get two Rx ring assigned for the Link. One 2718 * for TCP and one for UDP|SCTP. The respective SRS still do the 2719 * polling on the Rx ring. For Link that is plumbed for IP, there 2720 * is a TCP squeue which also does polling and can control the 2721 * the Rx ring directly (where SRS is just pass through). For 2722 * the following cases, the SRS does the polling underneath. 2723 * 1) non IP based Links (Links which are not plumbed via ifconfig) 2724 * and paths which have no IP squeues (UDP & SCTP) 2725 * 2) If B/W control is specified on the Link 2726 * 3) If S/W fanout is secified 2727 * 2728 * Note1: As of current implementation, we try to assign only 1 Rx 2729 * ring per Link and more than 1 Rx ring for primary Link for 2730 * H/W based fanout. We always create following softrings per SRS: 2731 * 1) TCP softring which is polled by TCP squeue where possible 2732 * (and also bypasses DLS) 2733 * 2) UDP/SCTP based which bypasses DLS 2734 * 3) OTH softring which goes via DLS (currently deal with IPv6 2735 * and non TCP/UDP/SCTP for IPv4 packets). 2736 * 2737 * It is necessary to create 3 softrings since SRS has to poll 2738 * the single Rx ring underneath and enforce any link level B/W 2739 * control (we can't switch the Rx ring in poll mode just based 2740 * on TCP squeue if the same Rx ring is sharing UDP and other 2741 * traffic as well). Once polling is done and any Link level B/W 2742 * control is specified, the packets are assigned to respective 2743 * softring based on protocol. Since TCP has IP based squeue 2744 * which benefits by polling, we separate TCP packets into 2745 * its own softring which can be polled by IP squeue. We need 2746 * to separate out UDP/SCTP to UDP softring since it can bypass 2747 * the DLS layer which has heavy performance advanatges and we 2748 * need a softring (OTH) for rest. 2749 * 2750 * ToDo: The 3 softrings for protocol are needed only till we can 2751 * get rid of DLS from datapath, make IPv4 and IPv6 paths 2752 * symmetric (deal with mac_header_info for v6 and polling for 2753 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2754 * are generic), and bring SAP based classification to MAC layer 2755 * 2756 * H/W and S/W based fanout and multiple Rx rings per Link 2757 * ------------------------------------------------------- 2758 * 2759 * In case, fanout is requested (or determined automatically based 2760 * on Link speed and processor speed), we try to assign multiple 2761 * Rx rings per Link with their respective SRS. In this case 2762 * the NIC should be capable of fanning out incoming packets between 2763 * the assigned Rx rings (H/W based fanout). All the SRS 2764 * individually switch their Rx ring between interrupt and polling 2765 * mode but share a common B/W control counter in case of Link 2766 * level B/W is specified. 2767 * 2768 * If S/W based fanout is specified in lieu of H/W based fanout, 2769 * the Link SRS creates the specified number of softrings for 2770 * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2771 * out to the correct softring based on their protocol and 2772 * protocol specific hash function. 2773 * 2774 * Primary and non primary MAC clients 2775 * ----------------------------------- 2776 * 2777 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2778 * and are a Layer 2 construct. 2779 * 2780 * Primary NIC: 2781 * The Link that owns the primary MAC address and typically 2782 * is used as the data NIC in non virtualized cases. As such 2783 * H/W resources are preferntially given to primary NIC. As 2784 * far as code is concerned, there is no difference in the 2785 * primary NIC vs VNICs. They are all treated as Links. 2786 * At the very first call to mac_unicast_add() we program the S/W 2787 * classifier for the primary MAC address, get a soft ring set 2788 * (and soft rings based on 'ip_soft_ring_cnt') 2789 * and a Rx ring assigned for polling to get enabled. 2790 * When IP get plumbed and negotiates polling, we can 2791 * let squeue do the polling on TCP softring. 2792 * 2793 * VNICs: 2794 * Same as any other Link. As long as the H/W resource assignments 2795 * are equal, the data path and setup for all Links is same. 2796 * 2797 * Flows: 2798 * Can be configured on Links. They have their own SRS and the 2799 * S/W classifier is programmed appropriately based on the flow. 2800 * The flows typically deal with layer 3 and above and 2801 * creates a soft ring set specific to the flow. The receive 2802 * side function is switched from mac_rx_srs_process to 2803 * mac_rx_srs_subflow_process which first tries to assign the 2804 * packet to appropriate flow SRS and failing which assigns it 2805 * to link SRS. This allows us to avoid the layered approach 2806 * which gets complex. 2807 * 2808 * By the time mac_datapath_setup() completes, we already have the 2809 * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2810 * and S/W classifiers programmed. IP is not plumbed yet (and might 2811 * never be for Virtual Machines guest OS path). When IP is plumbed 2812 * (for both NIC and VNIC), we do a capability negotiation for polling 2813 * and upcall functions etc. 2814 * 2815 * Rx ring Assignement NOTES 2816 * ------------------------- 2817 * 2818 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2819 * as NIC with a single default ring), we assign the only ring to 2820 * primary Link. The primary Link SRS can do polling on it as long as 2821 * it is the only link in use and we compare the MAC address for unicast 2822 * packets before accepting an incoming packet (there is no need for S/W 2823 * classification in this case). We disable polling on the only ring the 2824 * moment 2nd link gets created (the polling remains enabled even though 2825 * there are broadcast and * multicast flows created). 2826 * 2827 * If the NIC has more than 1 Rx ring, we assign the default ring (the 2828 * 1st ring) to deal with broadcast, multicast and traffic for other 2829 * NICs which needs S/W classification. We assign the primary mac 2830 * addresses to another ring by specifiying a classification rule for 2831 * primary unicast MAC address to the selected ring. The primary Link 2832 * (and its SRS) can continue to poll the assigned Rx ring at all times 2833 * independantly. 2834 * 2835 * Note: In future, if no fanout is specified, we try to assign 2 Rx 2836 * rings for the primary Link with the primary MAC address + TCP going 2837 * to one ring and primary MAC address + UDP|SCTP going to other ring. 2838 * Any remaining traffic for primary MAC address can go to the default 2839 * Rx ring and get S/W classified. This way the respective SRSs don't 2840 * need to do proto fanout and don't need to have softrings at all and 2841 * can poll their respective Rx rings. 2842 * 2843 * As an optimization, when a new NIC or VNIC is created, we can get 2844 * only one Rx ring and make it a TCP specific Rx ring and use the 2845 * H/W default Rx ring for the rest (this Rx ring is never polled). 2846 * 2847 * For clients that don't have MAC address, but want to receive and 2848 * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 2849 * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 2850 * always give the default group and use software classification (i.e. 2851 * even if this is the only client in the default group, we will 2852 * leave group as shared). 2853 */ 2854 int 2855 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2856 uint32_t link_type) 2857 { 2858 mac_impl_t *mip = mcip->mci_mip; 2859 mac_group_t *rgroup = NULL; 2860 mac_group_t *tgroup = NULL; 2861 mac_group_t *default_rgroup; 2862 mac_group_t *default_tgroup; 2863 int err; 2864 uint8_t *mac_addr; 2865 mac_group_state_t next_state; 2866 mac_client_impl_t *group_only_mcip; 2867 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2868 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2869 boolean_t rxhw; 2870 boolean_t txhw; 2871 boolean_t use_default = B_FALSE; 2872 cpupart_t *cpupart; 2873 boolean_t no_unicast; 2874 boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 2875 mac_client_impl_t *reloc_pmcip = NULL; 2876 2877 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2878 2879 switch (link_type) { 2880 case SRST_FLOW: 2881 mac_srs_group_setup(mcip, flent, link_type); 2882 return (0); 2883 2884 case SRST_LINK: 2885 no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2886 mac_addr = flent->fe_flow_desc.fd_dst_mac; 2887 2888 /* Default RX group */ 2889 default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2890 2891 /* Default TX group */ 2892 default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 2893 2894 if (no_unicast) { 2895 rgroup = default_rgroup; 2896 tgroup = default_tgroup; 2897 goto grp_found; 2898 } 2899 rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 2900 (mrp->mrp_nrxrings > 0 || 2901 (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 2902 txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 2903 (mrp->mrp_ntxrings > 0 || 2904 (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 2905 2906 /* 2907 * By default we have given the primary all the rings 2908 * i.e. the default group. Let's see if the primary 2909 * needs to be relocated so that the addition of this 2910 * client doesn't impact the primary's performance, 2911 * i.e. if the primary is in the default group and 2912 * we add this client, the primary will lose polling. 2913 * We do this only for NICs supporting dynamic ring 2914 * grouping and only when this is the first client 2915 * after the primary (i.e. nactiveclients is 2) 2916 */ 2917 if (!isprimary && mip->mi_nactiveclients == 2 && 2918 (group_only_mcip = mac_primary_client_handle(mip)) != 2919 NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 2920 reloc_pmcip = mac_check_primary_relocation( 2921 group_only_mcip, rxhw); 2922 } 2923 /* 2924 * Check to see if we can get an exclusive group for 2925 * this mac address or if there already exists a 2926 * group that has this mac address (case of VLANs). 2927 * If no groups are available, use the default group. 2928 */ 2929 rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 2930 if (rgroup == NULL && rxhw) { 2931 err = ENOSPC; 2932 goto setup_failed; 2933 } else if (rgroup == NULL) { 2934 rgroup = default_rgroup; 2935 } 2936 /* 2937 * Check to see if we can get an exclusive group for 2938 * this mac client. If no groups are available, use 2939 * the default group. 2940 */ 2941 tgroup = mac_reserve_tx_group(mcip, B_FALSE); 2942 if (tgroup == NULL && txhw) { 2943 if (rgroup != NULL && rgroup != default_rgroup) 2944 mac_release_rx_group(mcip, rgroup); 2945 err = ENOSPC; 2946 goto setup_failed; 2947 } else if (tgroup == NULL) { 2948 tgroup = default_tgroup; 2949 } 2950 2951 /* 2952 * Some NICs don't support any Rx rings, so there may not 2953 * even be a default group. 2954 */ 2955 grp_found: 2956 if (rgroup != NULL) { 2957 if (rgroup != default_rgroup && 2958 MAC_GROUP_NO_CLIENT(rgroup) && 2959 (rxhw || mcip->mci_share != NULL)) { 2960 MAC_RX_GRP_RESERVED(mip); 2961 if (mip->mi_rx_group_type == 2962 MAC_GROUP_TYPE_DYNAMIC) { 2963 MAC_RX_RING_RESERVED(mip, 2964 rgroup->mrg_cur_count); 2965 } 2966 } 2967 flent->fe_rx_ring_group = rgroup; 2968 /* 2969 * Add the client to the group. This could cause 2970 * either this group to move to the shared state or 2971 * cause the default group to move to the shared state. 2972 * The actions on this group are done here, while the 2973 * actions on the default group are postponed to 2974 * the end of this function. 2975 */ 2976 mac_group_add_client(rgroup, mcip); 2977 next_state = mac_group_next_state(rgroup, 2978 &group_only_mcip, default_rgroup, B_TRUE); 2979 mac_set_group_state(rgroup, next_state); 2980 } 2981 2982 if (tgroup != NULL) { 2983 if (tgroup != default_tgroup && 2984 MAC_GROUP_NO_CLIENT(tgroup) && 2985 (txhw || mcip->mci_share != NULL)) { 2986 MAC_TX_GRP_RESERVED(mip); 2987 if (mip->mi_tx_group_type == 2988 MAC_GROUP_TYPE_DYNAMIC) { 2989 MAC_TX_RING_RESERVED(mip, 2990 tgroup->mrg_cur_count); 2991 } 2992 } 2993 flent->fe_tx_ring_group = tgroup; 2994 mac_group_add_client(tgroup, mcip); 2995 next_state = mac_group_next_state(tgroup, 2996 &group_only_mcip, default_tgroup, B_FALSE); 2997 tgroup->mrg_state = next_state; 2998 } 2999 /* 3000 * Setup the Rx and Tx SRSes. If we got a pristine group 3001 * exclusively above, mac_srs_group_setup would simply create 3002 * the required SRSes. If we ended up sharing a previously 3003 * reserved group, mac_srs_group_setup would also dismantle the 3004 * SRSes of the previously exclusive group 3005 */ 3006 mac_srs_group_setup(mcip, flent, link_type); 3007 3008 /* We are setting up minimal datapath only */ 3009 if (no_unicast) 3010 break; 3011 /* Program the S/W Classifer */ 3012 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3013 goto setup_failed; 3014 3015 /* Program the H/W Classifier */ 3016 if ((err = mac_add_macaddr(mip, rgroup, mac_addr, 3017 (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0)) != 0) 3018 goto setup_failed; 3019 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 3020 ASSERT(mcip->mci_unicast != NULL); 3021 /* Initialize the v6 local addr used by link protection */ 3022 mac_protect_update_v6_local_addr(mcip); 3023 break; 3024 3025 default: 3026 ASSERT(B_FALSE); 3027 break; 3028 } 3029 3030 /* 3031 * All broadcast and multicast traffic is received only on the default 3032 * group. If we have setup the datapath for a non-default group above 3033 * then move the default group to shared state to allow distribution of 3034 * incoming broadcast traffic to the other groups and dismantle the 3035 * SRSes over the default group. 3036 */ 3037 if (rgroup != NULL) { 3038 if (rgroup != default_rgroup) { 3039 if (default_rgroup->mrg_state == 3040 MAC_GROUP_STATE_RESERVED) { 3041 group_only_mcip = MAC_GROUP_ONLY_CLIENT( 3042 default_rgroup); 3043 ASSERT(group_only_mcip != NULL && 3044 mip->mi_nactiveclients > 1); 3045 3046 mac_set_group_state(default_rgroup, 3047 MAC_GROUP_STATE_SHARED); 3048 mac_rx_srs_group_setup(group_only_mcip, 3049 group_only_mcip->mci_flent, SRST_LINK); 3050 pool_lock(); 3051 cpupart = mac_pset_find(mrp, &use_default); 3052 mac_fanout_setup(group_only_mcip, 3053 group_only_mcip->mci_flent, 3054 MCIP_RESOURCE_PROPS(group_only_mcip), 3055 mac_rx_deliver, group_only_mcip, NULL, 3056 cpupart); 3057 mac_set_pool_effective(use_default, cpupart, 3058 mrp, emrp); 3059 pool_unlock(); 3060 } 3061 ASSERT(default_rgroup->mrg_state == 3062 MAC_GROUP_STATE_SHARED); 3063 } 3064 /* 3065 * If we get an exclusive group for a VLAN MAC client we 3066 * need to take the s/w path to make the additional check for 3067 * the vid. Disable polling and set it to s/w classification. 3068 * Similarly for clients that don't have a unicast address. 3069 */ 3070 if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 3071 (i_mac_flow_vid(flent) != VLAN_ID_NONE || no_unicast)) { 3072 mac_rx_switch_grp_to_sw(rgroup); 3073 } 3074 } 3075 mac_set_rings_effective(mcip); 3076 return (0); 3077 3078 setup_failed: 3079 /* Switch the primary back to default group */ 3080 if (reloc_pmcip != NULL) { 3081 (void) mac_rx_switch_group(reloc_pmcip, 3082 reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 3083 } 3084 mac_datapath_teardown(mcip, flent, link_type); 3085 return (err); 3086 } 3087 3088 void 3089 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3090 uint32_t link_type) 3091 { 3092 mac_impl_t *mip = mcip->mci_mip; 3093 mac_group_t *group = NULL; 3094 mac_client_impl_t *grp_only_mcip; 3095 flow_entry_t *group_only_flent; 3096 mac_group_t *default_group; 3097 boolean_t check_default_group = B_FALSE; 3098 mac_group_state_t next_state; 3099 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 3100 3101 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3102 3103 switch (link_type) { 3104 case SRST_FLOW: 3105 mac_rx_srs_group_teardown(flent, B_FALSE); 3106 mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3107 return; 3108 3109 case SRST_LINK: 3110 /* Stop sending packets */ 3111 mac_tx_client_block(mcip); 3112 3113 /* Stop the packets coming from the H/W */ 3114 if (mcip->mci_unicast != NULL) { 3115 int err; 3116 err = mac_remove_macaddr(mcip->mci_unicast); 3117 if (err != 0) { 3118 cmn_err(CE_WARN, "%s: failed to remove a MAC" 3119 " address because of error 0x%x", 3120 mip->mi_name, err); 3121 } 3122 mcip->mci_unicast = NULL; 3123 } 3124 3125 /* Stop the packets coming from the S/W classifier */ 3126 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3127 mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3128 3129 /* Now quiesce and destroy all SRS and soft rings */ 3130 mac_rx_srs_group_teardown(flent, B_FALSE); 3131 mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 3132 3133 ASSERT((mcip->mci_flent == flent) && 3134 (flent->fe_next == NULL)); 3135 3136 /* 3137 * Release our hold on the group as well. We need 3138 * to check if the shared group has only one client 3139 * left who can use it exclusively. Also, if we 3140 * were the last client, release the group. 3141 */ 3142 group = flent->fe_rx_ring_group; 3143 default_group = MAC_DEFAULT_RX_GROUP(mip); 3144 if (group != NULL) { 3145 mac_group_remove_client(group, mcip); 3146 next_state = mac_group_next_state(group, 3147 &grp_only_mcip, default_group, B_TRUE); 3148 if (next_state == MAC_GROUP_STATE_RESERVED) { 3149 /* 3150 * Only one client left on this RX group. 3151 */ 3152 ASSERT(grp_only_mcip != NULL); 3153 mac_set_group_state(group, 3154 MAC_GROUP_STATE_RESERVED); 3155 group_only_flent = grp_only_mcip->mci_flent; 3156 3157 /* 3158 * The only remaining client has exclusive 3159 * access on the group. Allow it to 3160 * dynamically poll the H/W rings etc. 3161 */ 3162 mac_rx_srs_group_setup(grp_only_mcip, 3163 group_only_flent, SRST_LINK); 3164 mac_fanout_setup(grp_only_mcip, 3165 group_only_flent, 3166 MCIP_RESOURCE_PROPS(grp_only_mcip), 3167 mac_rx_deliver, grp_only_mcip, NULL, NULL); 3168 mac_rx_group_unmark(group, MR_INCIPIENT); 3169 mac_set_rings_effective(grp_only_mcip); 3170 } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3171 /* 3172 * This is a non-default group being freed up. 3173 * We need to reevaluate the default group 3174 * to see if the primary client can get 3175 * exclusive access to the default group. 3176 */ 3177 ASSERT(group != MAC_DEFAULT_RX_GROUP(mip)); 3178 if (mrp->mrp_mask & MRP_RX_RINGS) { 3179 MAC_RX_GRP_RELEASED(mip); 3180 if (mip->mi_rx_group_type == 3181 MAC_GROUP_TYPE_DYNAMIC) { 3182 MAC_RX_RING_RELEASED(mip, 3183 group->mrg_cur_count); 3184 } 3185 } 3186 mac_release_rx_group(mcip, group); 3187 mac_set_group_state(group, 3188 MAC_GROUP_STATE_REGISTERED); 3189 check_default_group = B_TRUE; 3190 } else { 3191 ASSERT(next_state == MAC_GROUP_STATE_SHARED); 3192 mac_set_group_state(group, 3193 MAC_GROUP_STATE_SHARED); 3194 mac_rx_group_unmark(group, MR_CONDEMNED); 3195 } 3196 flent->fe_rx_ring_group = NULL; 3197 } 3198 /* 3199 * Remove the client from the TX group. Additionally, if 3200 * this a non-default group, then we also need to release 3201 * the group. 3202 */ 3203 group = flent->fe_tx_ring_group; 3204 default_group = MAC_DEFAULT_TX_GROUP(mip); 3205 if (group != NULL) { 3206 mac_group_remove_client(group, mcip); 3207 next_state = mac_group_next_state(group, 3208 &grp_only_mcip, default_group, B_FALSE); 3209 if (next_state == MAC_GROUP_STATE_REGISTERED) { 3210 if (group != default_group) { 3211 if (mrp->mrp_mask & MRP_TX_RINGS) { 3212 MAC_TX_GRP_RELEASED(mip); 3213 if (mip->mi_tx_group_type == 3214 MAC_GROUP_TYPE_DYNAMIC) { 3215 MAC_TX_RING_RELEASED( 3216 mip, group-> 3217 mrg_cur_count); 3218 } 3219 } 3220 mac_release_tx_group(mcip, group); 3221 /* 3222 * If the default group is reserved, 3223 * then we need to set the effective 3224 * rings as we would have given 3225 * back some rings when the group 3226 * was released 3227 */ 3228 if (mip->mi_tx_group_type == 3229 MAC_GROUP_TYPE_DYNAMIC && 3230 default_group->mrg_state == 3231 MAC_GROUP_STATE_RESERVED) { 3232 grp_only_mcip = 3233 MAC_GROUP_ONLY_CLIENT 3234 (default_group); 3235 mac_set_rings_effective( 3236 grp_only_mcip); 3237 } 3238 } else { 3239 mac_ring_t *ring; 3240 int cnt; 3241 int ringcnt; 3242 3243 /* 3244 * Stop all the rings except the 3245 * default ring. 3246 */ 3247 ringcnt = group->mrg_cur_count; 3248 ring = group->mrg_rings; 3249 for (cnt = 0; cnt < ringcnt; cnt++) { 3250 if (ring->mr_state == 3251 MR_INUSE && ring != 3252 (mac_ring_t *) 3253 mip->mi_default_tx_ring) { 3254 mac_stop_ring(ring); 3255 ring->mr_flag = 0; 3256 } 3257 ring = ring->mr_next; 3258 } 3259 } 3260 } else if (next_state == MAC_GROUP_STATE_RESERVED) { 3261 mac_set_rings_effective(grp_only_mcip); 3262 } 3263 flent->fe_tx_ring_group = NULL; 3264 group->mrg_state = next_state; 3265 } 3266 break; 3267 default: 3268 ASSERT(B_FALSE); 3269 break; 3270 } 3271 3272 /* 3273 * The mac client using the default group gets exclusive access to the 3274 * default group if and only if it is the sole client on the entire 3275 * mip. If so set the group state to reserved, and set up the SRSes 3276 * over the default group. 3277 */ 3278 if (check_default_group) { 3279 default_group = MAC_DEFAULT_RX_GROUP(mip); 3280 ASSERT(default_group->mrg_state == MAC_GROUP_STATE_SHARED); 3281 next_state = mac_group_next_state(default_group, 3282 &grp_only_mcip, default_group, B_TRUE); 3283 if (next_state == MAC_GROUP_STATE_RESERVED) { 3284 ASSERT(grp_only_mcip != NULL && 3285 mip->mi_nactiveclients == 1); 3286 mac_set_group_state(default_group, 3287 MAC_GROUP_STATE_RESERVED); 3288 mac_rx_srs_group_setup(grp_only_mcip, 3289 grp_only_mcip->mci_flent, SRST_LINK); 3290 mac_fanout_setup(grp_only_mcip, 3291 grp_only_mcip->mci_flent, 3292 MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 3293 grp_only_mcip, NULL, NULL); 3294 mac_rx_group_unmark(default_group, MR_INCIPIENT); 3295 mac_set_rings_effective(grp_only_mcip); 3296 } 3297 } 3298 3299 /* 3300 * If the primary is the only one left and the MAC supports 3301 * dynamic grouping, we need to see if the primary needs to 3302 * be moved to the default group so that it can use all the 3303 * H/W rings. 3304 */ 3305 if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 3306 mip->mi_nactiveclients == 1 && 3307 mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 3308 default_group = MAC_DEFAULT_RX_GROUP(mip); 3309 grp_only_mcip = mac_primary_client_handle(mip); 3310 if (grp_only_mcip == NULL) 3311 return; 3312 group_only_flent = grp_only_mcip->mci_flent; 3313 mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 3314 /* 3315 * If the primary has an explicit property set, leave it 3316 * alone. 3317 */ 3318 if (mrp->mrp_mask & MRP_RX_RINGS) 3319 return; 3320 /* 3321 * Switch the primary to the default group. 3322 */ 3323 (void) mac_rx_switch_group(grp_only_mcip, 3324 group_only_flent->fe_rx_ring_group, default_group); 3325 } 3326 } 3327 3328 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3329 3330 static void 3331 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3332 { 3333 if (mac_srs->srs_type & SRST_TX) { 3334 mac_srs_tx_t *tx; 3335 3336 ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 3337 ASSERT(mac_srs->srs_udp_soft_rings == NULL); 3338 ASSERT(mac_srs->srs_oth_soft_rings == NULL); 3339 ASSERT(mac_srs->srs_tx_soft_rings != NULL); 3340 kmem_free(mac_srs->srs_tx_soft_rings, 3341 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3342 mac_srs->srs_tx_soft_rings = NULL; 3343 tx = &mac_srs->srs_tx; 3344 if (tx->st_soft_rings != NULL) { 3345 kmem_free(tx->st_soft_rings, 3346 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3347 } 3348 } else { 3349 ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3350 ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3351 kmem_free(mac_srs->srs_tcp_soft_rings, 3352 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3353 mac_srs->srs_tcp_soft_rings = NULL; 3354 ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3355 kmem_free(mac_srs->srs_udp_soft_rings, 3356 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3357 mac_srs->srs_udp_soft_rings = NULL; 3358 ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3359 kmem_free(mac_srs->srs_oth_soft_rings, 3360 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3361 mac_srs->srs_oth_soft_rings = NULL; 3362 } 3363 } 3364 3365 /* 3366 * An RX SRS is attached to at most one mac_ring. 3367 * A TX SRS has no rings. 3368 */ 3369 static void 3370 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3371 { 3372 mac_client_impl_t *mcip; 3373 mac_ring_t *ring; 3374 flow_entry_t *flent; 3375 3376 ring = mac_srs->srs_ring; 3377 if (mac_srs->srs_type & SRST_TX) { 3378 ASSERT(ring == NULL); 3379 return; 3380 } 3381 3382 if (ring == NULL) 3383 return; 3384 3385 /* 3386 * Broadcast flows don't have a client impl association, but they 3387 * use only soft rings. 3388 */ 3389 flent = mac_srs->srs_flent; 3390 mcip = flent->fe_mcip; 3391 ASSERT(mcip != NULL); 3392 3393 ring->mr_classify_type = MAC_NO_CLASSIFIER; 3394 ring->mr_srs = NULL; 3395 } 3396 3397 /* 3398 * Physical unlink and free of the data structures happen below. This is 3399 * driven from mac_flow_destroy(), on the last refrele of a flow. 3400 * 3401 * Assumes Rx srs is 1-1 mapped with an ring. 3402 */ 3403 void 3404 mac_srs_free(mac_soft_ring_set_t *mac_srs) 3405 { 3406 ASSERT(mac_srs->srs_mcip == NULL || 3407 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3408 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3409 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3410 3411 mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE); 3412 mac_srs_ring_free(mac_srs); 3413 mac_srs_soft_rings_free(mac_srs); 3414 mac_srs_fanout_list_free(mac_srs); 3415 3416 mac_srs->srs_bw = NULL; 3417 mac_srs_stat_delete(mac_srs); 3418 kmem_cache_free(mac_srs_cache, mac_srs); 3419 } 3420 3421 static void 3422 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3423 { 3424 mac_soft_ring_t *softring; 3425 3426 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3427 3428 mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3429 if (s_ring_flag == S_RING_CONDEMNED) { 3430 while (mac_srs->srs_soft_ring_condemned_count != 3431 mac_srs->srs_soft_ring_count) 3432 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3433 } else { 3434 while (mac_srs->srs_soft_ring_quiesced_count != 3435 mac_srs->srs_soft_ring_count) 3436 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3437 } 3438 mutex_exit(&mac_srs->srs_lock); 3439 3440 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3441 softring = softring->s_ring_next) 3442 (void) untimeout(softring->s_ring_tid); 3443 3444 (void) untimeout(mac_srs->srs_tid); 3445 3446 mutex_enter(&mac_srs->srs_lock); 3447 } 3448 3449 /* 3450 * The block comment above mac_rx_classify_flow_state_change explains the 3451 * background. At this point upcalls from the driver (both hardware classified 3452 * and software classified) have been cut off. We now need to quiesce the 3453 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3454 * the master controller. The steps involved are described below in the function 3455 */ 3456 void 3457 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3458 { 3459 uint_t s_ring_flag; 3460 uint_t srs_poll_wait_flag; 3461 3462 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3463 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3464 3465 if (mac_srs->srs_state & SRS_CONDEMNED) { 3466 s_ring_flag = S_RING_CONDEMNED; 3467 srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3468 } else { 3469 s_ring_flag = S_RING_QUIESCE; 3470 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3471 } 3472 3473 /* 3474 * In the case of Rx SRS wait till the poll thread is done. 3475 */ 3476 if ((mac_srs->srs_type & SRST_TX) == 0 && 3477 mac_srs->srs_poll_thr != NULL) { 3478 while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3479 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3480 3481 /* 3482 * Turn off polling as part of the quiesce operation. 3483 */ 3484 MAC_SRS_POLLING_OFF(mac_srs); 3485 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3486 } 3487 3488 /* 3489 * Then signal the soft ring worker threads to quiesce or quit 3490 * as needed and then wait till that happens. 3491 */ 3492 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3493 3494 if (mac_srs->srs_state & SRS_CONDEMNED) 3495 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3496 else 3497 mac_srs->srs_state |= SRS_QUIESCE_DONE; 3498 cv_signal(&mac_srs->srs_quiesce_done_cv); 3499 } 3500 3501 /* 3502 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3503 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3504 * or poll thread. This function is internal to the quiescing logic and is 3505 * called internally from the SRS quiesce or flow quiesce or client quiesce 3506 * higher level functions. 3507 */ 3508 void 3509 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3510 { 3511 mac_ring_t *ring; 3512 3513 ring = mac_srs->srs_ring; 3514 ASSERT(ring == NULL || ring->mr_refcnt == 0); 3515 3516 if (srs_flag == SRS_CONDEMNED) { 3517 /* 3518 * The SRS is going away. We need to unbind the SRS and SR 3519 * threads before removing from the global SRS list. Otherwise 3520 * there is a small window where the cpu reconfig callbacks 3521 * may miss the SRS in the list walk and DR could fail since 3522 * there are still bound threads. 3523 */ 3524 mac_srs_threads_unbind(mac_srs); 3525 mac_srs_remove_glist(mac_srs); 3526 } 3527 /* 3528 * Wakeup the SRS worker and poll threads. 3529 */ 3530 mutex_enter(&mac_srs->srs_lock); 3531 mac_srs->srs_state |= srs_flag; 3532 cv_signal(&mac_srs->srs_async); 3533 cv_signal(&mac_srs->srs_cv); 3534 mutex_exit(&mac_srs->srs_lock); 3535 } 3536 3537 /* 3538 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3539 * from the driver are done, then the Rx SRS is quiesced and only then can 3540 * we signal the soft rings. Thus this function can't be called arbitrarily 3541 * without satisfying the prerequisites. On the Tx side, the threads from 3542 * top need to quiesced, then the Tx SRS and only then can we signal the 3543 * Tx soft rings. 3544 */ 3545 static void 3546 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3547 { 3548 mac_soft_ring_t *softring; 3549 3550 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3551 softring = softring->s_ring_next) 3552 mac_soft_ring_signal(softring, sr_flag); 3553 } 3554 3555 /* 3556 * The block comment above mac_rx_classify_flow_state_change explains the 3557 * background. At this point the SRS is quiesced and we need to restart the 3558 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3559 * the master controller. The steps involved are described below in the function 3560 */ 3561 void 3562 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3563 { 3564 boolean_t iam_rx_srs; 3565 mac_soft_ring_t *softring; 3566 3567 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3568 if ((mac_srs->srs_type & SRST_TX) != 0) { 3569 iam_rx_srs = B_FALSE; 3570 ASSERT((mac_srs->srs_state & 3571 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3572 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3573 } else { 3574 iam_rx_srs = B_TRUE; 3575 ASSERT((mac_srs->srs_state & 3576 (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3577 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3578 if (mac_srs->srs_poll_thr != NULL) { 3579 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3580 SRS_POLL_THR_QUIESCED); 3581 } 3582 } 3583 3584 /* 3585 * Signal any quiesced soft ring workers to restart and wait for the 3586 * soft ring down count to come down to zero. 3587 */ 3588 if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3589 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3590 softring = softring->s_ring_next) { 3591 if (!(softring->s_ring_state & S_RING_QUIESCE)) 3592 continue; 3593 mac_soft_ring_signal(softring, S_RING_RESTART); 3594 } 3595 while (mac_srs->srs_soft_ring_quiesced_count != 0) 3596 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3597 } 3598 3599 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3600 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3601 /* 3602 * Signal the poll thread and ask it to restart. Wait till it 3603 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3604 * cleared. 3605 */ 3606 mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3607 cv_signal(&mac_srs->srs_cv); 3608 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3609 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3610 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3611 } 3612 /* Wake up any waiter waiting for the restart to complete */ 3613 mac_srs->srs_state |= SRS_RESTART_DONE; 3614 cv_signal(&mac_srs->srs_quiesce_done_cv); 3615 } 3616 3617 static void 3618 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3619 { 3620 mutex_enter(&mac_srs->srs_lock); 3621 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3622 ASSERT(mac_srs->srs_worker_cpuid == -1); 3623 mutex_exit(&mac_srs->srs_lock); 3624 return; 3625 } 3626 3627 mac_srs->srs_worker_cpuid = -1; 3628 mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3629 thread_affinity_clear(mac_srs->srs_worker); 3630 mutex_exit(&mac_srs->srs_lock); 3631 } 3632 3633 static void 3634 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3635 { 3636 mutex_enter(&mac_srs->srs_lock); 3637 if (mac_srs->srs_poll_thr == NULL || 3638 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3639 ASSERT(mac_srs->srs_poll_cpuid == -1); 3640 mutex_exit(&mac_srs->srs_lock); 3641 return; 3642 } 3643 3644 mac_srs->srs_poll_cpuid = -1; 3645 mac_srs->srs_state &= ~SRS_POLL_BOUND; 3646 thread_affinity_clear(mac_srs->srs_poll_thr); 3647 mutex_exit(&mac_srs->srs_lock); 3648 } 3649 3650 static void 3651 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3652 { 3653 mac_soft_ring_t *soft_ring; 3654 3655 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3656 3657 mutex_enter(&cpu_lock); 3658 mac_srs_worker_unbind(mac_srs); 3659 if (!(mac_srs->srs_type & SRST_TX)) 3660 mac_srs_poll_unbind(mac_srs); 3661 3662 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3663 soft_ring = soft_ring->s_ring_next) { 3664 mac_soft_ring_unbind(soft_ring); 3665 } 3666 mutex_exit(&cpu_lock); 3667 } 3668 3669 /* 3670 * When a CPU is going away, unbind all MAC threads which are bound 3671 * to that CPU. The affinity of the thread to the CPU is saved to allow 3672 * the thread to be rebound to the CPU if it comes back online. 3673 */ 3674 static void 3675 mac_walk_srs_and_unbind(int cpuid) 3676 { 3677 mac_soft_ring_set_t *mac_srs; 3678 mac_soft_ring_t *soft_ring; 3679 3680 rw_enter(&mac_srs_g_lock, RW_READER); 3681 3682 if ((mac_srs = mac_srs_g_list) == NULL) 3683 goto done; 3684 3685 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3686 if (mac_srs->srs_worker_cpuid == cpuid) { 3687 mac_srs->srs_worker_cpuid_save = cpuid; 3688 mac_srs_worker_unbind(mac_srs); 3689 } 3690 3691 if (!(mac_srs->srs_type & SRST_TX)) { 3692 if (mac_srs->srs_poll_cpuid == cpuid) { 3693 mac_srs->srs_poll_cpuid_save = cpuid; 3694 mac_srs_poll_unbind(mac_srs); 3695 } 3696 } 3697 3698 /* Next tackle the soft rings associated with the srs */ 3699 mutex_enter(&mac_srs->srs_lock); 3700 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3701 soft_ring = soft_ring->s_ring_next) { 3702 if (soft_ring->s_ring_cpuid == cpuid) { 3703 soft_ring->s_ring_cpuid_save = cpuid; 3704 mac_soft_ring_unbind(soft_ring); 3705 } 3706 } 3707 mutex_exit(&mac_srs->srs_lock); 3708 } 3709 done: 3710 rw_exit(&mac_srs_g_lock); 3711 } 3712 3713 /* TX SETUP and TEARDOWN ROUTINES */ 3714 3715 /* 3716 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3717 * handle the case where the number of rings is one. I.e. there is 3718 * a ring pointed to by mac_srs->srs_tx_arg2. 3719 */ 3720 void 3721 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3722 { 3723 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3724 mac_soft_ring_t *soft_ring; 3725 int count = mac_srs->srs_tx_ring_count; 3726 uint32_t soft_ring_type = ST_RING_TX; 3727 uint_t ring_info; 3728 3729 ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3730 ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 3731 if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 3732 soft_ring_type |= ST_RING_WORKER_ONLY; 3733 soft_ring = mac_soft_ring_create(count, 0, 3734 soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3735 NULL, mcip, (mac_resource_handle_t)tx_ring); 3736 mac_srs->srs_tx_ring_count++; 3737 mac_srs_update_fanout_list(mac_srs); 3738 /* 3739 * put this soft ring in quiesce mode too so when we restart 3740 * all soft rings in the srs are in the same state. 3741 */ 3742 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3743 } 3744 3745 static void 3746 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3747 { 3748 int sringcnt; 3749 3750 mutex_enter(&mac_srs->srs_lock); 3751 sringcnt = mac_srs->srs_soft_ring_count; 3752 ASSERT(sringcnt > 0); 3753 mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3754 3755 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3756 while (mac_srs->srs_soft_ring_condemned_count != 1) 3757 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3758 3759 if (softring == mac_srs->srs_soft_ring_head) { 3760 mac_srs->srs_soft_ring_head = softring->s_ring_next; 3761 if (mac_srs->srs_soft_ring_head != NULL) { 3762 mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3763 } else { 3764 mac_srs->srs_soft_ring_tail = NULL; 3765 } 3766 } else { 3767 softring->s_ring_prev->s_ring_next = 3768 softring->s_ring_next; 3769 if (softring->s_ring_next != NULL) { 3770 softring->s_ring_next->s_ring_prev = 3771 softring->s_ring_prev; 3772 } else { 3773 mac_srs->srs_soft_ring_tail = 3774 softring->s_ring_prev; 3775 } 3776 } 3777 mac_srs->srs_soft_ring_count--; 3778 3779 mac_srs->srs_soft_ring_condemned_count--; 3780 mutex_exit(&mac_srs->srs_lock); 3781 3782 mac_soft_ring_free(softring); 3783 } 3784 3785 void 3786 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3787 { 3788 int i; 3789 mac_soft_ring_t *soft_ring, *remove_sring; 3790 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3791 3792 mutex_enter(&mac_srs->srs_lock); 3793 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 3794 soft_ring = mac_srs->srs_tx_soft_rings[i]; 3795 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3796 break; 3797 } 3798 mutex_exit(&mac_srs->srs_lock); 3799 ASSERT(i < mac_srs->srs_tx_ring_count); 3800 remove_sring = soft_ring; 3801 /* 3802 * In the case of aggr, the soft ring associated with a Tx ring 3803 * is also stored in st_soft_rings[] array. That entry should 3804 * be removed. 3805 */ 3806 if (mcip->mci_state_flags & MCIS_IS_AGGR) { 3807 mac_srs_tx_t *tx = &mac_srs->srs_tx; 3808 3809 ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 3810 tx->st_soft_rings[tx_ring->mr_index] = NULL; 3811 } 3812 mac_soft_ring_remove(mac_srs, remove_sring); 3813 mac_srs_update_fanout_list(mac_srs); 3814 } 3815 3816 /* 3817 * mac_tx_srs_setup(): 3818 * Used to setup Tx rings. If no free Tx ring is available, then default 3819 * Tx ring is used. 3820 */ 3821 void 3822 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3823 { 3824 mac_impl_t *mip = mcip->mci_mip; 3825 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 3826 int i; 3827 int tx_ring_count = 0; 3828 uint32_t soft_ring_type; 3829 mac_group_t *grp = NULL; 3830 mac_ring_t *ring; 3831 mac_srs_tx_t *tx = &tx_srs->srs_tx; 3832 boolean_t is_aggr; 3833 uint_t ring_info = 0; 3834 3835 is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR) != 0; 3836 grp = flent->fe_tx_ring_group; 3837 if (grp == NULL) { 3838 ring = (mac_ring_t *)mip->mi_default_tx_ring; 3839 goto no_group; 3840 } 3841 tx_ring_count = grp->mrg_cur_count; 3842 ring = grp->mrg_rings; 3843 /* 3844 * An attempt is made to reserve 'tx_ring_count' number 3845 * of Tx rings. If tx_ring_count is 0, default Tx ring 3846 * is used. If it is 1, an attempt is made to reserve one 3847 * Tx ring. In both the cases, the ring information is 3848 * stored in Tx SRS. If multiple Tx rings are specified, 3849 * then each Tx ring will have a Tx-side soft ring. All 3850 * these soft rings will be hang off Tx SRS. 3851 */ 3852 switch (grp->mrg_state) { 3853 case MAC_GROUP_STATE_SHARED: 3854 case MAC_GROUP_STATE_RESERVED: 3855 if (tx_ring_count <= 1 && !is_aggr) { 3856 no_group: 3857 if (ring != NULL && 3858 ring->mr_state != MR_INUSE) { 3859 (void) mac_start_ring(ring); 3860 ring_info = mac_hwring_getinfo( 3861 (mac_ring_handle_t)ring); 3862 } 3863 tx->st_arg2 = (void *)ring; 3864 mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 3865 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3866 tx->st_mode = SRS_TX_BW; 3867 } else if (mac_tx_serialize || 3868 (ring_info & MAC_RING_TX_SERIALIZE)) { 3869 tx->st_mode = SRS_TX_SERIALIZE; 3870 } else { 3871 tx->st_mode = SRS_TX_DEFAULT; 3872 } 3873 break; 3874 } 3875 soft_ring_type = ST_RING_TX; 3876 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3877 tx->st_mode = is_aggr ? 3878 SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3879 } else { 3880 tx->st_mode = is_aggr ? SRS_TX_AGGR : 3881 SRS_TX_FANOUT; 3882 } 3883 for (i = 0; i < tx_ring_count; i++) { 3884 ASSERT(ring != NULL); 3885 switch (ring->mr_state) { 3886 case MR_INUSE: 3887 case MR_FREE: 3888 ASSERT(ring->mr_srs == NULL); 3889 3890 if (ring->mr_state != MR_INUSE) 3891 (void) mac_start_ring(ring); 3892 ring_info = mac_hwring_getinfo( 3893 (mac_ring_handle_t)ring); 3894 if (mac_tx_serialize || (ring_info & 3895 MAC_RING_TX_SERIALIZE)) { 3896 soft_ring_type |= 3897 ST_RING_WORKER_ONLY; 3898 } 3899 (void) mac_soft_ring_create(i, 0, 3900 soft_ring_type, maxclsyspri, 3901 mcip, tx_srs, -1, NULL, mcip, 3902 (mac_resource_handle_t)ring); 3903 break; 3904 default: 3905 cmn_err(CE_PANIC, 3906 "srs_setup: mcip = %p " 3907 "trying to add UNKNOWN ring = %p\n", 3908 (void *)mcip, (void *)ring); 3909 break; 3910 } 3911 ring = ring->mr_next; 3912 } 3913 mac_srs_update_fanout_list(tx_srs); 3914 break; 3915 default: 3916 ASSERT(B_FALSE); 3917 break; 3918 } 3919 tx->st_func = mac_tx_get_func(tx->st_mode); 3920 if (is_aggr) { 3921 VERIFY(i_mac_capab_get((mac_handle_t)mip, 3922 MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 3923 } 3924 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 3925 int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 3926 } 3927 3928 /* 3929 * Update the fanout of a client if its recorded link speed doesn't match 3930 * its current link speed. 3931 */ 3932 void 3933 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 3934 { 3935 uint64_t link_speed; 3936 mac_resource_props_t *mcip_mrp; 3937 flow_entry_t *flent = mcip->mci_flent; 3938 mac_soft_ring_set_t *rx_srs; 3939 mac_cpus_t *srs_cpu; 3940 int soft_ring_count, maxcpus; 3941 3942 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 3943 3944 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 3945 MAC_STAT_IFSPEED); 3946 3947 if ((link_speed != 0) && 3948 (link_speed != mcip->mci_flent->fe_nic_speed)) { 3949 mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 3950 /* 3951 * Before calling mac_fanout_setup(), check to see if 3952 * the SRSes already have the right number of soft 3953 * rings. mac_fanout_setup() is a heavy duty operation 3954 * where new cpu bindings are done for SRS and soft 3955 * ring threads and interrupts re-targeted. 3956 */ 3957 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 3958 soft_ring_count = mac_compute_soft_ring_count(flent, 3959 flent->fe_rx_srs_cnt - 1, maxcpus); 3960 /* 3961 * If soft_ring_count returned by 3962 * mac_compute_soft_ring_count() is 0, bump it 3963 * up by 1 because we always have atleast one 3964 * TCP, UDP, and OTH soft ring associated with 3965 * an SRS. 3966 */ 3967 soft_ring_count = (soft_ring_count == 0) ? 3968 1 : soft_ring_count; 3969 rx_srs = flent->fe_rx_srs[0]; 3970 srs_cpu = &rx_srs->srs_cpu; 3971 if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 3972 mac_fanout_setup(mcip, flent, mcip_mrp, 3973 mac_rx_deliver, mcip, NULL, cpupart); 3974 } 3975 } 3976 } 3977 3978 /* 3979 * Walk through the list of mac clients for the MAC. 3980 * For each active mac client, recompute the number of soft rings 3981 * associated with every client, only if current speed is different 3982 * from the speed that was previously used for soft ring computation. 3983 * If the cable is disconnected whlie the NIC is started, we would get 3984 * notification with speed set to 0. We do not recompute in that case. 3985 */ 3986 void 3987 mac_fanout_recompute(mac_impl_t *mip) 3988 { 3989 mac_client_impl_t *mcip; 3990 cpupart_t *cpupart; 3991 boolean_t use_default; 3992 mac_resource_props_t *mrp, *emrp; 3993 3994 i_mac_perim_enter(mip); 3995 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 3996 mip->mi_linkstate != LINK_STATE_UP) { 3997 i_mac_perim_exit(mip); 3998 return; 3999 } 4000 4001 for (mcip = mip->mi_clients_list; mcip != NULL; 4002 mcip = mcip->mci_client_next) { 4003 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 4004 !MCIP_DATAPATH_SETUP(mcip)) 4005 continue; 4006 mrp = MCIP_RESOURCE_PROPS(mcip); 4007 emrp = MCIP_EFFECTIVE_PROPS(mcip); 4008 use_default = B_FALSE; 4009 pool_lock(); 4010 cpupart = mac_pset_find(mrp, &use_default); 4011 mac_fanout_recompute_client(mcip, cpupart); 4012 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 4013 pool_unlock(); 4014 } 4015 i_mac_perim_exit(mip); 4016 } 4017 4018 /* 4019 * Given a MAC, change the polling state for all its MAC clients. 'enable' is 4020 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 4021 * default. 4022 */ 4023 void 4024 mac_poll_state_change(mac_handle_t mh, boolean_t enable) 4025 { 4026 mac_impl_t *mip = (mac_impl_t *)mh; 4027 mac_client_impl_t *mcip; 4028 4029 i_mac_perim_enter(mip); 4030 if (enable) 4031 mip->mi_state_flags &= ~MIS_POLL_DISABLE; 4032 else 4033 mip->mi_state_flags |= MIS_POLL_DISABLE; 4034 for (mcip = mip->mi_clients_list; mcip != NULL; 4035 mcip = mcip->mci_client_next) 4036 mac_client_update_classifier(mcip, B_TRUE); 4037 i_mac_perim_exit(mip); 4038 } 4039