1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet6.h" 35 #include "opt_pcbgroup.h" 36 37 #ifndef PCBGROUP 38 #error "options RSS depends on options PCBGROUP" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/mbuf.h> 43 #include <sys/socket.h> 44 #include <sys/priv.h> 45 #include <sys/kernel.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/sbuf.h> 49 50 #include <net/if.h> 51 #include <net/if_var.h> 52 #include <net/netisr.h> 53 54 #include <netinet/in.h> 55 #include <netinet/in_pcb.h> 56 #include <netinet/in_rss.h> 57 #include <netinet/in_var.h> 58 #include <netinet/toeplitz.h> 59 60 /*- 61 * Operating system parts of receiver-side scaling (RSS), which allows 62 * network cards to direct flows to particular receive queues based on hashes 63 * of header tuples. This implementation aligns RSS buckets with connection 64 * groups at the TCP/IP layer, so each bucket is associated with exactly one 65 * group. As a result, the group lookup structures (and lock) should have an 66 * effective affinity with exactly one CPU. 67 * 68 * Network device drivers needing to configure RSS will query this framework 69 * for parameters, such as the current RSS key, hashing policies, number of 70 * bits, and indirection table mapping hashes to buckets and CPUs. They may 71 * provide their own supplementary information, such as queue<->CPU bindings. 72 * It is the responsibility of the network device driver to inject packets 73 * into the stack on as close to the right CPU as possible, if playing by RSS 74 * rules. 75 * 76 * TODO: 77 * 78 * - Synchronization for rss_key and other future-configurable parameters. 79 * - Event handler drivers can register to pick up RSS configuration changes. 80 * - Should we allow rss_basecpu to be configured? 81 * - Randomize key on boot. 82 * - IPv6 support. 83 * - Statistics on how often there's a misalignment between hardware 84 * placement and pcbgroup expectations. 85 */ 86 87 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); 88 89 /* 90 * Toeplitz is the only required hash function in the RSS spec, so use it by 91 * default. 92 */ 93 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 94 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 95 "RSS hash algorithm"); 96 97 /* 98 * Size of the indirection table; at most 128 entries per the RSS spec. We 99 * size it to at least 2 times the number of CPUs by default to allow useful 100 * rebalancing. If not set explicitly with a loader tunable, we tune based 101 * on the number of CPUs present. 102 * 103 * XXXRW: buckets might be better to use for the tunable than bits. 104 */ 105 static u_int rss_bits; 106 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 107 "RSS bits"); 108 109 static u_int rss_mask; 110 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 111 "RSS mask"); 112 113 static const u_int rss_maxbits = RSS_MAXBITS; 114 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 115 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 116 117 /* 118 * RSS's own count of the number of CPUs it could be using for processing. 119 * Bounded to 64 by RSS constants. 120 */ 121 static u_int rss_ncpus; 122 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 123 "Number of CPUs available to RSS"); 124 125 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 126 static const u_int rss_maxcpus = RSS_MAXCPUS; 127 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 128 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 129 130 /* 131 * Variable exists just for reporting rss_bits in a user-friendly way. 132 */ 133 static u_int rss_buckets; 134 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 135 "RSS buckets"); 136 137 /* 138 * Base CPU number; devices will add this to all CPU numbers returned by the 139 * RSS indirection table. Currently unmodifable in FreeBSD. 140 */ 141 static const u_int rss_basecpu; 142 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 143 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 144 145 /* 146 * RSS secret key, intended to prevent attacks on load-balancing. Its 147 * effectiveness may be limited by algorithm choice and available entropy 148 * during the boot. 149 * 150 * XXXRW: And that we don't randomize it yet! 151 * 152 * This is the default Microsoft RSS specification key which is also 153 * the Chelsio T5 firmware default key. 154 */ 155 static uint8_t rss_key[RSS_KEYSIZE] = { 156 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 157 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 158 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 159 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 160 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 161 }; 162 163 /* 164 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 165 * Drivers may supplement this table with a seperate CPU<->queue table when 166 * programming devices. 167 */ 168 struct rss_table_entry { 169 uint8_t rte_cpu; /* CPU affinity of bucket. */ 170 }; 171 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 172 173 static void 174 rss_init(__unused void *arg) 175 { 176 u_int i; 177 u_int cpuid; 178 179 /* 180 * Validate tunables, coerce to sensible values. 181 */ 182 switch (rss_hashalgo) { 183 case RSS_HASH_TOEPLITZ: 184 case RSS_HASH_NAIVE: 185 break; 186 187 default: 188 printf("%s: invalid RSS hashalgo %u, coercing to %u", 189 __func__, rss_hashalgo, RSS_HASH_TOEPLITZ); 190 rss_hashalgo = RSS_HASH_TOEPLITZ; 191 } 192 193 /* 194 * Count available CPUs. 195 * 196 * XXXRW: Note incorrect assumptions regarding contiguity of this set 197 * elsewhere. 198 */ 199 rss_ncpus = 0; 200 for (i = 0; i <= mp_maxid; i++) { 201 if (CPU_ABSENT(i)) 202 continue; 203 rss_ncpus++; 204 } 205 if (rss_ncpus > RSS_MAXCPUS) 206 rss_ncpus = RSS_MAXCPUS; 207 208 /* 209 * Tune RSS table entries to be no less than 2x the number of CPUs 210 * -- unless we're running uniprocessor, in which case there's not 211 * much point in having buckets to rearrange for load-balancing! 212 */ 213 if (rss_ncpus > 1) { 214 if (rss_bits == 0) 215 rss_bits = fls(rss_ncpus - 1) + 1; 216 217 /* 218 * Microsoft limits RSS table entries to 128, so apply that 219 * limit to both auto-detected CPU counts and user-configured 220 * ones. 221 */ 222 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 223 printf("%s: RSS bits %u not valid, coercing to %u", 224 __func__, rss_bits, RSS_MAXBITS); 225 rss_bits = RSS_MAXBITS; 226 } 227 228 /* 229 * Figure out how many buckets to use; warn if less than the 230 * number of configured CPUs, although this is not a fatal 231 * problem. 232 */ 233 rss_buckets = (1 << rss_bits); 234 if (rss_buckets < rss_ncpus) 235 printf("%s: WARNING: rss_buckets (%u) less than " 236 "rss_ncpus (%u)\n", __func__, rss_buckets, 237 rss_ncpus); 238 rss_mask = rss_buckets - 1; 239 } else { 240 rss_bits = 0; 241 rss_buckets = 1; 242 rss_mask = 0; 243 } 244 245 /* 246 * Set up initial CPU assignments: round-robin by default. 247 */ 248 cpuid = CPU_FIRST(); 249 for (i = 0; i < rss_buckets; i++) { 250 rss_table[i].rte_cpu = cpuid; 251 cpuid = CPU_NEXT(cpuid); 252 } 253 254 /* 255 * Randomize rrs_key. 256 * 257 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 258 * loop to check for "bad" RSS keys. 259 */ 260 } 261 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 262 263 static uint32_t 264 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 265 const uint8_t *data) 266 { 267 uint32_t v; 268 u_int i; 269 270 v = 0; 271 for (i = 0; i < keylen; i++) 272 v += key[i]; 273 for (i = 0; i < datalen; i++) 274 v += data[i]; 275 return (v); 276 } 277 278 static uint32_t 279 rss_hash(u_int datalen, const uint8_t *data) 280 { 281 282 switch (rss_hashalgo) { 283 case RSS_HASH_TOEPLITZ: 284 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 285 data)); 286 287 case RSS_HASH_NAIVE: 288 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 289 data)); 290 291 default: 292 panic("%s: unsupported/unknown hashalgo %d", __func__, 293 rss_hashalgo); 294 } 295 } 296 297 /* 298 * Hash an IPv4 2-tuple. 299 */ 300 uint32_t 301 rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst) 302 { 303 uint8_t data[sizeof(src) + sizeof(dst)]; 304 u_int datalen; 305 306 datalen = 0; 307 bcopy(&src, &data[datalen], sizeof(src)); 308 datalen += sizeof(src); 309 bcopy(&dst, &data[datalen], sizeof(dst)); 310 datalen += sizeof(dst); 311 return (rss_hash(datalen, data)); 312 } 313 314 /* 315 * Hash an IPv4 4-tuple. 316 */ 317 uint32_t 318 rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst, 319 u_short dstport) 320 { 321 uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) + 322 sizeof(dstport)]; 323 u_int datalen; 324 325 datalen = 0; 326 bcopy(&src, &data[datalen], sizeof(src)); 327 datalen += sizeof(src); 328 bcopy(&dst, &data[datalen], sizeof(dst)); 329 datalen += sizeof(dst); 330 bcopy(&srcport, &data[datalen], sizeof(srcport)); 331 datalen += sizeof(srcport); 332 bcopy(&dstport, &data[datalen], sizeof(dstport)); 333 datalen += sizeof(dstport); 334 return (rss_hash(datalen, data)); 335 } 336 337 #ifdef INET6 338 /* 339 * Hash an IPv6 2-tuple. 340 */ 341 uint32_t 342 rss_hash_ip6_2tuple(struct in6_addr src, struct in6_addr dst) 343 { 344 uint8_t data[sizeof(src) + sizeof(dst)]; 345 u_int datalen; 346 347 datalen = 0; 348 bcopy(&src, &data[datalen], sizeof(src)); 349 datalen += sizeof(src); 350 bcopy(&dst, &data[datalen], sizeof(dst)); 351 datalen += sizeof(dst); 352 return (rss_hash(datalen, data)); 353 } 354 355 /* 356 * Hash an IPv6 4-tuple. 357 */ 358 uint32_t 359 rss_hash_ip6_4tuple(struct in6_addr src, u_short srcport, 360 struct in6_addr dst, u_short dstport) 361 { 362 uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) + 363 sizeof(dstport)]; 364 u_int datalen; 365 366 datalen = 0; 367 bcopy(&src, &data[datalen], sizeof(src)); 368 datalen += sizeof(src); 369 bcopy(&dst, &data[datalen], sizeof(dst)); 370 datalen += sizeof(dst); 371 bcopy(&srcport, &data[datalen], sizeof(srcport)); 372 datalen += sizeof(srcport); 373 bcopy(&dstport, &data[datalen], sizeof(dstport)); 374 datalen += sizeof(dstport); 375 return (rss_hash(datalen, data)); 376 } 377 #endif /* INET6 */ 378 379 /* 380 * Query the number of RSS bits in use. 381 */ 382 u_int 383 rss_getbits(void) 384 { 385 386 return (rss_bits); 387 } 388 389 /* 390 * Query the RSS bucket associated with an RSS hash. 391 */ 392 u_int 393 rss_getbucket(u_int hash) 394 { 395 396 return (hash & rss_mask); 397 } 398 399 /* 400 * Query the RSS layer bucket associated with the given 401 * entry in the RSS hash space. 402 * 403 * The RSS indirection table is 0 .. rss_buckets-1, 404 * covering the low 'rss_bits' of the total 128 slot 405 * RSS indirection table. So just mask off rss_bits and 406 * return that. 407 * 408 * NIC drivers can then iterate over the 128 slot RSS 409 * indirection table and fetch which RSS bucket to 410 * map it to. This will typically be a CPU queue 411 */ 412 u_int 413 rss_get_indirection_to_bucket(u_int index) 414 { 415 416 return (index & rss_mask); 417 } 418 419 /* 420 * Query the RSS CPU associated with an RSS bucket. 421 */ 422 u_int 423 rss_getcpu(u_int bucket) 424 { 425 426 return (rss_table[bucket].rte_cpu); 427 } 428 429 /* 430 * netisr CPU affinity lookup given just the hash and hashtype. 431 */ 432 u_int 433 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 434 { 435 436 switch (hash_type) { 437 case M_HASHTYPE_RSS_IPV4: 438 case M_HASHTYPE_RSS_TCP_IPV4: 439 case M_HASHTYPE_RSS_UDP_IPV4: 440 case M_HASHTYPE_RSS_IPV6: 441 case M_HASHTYPE_RSS_TCP_IPV6: 442 case M_HASHTYPE_RSS_UDP_IPV6: 443 return (rss_getcpu(rss_getbucket(hash_val))); 444 default: 445 return (NETISR_CPUID_NONE); 446 } 447 } 448 449 /* 450 * Query the RSS bucket associated with the given hash value and 451 * type. 452 */ 453 int 454 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 455 { 456 457 switch (hash_type) { 458 case M_HASHTYPE_RSS_IPV4: 459 case M_HASHTYPE_RSS_TCP_IPV4: 460 case M_HASHTYPE_RSS_UDP_IPV4: 461 case M_HASHTYPE_RSS_IPV6: 462 case M_HASHTYPE_RSS_TCP_IPV6: 463 case M_HASHTYPE_RSS_UDP_IPV6: 464 *bucket_id = rss_getbucket(hash_val); 465 return (0); 466 default: 467 return (-1); 468 } 469 } 470 471 /* 472 * netisr CPU affinity lookup routine for use by protocols. 473 */ 474 struct mbuf * 475 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 476 { 477 478 M_ASSERTPKTHDR(m); 479 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 480 return (m); 481 } 482 483 int 484 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 485 { 486 487 M_ASSERTPKTHDR(m); 488 489 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 490 bucket_id)); 491 } 492 493 /* 494 * Query the RSS hash algorithm. 495 */ 496 u_int 497 rss_gethashalgo(void) 498 { 499 500 return (rss_hashalgo); 501 } 502 503 /* 504 * Query the current RSS key; likely to be used by device drivers when 505 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 506 * 507 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 508 */ 509 void 510 rss_getkey(uint8_t *key) 511 { 512 513 bcopy(rss_key, key, sizeof(rss_key)); 514 } 515 516 /* 517 * Query the number of buckets; this may be used by both network device 518 * drivers, which will need to populate hardware shadows of the software 519 * indirection table, and the network stack itself (such as when deciding how 520 * many connection groups to allocate). 521 */ 522 u_int 523 rss_getnumbuckets(void) 524 { 525 526 return (rss_buckets); 527 } 528 529 /* 530 * Query the number of CPUs in use by RSS; may be useful to device drivers 531 * trying to figure out how to map a larger number of CPUs into a smaller 532 * number of receive queues. 533 */ 534 u_int 535 rss_getnumcpus(void) 536 { 537 538 return (rss_ncpus); 539 } 540 541 /* 542 * Return the supported RSS hash configuration. 543 * 544 * NICs should query this to determine what to configure in their redirection 545 * matching table. 546 */ 547 u_int 548 rss_gethashconfig(void) 549 { 550 /* Return 4-tuple for TCP; 2-tuple for others */ 551 /* 552 * UDP may fragment more often than TCP and thus we'll end up with 553 * NICs returning 2-tuple fragments. 554 * udp_init() and udplite_init() both currently initialise things 555 * as 2-tuple. 556 * So for now disable UDP 4-tuple hashing until all of the other 557 * pieces are in place. 558 */ 559 return ( 560 RSS_HASHTYPE_RSS_IPV4 561 | RSS_HASHTYPE_RSS_TCP_IPV4 562 | RSS_HASHTYPE_RSS_IPV6 563 | RSS_HASHTYPE_RSS_TCP_IPV6 564 | RSS_HASHTYPE_RSS_IPV6_EX 565 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 566 #if 0 567 | RSS_HASHTYPE_RSS_UDP_IPV4 568 | RSS_HASHTYPE_RSS_UDP_IPV4_EX 569 | RSS_HASHTYPE_RSS_UDP_IPV6 570 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 571 #endif 572 ); 573 } 574 575 /* 576 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 577 * it appearing in debugging output unnecessarily. 578 */ 579 static int 580 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 581 { 582 uint8_t temp_rss_key[RSS_KEYSIZE]; 583 int error; 584 585 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 586 if (error) 587 return (error); 588 589 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 590 error = sysctl_handle_opaque(oidp, temp_rss_key, 591 sizeof(temp_rss_key), req); 592 if (error) 593 return (error); 594 if (req->newptr != NULL) { 595 /* XXXRW: Not yet. */ 596 return (EINVAL); 597 } 598 return (0); 599 } 600 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 601 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 602 "", "RSS keying material"); 603 604 static int 605 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 606 { 607 struct sbuf *sb; 608 int error; 609 int i; 610 611 error = 0; 612 error = sysctl_wire_old_buffer(req, 0); 613 if (error != 0) 614 return (error); 615 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 616 if (sb == NULL) 617 return (ENOMEM); 618 for (i = 0; i < rss_buckets; i++) { 619 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 620 i, 621 rss_getcpu(i)); 622 } 623 error = sbuf_finish(sb); 624 sbuf_delete(sb); 625 626 return (error); 627 } 628 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 629 CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 630 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 631