1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet6.h" 35 #include "opt_pcbgroup.h" 36 37 #ifndef PCBGROUP 38 #error "options RSS depends on options PCBGROUP" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/mbuf.h> 43 #include <sys/socket.h> 44 #include <sys/priv.h> 45 #include <sys/kernel.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/sbuf.h> 49 50 #include <net/if.h> 51 #include <net/if_var.h> 52 #include <net/netisr.h> 53 54 #include <netinet/in.h> 55 #include <netinet/in_pcb.h> 56 #include <netinet/in_rss.h> 57 #include <netinet/in_var.h> 58 #include <netinet/toeplitz.h> 59 60 /*- 61 * Operating system parts of receiver-side scaling (RSS), which allows 62 * network cards to direct flows to particular receive queues based on hashes 63 * of header tuples. This implementation aligns RSS buckets with connection 64 * groups at the TCP/IP layer, so each bucket is associated with exactly one 65 * group. As a result, the group lookup structures (and lock) should have an 66 * effective affinity with exactly one CPU. 67 * 68 * Network device drivers needing to configure RSS will query this framework 69 * for parameters, such as the current RSS key, hashing policies, number of 70 * bits, and indirection table mapping hashes to buckets and CPUs. They may 71 * provide their own supplementary information, such as queue<->CPU bindings. 72 * It is the responsibility of the network device driver to inject packets 73 * into the stack on as close to the right CPU as possible, if playing by RSS 74 * rules. 75 * 76 * TODO: 77 * 78 * - Synchronization for rss_key and other future-configurable parameters. 79 * - Event handler drivers can register to pick up RSS configuration changes. 80 * - Should we allow rss_basecpu to be configured? 81 * - Randomize key on boot. 82 * - IPv6 support. 83 * - Statistics on how often there's a misalignment between hardware 84 * placement and pcbgroup expectations. 85 */ 86 87 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); 88 89 /* 90 * Toeplitz is the only required hash function in the RSS spec, so use it by 91 * default. 92 */ 93 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 94 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RD, &rss_hashalgo, 0, 95 "RSS hash algorithm"); 96 TUNABLE_INT("net.inet.rss.hashalgo", &rss_hashalgo); 97 98 /* 99 * Size of the indirection table; at most 128 entries per the RSS spec. We 100 * size it to at least 2 times the number of CPUs by default to allow useful 101 * rebalancing. If not set explicitly with a loader tunable, we tune based 102 * on the number of CPUs present. 103 * 104 * XXXRW: buckets might be better to use for the tunable than bits. 105 */ 106 static u_int rss_bits; 107 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RD, &rss_bits, 0, 108 "RSS bits"); 109 TUNABLE_INT("net.inet.rss.bits", &rss_bits); 110 111 static u_int rss_mask; 112 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 113 "RSS mask"); 114 115 static const u_int rss_maxbits = RSS_MAXBITS; 116 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 117 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 118 119 /* 120 * RSS's own count of the number of CPUs it could be using for processing. 121 * Bounded to 64 by RSS constants. 122 */ 123 static u_int rss_ncpus; 124 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 125 "Number of CPUs available to RSS"); 126 127 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 128 static const u_int rss_maxcpus = RSS_MAXCPUS; 129 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 130 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 131 132 /* 133 * Variable exists just for reporting rss_bits in a user-friendly way. 134 */ 135 static u_int rss_buckets; 136 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 137 "RSS buckets"); 138 139 /* 140 * Base CPU number; devices will add this to all CPU numbers returned by the 141 * RSS indirection table. Currently unmodifable in FreeBSD. 142 */ 143 static const u_int rss_basecpu; 144 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 145 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 146 147 /* 148 * RSS secret key, intended to prevent attacks on load-balancing. Its 149 * effectiveness may be limited by algorithm choice and available entropy 150 * during the boot. 151 * 152 * XXXRW: And that we don't randomize it yet! 153 * 154 * XXXRW: This default is actually the default key from Chelsio T3 cards, as 155 * it offers reasonable distribution, unlike all-0 keys which always 156 * generate a hash of 0 (upsettingly). 157 */ 158 static uint8_t rss_key[RSS_KEYSIZE] = { 159 0x43, 0xa3, 0x8f, 0xb0, 0x41, 0x67, 0x25, 0x3d, 160 0x25, 0x5b, 0x0e, 0xc2, 0x6d, 0x5a, 0x56, 0xda, 161 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 162 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 163 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 164 }; 165 166 /* 167 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 168 * Drivers may supplement this table with a seperate CPU<->queue table when 169 * programming devices. 170 */ 171 struct rss_table_entry { 172 uint8_t rte_cpu; /* CPU affinity of bucket. */ 173 }; 174 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 175 176 static void 177 rss_init(__unused void *arg) 178 { 179 u_int i; 180 u_int cpuid; 181 182 /* 183 * Validate tunables, coerce to sensible values. 184 */ 185 switch (rss_hashalgo) { 186 case RSS_HASH_TOEPLITZ: 187 case RSS_HASH_NAIVE: 188 break; 189 190 default: 191 printf("%s: invalid RSS hashalgo %u, coercing to %u", 192 __func__, rss_hashalgo, RSS_HASH_TOEPLITZ); 193 rss_hashalgo = RSS_HASH_TOEPLITZ; 194 } 195 196 /* 197 * Count available CPUs. 198 * 199 * XXXRW: Note incorrect assumptions regarding contiguity of this set 200 * elsewhere. 201 */ 202 rss_ncpus = 0; 203 for (i = 0; i <= mp_maxid; i++) { 204 if (CPU_ABSENT(i)) 205 continue; 206 rss_ncpus++; 207 } 208 if (rss_ncpus > RSS_MAXCPUS) 209 rss_ncpus = RSS_MAXCPUS; 210 211 /* 212 * Tune RSS table entries to be no less than 2x the number of CPUs 213 * -- unless we're running uniprocessor, in which case there's not 214 * much point in having buckets to rearrange for load-balancing! 215 */ 216 if (rss_ncpus > 1) { 217 if (rss_bits == 0) 218 rss_bits = fls(rss_ncpus - 1) + 1; 219 220 /* 221 * Microsoft limits RSS table entries to 128, so apply that 222 * limit to both auto-detected CPU counts and user-configured 223 * ones. 224 */ 225 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 226 printf("%s: RSS bits %u not valid, coercing to %u", 227 __func__, rss_bits, RSS_MAXBITS); 228 rss_bits = RSS_MAXBITS; 229 } 230 231 /* 232 * Figure out how many buckets to use; warn if less than the 233 * number of configured CPUs, although this is not a fatal 234 * problem. 235 */ 236 rss_buckets = (1 << rss_bits); 237 if (rss_buckets < rss_ncpus) 238 printf("%s: WARNING: rss_buckets (%u) less than " 239 "rss_ncpus (%u)\n", __func__, rss_buckets, 240 rss_ncpus); 241 rss_mask = rss_buckets - 1; 242 } else { 243 rss_bits = 0; 244 rss_buckets = 1; 245 rss_mask = 0; 246 } 247 248 /* 249 * Set up initial CPU assignments: round-robin by default. 250 */ 251 cpuid = CPU_FIRST(); 252 for (i = 0; i < rss_buckets; i++) { 253 rss_table[i].rte_cpu = cpuid; 254 cpuid = CPU_NEXT(cpuid); 255 } 256 257 /* 258 * Randomize rrs_key. 259 * 260 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 261 * loop to check for "bad" RSS keys. 262 */ 263 } 264 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 265 266 static uint32_t 267 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 268 const uint8_t *data) 269 { 270 uint32_t v; 271 u_int i; 272 273 v = 0; 274 for (i = 0; i < keylen; i++) 275 v += key[i]; 276 for (i = 0; i < datalen; i++) 277 v += data[i]; 278 return (v); 279 } 280 281 static uint32_t 282 rss_hash(u_int datalen, const uint8_t *data) 283 { 284 285 switch (rss_hashalgo) { 286 case RSS_HASH_TOEPLITZ: 287 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 288 data)); 289 290 case RSS_HASH_NAIVE: 291 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 292 data)); 293 294 default: 295 panic("%s: unsupported/unknown hashalgo %d", __func__, 296 rss_hashalgo); 297 } 298 } 299 300 /* 301 * Hash an IPv4 2-tuple. 302 */ 303 uint32_t 304 rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst) 305 { 306 uint8_t data[sizeof(src) + sizeof(dst)]; 307 u_int datalen; 308 309 datalen = 0; 310 bcopy(&src, &data[datalen], sizeof(src)); 311 datalen += sizeof(src); 312 bcopy(&dst, &data[datalen], sizeof(dst)); 313 datalen += sizeof(dst); 314 return (rss_hash(datalen, data)); 315 } 316 317 /* 318 * Hash an IPv4 4-tuple. 319 */ 320 uint32_t 321 rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst, 322 u_short dstport) 323 { 324 uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) + 325 sizeof(dstport)]; 326 u_int datalen; 327 328 datalen = 0; 329 bcopy(&src, &data[datalen], sizeof(src)); 330 datalen += sizeof(src); 331 bcopy(&dst, &data[datalen], sizeof(dst)); 332 datalen += sizeof(dst); 333 bcopy(&srcport, &data[datalen], sizeof(srcport)); 334 datalen += sizeof(srcport); 335 bcopy(&dstport, &data[datalen], sizeof(dstport)); 336 datalen += sizeof(dstport); 337 return (rss_hash(datalen, data)); 338 } 339 340 #ifdef INET6 341 /* 342 * Hash an IPv6 2-tuple. 343 */ 344 uint32_t 345 rss_hash_ip6_2tuple(struct in6_addr src, struct in6_addr dst) 346 { 347 uint8_t data[sizeof(src) + sizeof(dst)]; 348 u_int datalen; 349 350 datalen = 0; 351 bcopy(&src, &data[datalen], sizeof(src)); 352 datalen += sizeof(src); 353 bcopy(&dst, &data[datalen], sizeof(dst)); 354 datalen += sizeof(dst); 355 return (rss_hash(datalen, data)); 356 } 357 358 /* 359 * Hash an IPv6 4-tuple. 360 */ 361 uint32_t 362 rss_hash_ip6_4tuple(struct in6_addr src, u_short srcport, 363 struct in6_addr dst, u_short dstport) 364 { 365 uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) + 366 sizeof(dstport)]; 367 u_int datalen; 368 369 datalen = 0; 370 bcopy(&src, &data[datalen], sizeof(src)); 371 datalen += sizeof(src); 372 bcopy(&dst, &data[datalen], sizeof(dst)); 373 datalen += sizeof(dst); 374 bcopy(&srcport, &data[datalen], sizeof(srcport)); 375 datalen += sizeof(srcport); 376 bcopy(&dstport, &data[datalen], sizeof(dstport)); 377 datalen += sizeof(dstport); 378 return (rss_hash(datalen, data)); 379 } 380 #endif /* INET6 */ 381 382 /* 383 * Query the number of RSS bits in use. 384 */ 385 u_int 386 rss_getbits(void) 387 { 388 389 return (rss_bits); 390 } 391 392 /* 393 * Query the RSS bucket associated with an RSS hash. 394 */ 395 u_int 396 rss_getbucket(u_int hash) 397 { 398 399 return (hash & rss_mask); 400 } 401 402 /* 403 * Query the RSS layer bucket associated with the given 404 * entry in the RSS hash space. 405 * 406 * The RSS indirection table is 0 .. rss_buckets-1, 407 * covering the low 'rss_bits' of the total 128 slot 408 * RSS indirection table. So just mask off rss_bits and 409 * return that. 410 * 411 * NIC drivers can then iterate over the 128 slot RSS 412 * indirection table and fetch which RSS bucket to 413 * map it to. This will typically be a CPU queue 414 */ 415 u_int 416 rss_get_indirection_to_bucket(u_int index) 417 { 418 419 return (index & rss_mask); 420 } 421 422 /* 423 * Query the RSS CPU associated with an RSS bucket. 424 */ 425 u_int 426 rss_getcpu(u_int bucket) 427 { 428 429 return (rss_table[bucket].rte_cpu); 430 } 431 432 /* 433 * netisr CPU affinity lookup given just the hash and hashtype. 434 */ 435 u_int 436 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 437 { 438 439 switch (hash_type) { 440 case M_HASHTYPE_RSS_IPV4: 441 case M_HASHTYPE_RSS_TCP_IPV4: 442 return (rss_getcpu(rss_getbucket(hash_val))); 443 default: 444 return (NETISR_CPUID_NONE); 445 } 446 } 447 448 /* 449 * Query the RSS bucket associated with the given hash value and 450 * type. 451 */ 452 int 453 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 454 { 455 456 switch (hash_type) { 457 case M_HASHTYPE_RSS_IPV4: 458 case M_HASHTYPE_RSS_TCP_IPV4: 459 *bucket_id = rss_getbucket(hash_val); 460 return (0); 461 default: 462 return (-1); 463 } 464 } 465 466 /* 467 * netisr CPU affinity lookup routine for use by protocols. 468 */ 469 struct mbuf * 470 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 471 { 472 473 M_ASSERTPKTHDR(m); 474 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 475 return (m); 476 } 477 478 int 479 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 480 { 481 482 M_ASSERTPKTHDR(m); 483 484 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 485 bucket_id)); 486 } 487 488 /* 489 * Query the RSS hash algorithm. 490 */ 491 u_int 492 rss_gethashalgo(void) 493 { 494 495 return (rss_hashalgo); 496 } 497 498 /* 499 * Query the current RSS key; likely to be used by device drivers when 500 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 501 * 502 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 503 */ 504 void 505 rss_getkey(uint8_t *key) 506 { 507 508 bcopy(rss_key, key, sizeof(rss_key)); 509 } 510 511 /* 512 * Query the number of buckets; this may be used by both network device 513 * drivers, which will need to populate hardware shadows of the software 514 * indirection table, and the network stack itself (such as when deciding how 515 * many connection groups to allocate). 516 */ 517 u_int 518 rss_getnumbuckets(void) 519 { 520 521 return (rss_buckets); 522 } 523 524 /* 525 * Query the number of CPUs in use by RSS; may be useful to device drivers 526 * trying to figure out how to map a larger number of CPUs into a smaller 527 * number of receive queues. 528 */ 529 u_int 530 rss_getnumcpus(void) 531 { 532 533 return (rss_ncpus); 534 } 535 536 /* 537 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 538 * it appearing in debugging output unnecessarily. 539 */ 540 static int 541 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 542 { 543 uint8_t temp_rss_key[RSS_KEYSIZE]; 544 int error; 545 546 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 547 if (error) 548 return (error); 549 550 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 551 error = sysctl_handle_opaque(oidp, temp_rss_key, 552 sizeof(temp_rss_key), req); 553 if (error) 554 return (error); 555 if (req->newptr != NULL) { 556 /* XXXRW: Not yet. */ 557 return (EINVAL); 558 } 559 return (0); 560 } 561 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 562 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 563 "", "RSS keying material"); 564 565 static int 566 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 567 { 568 struct sbuf *sb; 569 int error; 570 int i; 571 572 error = 0; 573 error = sysctl_wire_old_buffer(req, 0); 574 if (error != 0) 575 return (error); 576 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 577 if (sb == NULL) 578 return (ENOMEM); 579 for (i = 0; i < rss_buckets; i++) { 580 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 581 i, 582 rss_getcpu(i)); 583 } 584 error = sbuf_finish(sb); 585 sbuf_delete(sb); 586 587 return (error); 588 } 589 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 590 CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 591 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 592