1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet6.h" 35 #include "opt_pcbgroup.h" 36 37 #ifndef PCBGROUP 38 #error "options RSS depends on options PCBGROUP" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/mbuf.h> 43 #include <sys/socket.h> 44 #include <sys/priv.h> 45 #include <sys/kernel.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/sbuf.h> 49 50 #include <net/if.h> 51 #include <net/if_var.h> 52 #include <net/netisr.h> 53 #include <net/rss_config.h> 54 #include <net/toeplitz.h> 55 56 /*- 57 * Operating system parts of receiver-side scaling (RSS), which allows 58 * network cards to direct flows to particular receive queues based on hashes 59 * of header tuples. This implementation aligns RSS buckets with connection 60 * groups at the TCP/IP layer, so each bucket is associated with exactly one 61 * group. As a result, the group lookup structures (and lock) should have an 62 * effective affinity with exactly one CPU. 63 * 64 * Network device drivers needing to configure RSS will query this framework 65 * for parameters, such as the current RSS key, hashing policies, number of 66 * bits, and indirection table mapping hashes to buckets and CPUs. They may 67 * provide their own supplementary information, such as queue<->CPU bindings. 68 * It is the responsibility of the network device driver to inject packets 69 * into the stack on as close to the right CPU as possible, if playing by RSS 70 * rules. 71 * 72 * TODO: 73 * 74 * - Synchronization for rss_key and other future-configurable parameters. 75 * - Event handler drivers can register to pick up RSS configuration changes. 76 * - Should we allow rss_basecpu to be configured? 77 * - Randomize key on boot. 78 * - IPv6 support. 79 * - Statistics on how often there's a misalignment between hardware 80 * placement and pcbgroup expectations. 81 */ 82 83 SYSCTL_DECL(_net_inet); 84 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 85 "Receive-side steering"); 86 87 /* 88 * Toeplitz is the only required hash function in the RSS spec, so use it by 89 * default. 90 */ 91 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 92 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 93 "RSS hash algorithm"); 94 95 /* 96 * Size of the indirection table; at most 128 entries per the RSS spec. We 97 * size it to at least 2 times the number of CPUs by default to allow useful 98 * rebalancing. If not set explicitly with a loader tunable, we tune based 99 * on the number of CPUs present. 100 * 101 * XXXRW: buckets might be better to use for the tunable than bits. 102 */ 103 static u_int rss_bits; 104 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 105 "RSS bits"); 106 107 static u_int rss_mask; 108 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 109 "RSS mask"); 110 111 static const u_int rss_maxbits = RSS_MAXBITS; 112 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 113 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 114 115 /* 116 * RSS's own count of the number of CPUs it could be using for processing. 117 * Bounded to 64 by RSS constants. 118 */ 119 static u_int rss_ncpus; 120 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 121 "Number of CPUs available to RSS"); 122 123 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 124 static const u_int rss_maxcpus = RSS_MAXCPUS; 125 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 126 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 127 128 /* 129 * Variable exists just for reporting rss_bits in a user-friendly way. 130 */ 131 static u_int rss_buckets; 132 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 133 "RSS buckets"); 134 135 /* 136 * Base CPU number; devices will add this to all CPU numbers returned by the 137 * RSS indirection table. Currently unmodifable in FreeBSD. 138 */ 139 static const u_int rss_basecpu; 140 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 141 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 142 143 /* 144 * Print verbose debugging messages. 145 * 0 - disable 146 * non-zero - enable 147 */ 148 int rss_debug = 0; 149 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0, 150 "RSS debug level"); 151 152 /* 153 * RSS secret key, intended to prevent attacks on load-balancing. Its 154 * effectiveness may be limited by algorithm choice and available entropy 155 * during the boot. 156 * 157 * XXXRW: And that we don't randomize it yet! 158 * 159 * This is the default Microsoft RSS specification key which is also 160 * the Chelsio T5 firmware default key. 161 */ 162 static uint8_t rss_key[RSS_KEYSIZE] = { 163 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 164 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 165 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 166 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 167 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 168 }; 169 170 /* 171 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 172 * Drivers may supplement this table with a separate CPU<->queue table when 173 * programming devices. 174 */ 175 struct rss_table_entry { 176 uint8_t rte_cpu; /* CPU affinity of bucket. */ 177 }; 178 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 179 180 static void 181 rss_init(__unused void *arg) 182 { 183 u_int i; 184 u_int cpuid; 185 186 /* 187 * Validate tunables, coerce to sensible values. 188 */ 189 switch (rss_hashalgo) { 190 case RSS_HASH_TOEPLITZ: 191 case RSS_HASH_NAIVE: 192 break; 193 194 default: 195 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", 196 rss_hashalgo, RSS_HASH_TOEPLITZ); 197 rss_hashalgo = RSS_HASH_TOEPLITZ; 198 } 199 200 /* 201 * Count available CPUs. 202 * 203 * XXXRW: Note incorrect assumptions regarding contiguity of this set 204 * elsewhere. 205 */ 206 rss_ncpus = 0; 207 for (i = 0; i <= mp_maxid; i++) { 208 if (CPU_ABSENT(i)) 209 continue; 210 rss_ncpus++; 211 } 212 if (rss_ncpus > RSS_MAXCPUS) 213 rss_ncpus = RSS_MAXCPUS; 214 215 /* 216 * Tune RSS table entries to be no less than 2x the number of CPUs 217 * -- unless we're running uniprocessor, in which case there's not 218 * much point in having buckets to rearrange for load-balancing! 219 */ 220 if (rss_ncpus > 1) { 221 if (rss_bits == 0) 222 rss_bits = fls(rss_ncpus - 1) + 1; 223 224 /* 225 * Microsoft limits RSS table entries to 128, so apply that 226 * limit to both auto-detected CPU counts and user-configured 227 * ones. 228 */ 229 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 230 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", 231 rss_bits, RSS_MAXBITS); 232 rss_bits = RSS_MAXBITS; 233 } 234 235 /* 236 * Figure out how many buckets to use; warn if less than the 237 * number of configured CPUs, although this is not a fatal 238 * problem. 239 */ 240 rss_buckets = (1 << rss_bits); 241 if (rss_buckets < rss_ncpus) 242 RSS_DEBUG("WARNING: rss_buckets (%u) less than " 243 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); 244 rss_mask = rss_buckets - 1; 245 } else { 246 rss_bits = 0; 247 rss_buckets = 1; 248 rss_mask = 0; 249 } 250 251 /* 252 * Set up initial CPU assignments: round-robin by default. 253 */ 254 cpuid = CPU_FIRST(); 255 for (i = 0; i < rss_buckets; i++) { 256 rss_table[i].rte_cpu = cpuid; 257 cpuid = CPU_NEXT(cpuid); 258 } 259 260 /* 261 * Randomize rrs_key. 262 * 263 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 264 * loop to check for "bad" RSS keys. 265 */ 266 } 267 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 268 269 static uint32_t 270 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 271 const uint8_t *data) 272 { 273 uint32_t v; 274 u_int i; 275 276 v = 0; 277 for (i = 0; i < keylen; i++) 278 v += key[i]; 279 for (i = 0; i < datalen; i++) 280 v += data[i]; 281 return (v); 282 } 283 284 uint32_t 285 rss_hash(u_int datalen, const uint8_t *data) 286 { 287 288 switch (rss_hashalgo) { 289 case RSS_HASH_TOEPLITZ: 290 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 291 data)); 292 293 case RSS_HASH_NAIVE: 294 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 295 data)); 296 297 default: 298 panic("%s: unsupported/unknown hashalgo %d", __func__, 299 rss_hashalgo); 300 } 301 } 302 303 /* 304 * Query the number of RSS bits in use. 305 */ 306 u_int 307 rss_getbits(void) 308 { 309 310 return (rss_bits); 311 } 312 313 /* 314 * Query the RSS bucket associated with an RSS hash. 315 */ 316 u_int 317 rss_getbucket(u_int hash) 318 { 319 320 return (hash & rss_mask); 321 } 322 323 /* 324 * Query the RSS layer bucket associated with the given 325 * entry in the RSS hash space. 326 * 327 * The RSS indirection table is 0 .. rss_buckets-1, 328 * covering the low 'rss_bits' of the total 128 slot 329 * RSS indirection table. So just mask off rss_bits and 330 * return that. 331 * 332 * NIC drivers can then iterate over the 128 slot RSS 333 * indirection table and fetch which RSS bucket to 334 * map it to. This will typically be a CPU queue 335 */ 336 u_int 337 rss_get_indirection_to_bucket(u_int index) 338 { 339 340 return (index & rss_mask); 341 } 342 343 /* 344 * Query the RSS CPU associated with an RSS bucket. 345 */ 346 u_int 347 rss_getcpu(u_int bucket) 348 { 349 350 return (rss_table[bucket].rte_cpu); 351 } 352 353 /* 354 * netisr CPU affinity lookup given just the hash and hashtype. 355 */ 356 u_int 357 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 358 { 359 360 switch (hash_type) { 361 case M_HASHTYPE_RSS_IPV4: 362 case M_HASHTYPE_RSS_TCP_IPV4: 363 case M_HASHTYPE_RSS_UDP_IPV4: 364 case M_HASHTYPE_RSS_IPV6: 365 case M_HASHTYPE_RSS_TCP_IPV6: 366 case M_HASHTYPE_RSS_UDP_IPV6: 367 return (rss_getcpu(rss_getbucket(hash_val))); 368 default: 369 return (NETISR_CPUID_NONE); 370 } 371 } 372 373 /* 374 * Query the RSS bucket associated with the given hash value and 375 * type. 376 */ 377 int 378 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 379 { 380 381 switch (hash_type) { 382 case M_HASHTYPE_RSS_IPV4: 383 case M_HASHTYPE_RSS_TCP_IPV4: 384 case M_HASHTYPE_RSS_UDP_IPV4: 385 case M_HASHTYPE_RSS_IPV6: 386 case M_HASHTYPE_RSS_TCP_IPV6: 387 case M_HASHTYPE_RSS_UDP_IPV6: 388 *bucket_id = rss_getbucket(hash_val); 389 return (0); 390 default: 391 return (-1); 392 } 393 } 394 395 /* 396 * netisr CPU affinity lookup routine for use by protocols. 397 */ 398 struct mbuf * 399 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 400 { 401 402 M_ASSERTPKTHDR(m); 403 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 404 return (m); 405 } 406 407 int 408 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 409 { 410 411 M_ASSERTPKTHDR(m); 412 413 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 414 bucket_id)); 415 } 416 417 /* 418 * Query the RSS hash algorithm. 419 */ 420 u_int 421 rss_gethashalgo(void) 422 { 423 424 return (rss_hashalgo); 425 } 426 427 /* 428 * Query the current RSS key; likely to be used by device drivers when 429 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 430 * 431 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 432 */ 433 void 434 rss_getkey(uint8_t *key) 435 { 436 437 bcopy(rss_key, key, sizeof(rss_key)); 438 } 439 440 /* 441 * Query the number of buckets; this may be used by both network device 442 * drivers, which will need to populate hardware shadows of the software 443 * indirection table, and the network stack itself (such as when deciding how 444 * many connection groups to allocate). 445 */ 446 u_int 447 rss_getnumbuckets(void) 448 { 449 450 return (rss_buckets); 451 } 452 453 /* 454 * Query the number of CPUs in use by RSS; may be useful to device drivers 455 * trying to figure out how to map a larger number of CPUs into a smaller 456 * number of receive queues. 457 */ 458 u_int 459 rss_getnumcpus(void) 460 { 461 462 return (rss_ncpus); 463 } 464 465 /* 466 * Return the supported RSS hash configuration. 467 * 468 * NICs should query this to determine what to configure in their redirection 469 * matching table. 470 */ 471 inline u_int 472 rss_gethashconfig(void) 473 { 474 475 /* Return 4-tuple for TCP; 2-tuple for others */ 476 /* 477 * UDP may fragment more often than TCP and thus we'll end up with 478 * NICs returning 2-tuple fragments. 479 * udp_init() and udplite_init() both currently initialise things 480 * as 2-tuple. 481 * So for now disable UDP 4-tuple hashing until all of the other 482 * pieces are in place. 483 */ 484 return ( 485 RSS_HASHTYPE_RSS_IPV4 486 | RSS_HASHTYPE_RSS_TCP_IPV4 487 | RSS_HASHTYPE_RSS_IPV6 488 | RSS_HASHTYPE_RSS_TCP_IPV6 489 | RSS_HASHTYPE_RSS_IPV6_EX 490 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 491 #if 0 492 | RSS_HASHTYPE_RSS_UDP_IPV4 493 | RSS_HASHTYPE_RSS_UDP_IPV6 494 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 495 #endif 496 ); 497 } 498 499 /* 500 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 501 * it appearing in debugging output unnecessarily. 502 */ 503 static int 504 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 505 { 506 uint8_t temp_rss_key[RSS_KEYSIZE]; 507 int error; 508 509 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 510 if (error) 511 return (error); 512 513 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 514 error = sysctl_handle_opaque(oidp, temp_rss_key, 515 sizeof(temp_rss_key), req); 516 if (error) 517 return (error); 518 if (req->newptr != NULL) { 519 /* XXXRW: Not yet. */ 520 return (EINVAL); 521 } 522 return (0); 523 } 524 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 525 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 526 "", "RSS keying material"); 527 528 static int 529 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 530 { 531 struct sbuf *sb; 532 int error; 533 int i; 534 535 error = 0; 536 error = sysctl_wire_old_buffer(req, 0); 537 if (error != 0) 538 return (error); 539 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 540 if (sb == NULL) 541 return (ENOMEM); 542 for (i = 0; i < rss_buckets; i++) { 543 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 544 i, 545 rss_getcpu(i)); 546 } 547 error = sbuf_finish(sb); 548 sbuf_delete(sb); 549 550 return (error); 551 } 552 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 553 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 554 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 555