1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet6.h" 35 #include "opt_pcbgroup.h" 36 37 #ifndef PCBGROUP 38 #error "options RSS depends on options PCBGROUP" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/mbuf.h> 43 #include <sys/socket.h> 44 #include <sys/priv.h> 45 #include <sys/kernel.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/sbuf.h> 49 50 #include <net/if.h> 51 #include <net/if_var.h> 52 #include <net/netisr.h> 53 #include <net/rss_config.h> 54 #include <net/toeplitz.h> 55 56 /*- 57 * Operating system parts of receiver-side scaling (RSS), which allows 58 * network cards to direct flows to particular receive queues based on hashes 59 * of header tuples. This implementation aligns RSS buckets with connection 60 * groups at the TCP/IP layer, so each bucket is associated with exactly one 61 * group. As a result, the group lookup structures (and lock) should have an 62 * effective affinity with exactly one CPU. 63 * 64 * Network device drivers needing to configure RSS will query this framework 65 * for parameters, such as the current RSS key, hashing policies, number of 66 * bits, and indirection table mapping hashes to buckets and CPUs. They may 67 * provide their own supplementary information, such as queue<->CPU bindings. 68 * It is the responsibility of the network device driver to inject packets 69 * into the stack on as close to the right CPU as possible, if playing by RSS 70 * rules. 71 * 72 * TODO: 73 * 74 * - Synchronization for rss_key and other future-configurable parameters. 75 * - Event handler drivers can register to pick up RSS configuration changes. 76 * - Should we allow rss_basecpu to be configured? 77 * - Randomize key on boot. 78 * - IPv6 support. 79 * - Statistics on how often there's a misalignment between hardware 80 * placement and pcbgroup expectations. 81 */ 82 83 SYSCTL_DECL(_net_inet); 84 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); 85 86 /* 87 * Toeplitz is the only required hash function in the RSS spec, so use it by 88 * default. 89 */ 90 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 91 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 92 "RSS hash algorithm"); 93 94 /* 95 * Size of the indirection table; at most 128 entries per the RSS spec. We 96 * size it to at least 2 times the number of CPUs by default to allow useful 97 * rebalancing. If not set explicitly with a loader tunable, we tune based 98 * on the number of CPUs present. 99 * 100 * XXXRW: buckets might be better to use for the tunable than bits. 101 */ 102 static u_int rss_bits; 103 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 104 "RSS bits"); 105 106 static u_int rss_mask; 107 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 108 "RSS mask"); 109 110 static const u_int rss_maxbits = RSS_MAXBITS; 111 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 112 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 113 114 /* 115 * RSS's own count of the number of CPUs it could be using for processing. 116 * Bounded to 64 by RSS constants. 117 */ 118 static u_int rss_ncpus; 119 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 120 "Number of CPUs available to RSS"); 121 122 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 123 static const u_int rss_maxcpus = RSS_MAXCPUS; 124 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 125 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 126 127 /* 128 * Variable exists just for reporting rss_bits in a user-friendly way. 129 */ 130 static u_int rss_buckets; 131 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 132 "RSS buckets"); 133 134 /* 135 * Base CPU number; devices will add this to all CPU numbers returned by the 136 * RSS indirection table. Currently unmodifable in FreeBSD. 137 */ 138 static const u_int rss_basecpu; 139 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 140 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 141 142 /* 143 * Print verbose debugging messages. 144 * 0 - disable 145 * non-zero - enable 146 */ 147 int rss_debug = 0; 148 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0, 149 "RSS debug level"); 150 151 /* 152 * RSS secret key, intended to prevent attacks on load-balancing. Its 153 * effectiveness may be limited by algorithm choice and available entropy 154 * during the boot. 155 * 156 * XXXRW: And that we don't randomize it yet! 157 * 158 * This is the default Microsoft RSS specification key which is also 159 * the Chelsio T5 firmware default key. 160 */ 161 static uint8_t rss_key[RSS_KEYSIZE] = { 162 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 163 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 164 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 165 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 166 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 167 }; 168 169 /* 170 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 171 * Drivers may supplement this table with a separate CPU<->queue table when 172 * programming devices. 173 */ 174 struct rss_table_entry { 175 uint8_t rte_cpu; /* CPU affinity of bucket. */ 176 }; 177 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 178 179 static void 180 rss_init(__unused void *arg) 181 { 182 u_int i; 183 u_int cpuid; 184 185 /* 186 * Validate tunables, coerce to sensible values. 187 */ 188 switch (rss_hashalgo) { 189 case RSS_HASH_TOEPLITZ: 190 case RSS_HASH_NAIVE: 191 break; 192 193 default: 194 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", 195 rss_hashalgo, RSS_HASH_TOEPLITZ); 196 rss_hashalgo = RSS_HASH_TOEPLITZ; 197 } 198 199 /* 200 * Count available CPUs. 201 * 202 * XXXRW: Note incorrect assumptions regarding contiguity of this set 203 * elsewhere. 204 */ 205 rss_ncpus = 0; 206 for (i = 0; i <= mp_maxid; i++) { 207 if (CPU_ABSENT(i)) 208 continue; 209 rss_ncpus++; 210 } 211 if (rss_ncpus > RSS_MAXCPUS) 212 rss_ncpus = RSS_MAXCPUS; 213 214 /* 215 * Tune RSS table entries to be no less than 2x the number of CPUs 216 * -- unless we're running uniprocessor, in which case there's not 217 * much point in having buckets to rearrange for load-balancing! 218 */ 219 if (rss_ncpus > 1) { 220 if (rss_bits == 0) 221 rss_bits = fls(rss_ncpus - 1) + 1; 222 223 /* 224 * Microsoft limits RSS table entries to 128, so apply that 225 * limit to both auto-detected CPU counts and user-configured 226 * ones. 227 */ 228 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 229 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", 230 rss_bits, RSS_MAXBITS); 231 rss_bits = RSS_MAXBITS; 232 } 233 234 /* 235 * Figure out how many buckets to use; warn if less than the 236 * number of configured CPUs, although this is not a fatal 237 * problem. 238 */ 239 rss_buckets = (1 << rss_bits); 240 if (rss_buckets < rss_ncpus) 241 RSS_DEBUG("WARNING: rss_buckets (%u) less than " 242 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); 243 rss_mask = rss_buckets - 1; 244 } else { 245 rss_bits = 0; 246 rss_buckets = 1; 247 rss_mask = 0; 248 } 249 250 /* 251 * Set up initial CPU assignments: round-robin by default. 252 */ 253 cpuid = CPU_FIRST(); 254 for (i = 0; i < rss_buckets; i++) { 255 rss_table[i].rte_cpu = cpuid; 256 cpuid = CPU_NEXT(cpuid); 257 } 258 259 /* 260 * Randomize rrs_key. 261 * 262 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 263 * loop to check for "bad" RSS keys. 264 */ 265 } 266 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 267 268 static uint32_t 269 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 270 const uint8_t *data) 271 { 272 uint32_t v; 273 u_int i; 274 275 v = 0; 276 for (i = 0; i < keylen; i++) 277 v += key[i]; 278 for (i = 0; i < datalen; i++) 279 v += data[i]; 280 return (v); 281 } 282 283 uint32_t 284 rss_hash(u_int datalen, const uint8_t *data) 285 { 286 287 switch (rss_hashalgo) { 288 case RSS_HASH_TOEPLITZ: 289 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 290 data)); 291 292 case RSS_HASH_NAIVE: 293 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 294 data)); 295 296 default: 297 panic("%s: unsupported/unknown hashalgo %d", __func__, 298 rss_hashalgo); 299 } 300 } 301 302 /* 303 * Query the number of RSS bits in use. 304 */ 305 u_int 306 rss_getbits(void) 307 { 308 309 return (rss_bits); 310 } 311 312 /* 313 * Query the RSS bucket associated with an RSS hash. 314 */ 315 u_int 316 rss_getbucket(u_int hash) 317 { 318 319 return (hash & rss_mask); 320 } 321 322 /* 323 * Query the RSS layer bucket associated with the given 324 * entry in the RSS hash space. 325 * 326 * The RSS indirection table is 0 .. rss_buckets-1, 327 * covering the low 'rss_bits' of the total 128 slot 328 * RSS indirection table. So just mask off rss_bits and 329 * return that. 330 * 331 * NIC drivers can then iterate over the 128 slot RSS 332 * indirection table and fetch which RSS bucket to 333 * map it to. This will typically be a CPU queue 334 */ 335 u_int 336 rss_get_indirection_to_bucket(u_int index) 337 { 338 339 return (index & rss_mask); 340 } 341 342 /* 343 * Query the RSS CPU associated with an RSS bucket. 344 */ 345 u_int 346 rss_getcpu(u_int bucket) 347 { 348 349 return (rss_table[bucket].rte_cpu); 350 } 351 352 /* 353 * netisr CPU affinity lookup given just the hash and hashtype. 354 */ 355 u_int 356 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 357 { 358 359 switch (hash_type) { 360 case M_HASHTYPE_RSS_IPV4: 361 case M_HASHTYPE_RSS_TCP_IPV4: 362 case M_HASHTYPE_RSS_UDP_IPV4: 363 case M_HASHTYPE_RSS_IPV6: 364 case M_HASHTYPE_RSS_TCP_IPV6: 365 case M_HASHTYPE_RSS_UDP_IPV6: 366 return (rss_getcpu(rss_getbucket(hash_val))); 367 default: 368 return (NETISR_CPUID_NONE); 369 } 370 } 371 372 /* 373 * Query the RSS bucket associated with the given hash value and 374 * type. 375 */ 376 int 377 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 378 { 379 380 switch (hash_type) { 381 case M_HASHTYPE_RSS_IPV4: 382 case M_HASHTYPE_RSS_TCP_IPV4: 383 case M_HASHTYPE_RSS_UDP_IPV4: 384 case M_HASHTYPE_RSS_IPV6: 385 case M_HASHTYPE_RSS_TCP_IPV6: 386 case M_HASHTYPE_RSS_UDP_IPV6: 387 *bucket_id = rss_getbucket(hash_val); 388 return (0); 389 default: 390 return (-1); 391 } 392 } 393 394 /* 395 * netisr CPU affinity lookup routine for use by protocols. 396 */ 397 struct mbuf * 398 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 399 { 400 401 M_ASSERTPKTHDR(m); 402 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 403 return (m); 404 } 405 406 int 407 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 408 { 409 410 M_ASSERTPKTHDR(m); 411 412 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 413 bucket_id)); 414 } 415 416 /* 417 * Query the RSS hash algorithm. 418 */ 419 u_int 420 rss_gethashalgo(void) 421 { 422 423 return (rss_hashalgo); 424 } 425 426 /* 427 * Query the current RSS key; likely to be used by device drivers when 428 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 429 * 430 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 431 */ 432 void 433 rss_getkey(uint8_t *key) 434 { 435 436 bcopy(rss_key, key, sizeof(rss_key)); 437 } 438 439 /* 440 * Query the number of buckets; this may be used by both network device 441 * drivers, which will need to populate hardware shadows of the software 442 * indirection table, and the network stack itself (such as when deciding how 443 * many connection groups to allocate). 444 */ 445 u_int 446 rss_getnumbuckets(void) 447 { 448 449 return (rss_buckets); 450 } 451 452 /* 453 * Query the number of CPUs in use by RSS; may be useful to device drivers 454 * trying to figure out how to map a larger number of CPUs into a smaller 455 * number of receive queues. 456 */ 457 u_int 458 rss_getnumcpus(void) 459 { 460 461 return (rss_ncpus); 462 } 463 464 /* 465 * Return the supported RSS hash configuration. 466 * 467 * NICs should query this to determine what to configure in their redirection 468 * matching table. 469 */ 470 inline u_int 471 rss_gethashconfig(void) 472 { 473 474 /* Return 4-tuple for TCP; 2-tuple for others */ 475 /* 476 * UDP may fragment more often than TCP and thus we'll end up with 477 * NICs returning 2-tuple fragments. 478 * udp_init() and udplite_init() both currently initialise things 479 * as 2-tuple. 480 * So for now disable UDP 4-tuple hashing until all of the other 481 * pieces are in place. 482 */ 483 return ( 484 RSS_HASHTYPE_RSS_IPV4 485 | RSS_HASHTYPE_RSS_TCP_IPV4 486 | RSS_HASHTYPE_RSS_IPV6 487 | RSS_HASHTYPE_RSS_TCP_IPV6 488 | RSS_HASHTYPE_RSS_IPV6_EX 489 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 490 #if 0 491 | RSS_HASHTYPE_RSS_UDP_IPV4 492 | RSS_HASHTYPE_RSS_UDP_IPV6 493 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 494 #endif 495 ); 496 } 497 498 /* 499 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 500 * it appearing in debugging output unnecessarily. 501 */ 502 static int 503 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 504 { 505 uint8_t temp_rss_key[RSS_KEYSIZE]; 506 int error; 507 508 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 509 if (error) 510 return (error); 511 512 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 513 error = sysctl_handle_opaque(oidp, temp_rss_key, 514 sizeof(temp_rss_key), req); 515 if (error) 516 return (error); 517 if (req->newptr != NULL) { 518 /* XXXRW: Not yet. */ 519 return (EINVAL); 520 } 521 return (0); 522 } 523 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 524 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 525 "", "RSS keying material"); 526 527 static int 528 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 529 { 530 struct sbuf *sb; 531 int error; 532 int i; 533 534 error = 0; 535 error = sysctl_wire_old_buffer(req, 0); 536 if (error != 0) 537 return (error); 538 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 539 if (sb == NULL) 540 return (ENOMEM); 541 for (i = 0; i < rss_buckets; i++) { 542 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 543 i, 544 rss_getcpu(i)); 545 } 546 error = sbuf_finish(sb); 547 sbuf_delete(sb); 548 549 return (error); 550 } 551 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 552 CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 553 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 554