1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 31 #include "opt_inet6.h" 32 #include "opt_inet.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/mbuf.h> 37 #include <sys/socket.h> 38 #include <sys/priv.h> 39 #include <sys/kernel.h> 40 #include <sys/smp.h> 41 #include <sys/sysctl.h> 42 #include <sys/sbuf.h> 43 44 #include <net/if.h> 45 #include <net/if_var.h> 46 #include <net/netisr.h> 47 #include <net/rss_config.h> 48 #include <net/toeplitz.h> 49 50 /*- 51 * Operating system parts of receiver-side scaling (RSS), which allows 52 * network cards to direct flows to particular receive queues based on hashes 53 * of header tuples. This implementation aligns RSS buckets with connection 54 * groups at the TCP/IP layer, so each bucket is associated with exactly one 55 * group. As a result, the group lookup structures (and lock) should have an 56 * effective affinity with exactly one CPU. 57 * 58 * Network device drivers needing to configure RSS will query this framework 59 * for parameters, such as the current RSS key, hashing policies, number of 60 * bits, and indirection table mapping hashes to buckets and CPUs. They may 61 * provide their own supplementary information, such as queue<->CPU bindings. 62 * It is the responsibility of the network device driver to inject packets 63 * into the stack on as close to the right CPU as possible, if playing by RSS 64 * rules. 65 * 66 * TODO: 67 * 68 * - Synchronization for rss_key and other future-configurable parameters. 69 * - Event handler drivers can register to pick up RSS configuration changes. 70 * - Should we allow rss_basecpu to be configured? 71 * - Randomize key on boot. 72 * - IPv6 support. 73 * - Statistics on how often there's a misalignment between hardware 74 * placement and pcbgroup expectations. 75 */ 76 77 #if !defined(INET) && !defined(INET6) 78 #define _net_inet _net 79 #define _net_inet_rss _net_rss 80 #endif 81 SYSCTL_DECL(_net_inet); 82 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 83 "Receive-side steering"); 84 85 /* 86 * Toeplitz is the only required hash function in the RSS spec, so use it by 87 * default. 88 */ 89 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 90 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 91 "RSS hash algorithm"); 92 93 #ifdef RSS 94 /* 95 * Size of the indirection table; at most 128 entries per the RSS spec. We 96 * size it to at least 2 times the number of CPUs by default to allow useful 97 * rebalancing. If not set explicitly with a loader tunable, we tune based 98 * on the number of CPUs present. 99 * 100 * XXXRW: buckets might be better to use for the tunable than bits. 101 */ 102 static u_int rss_bits; 103 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 104 "RSS bits"); 105 106 static u_int rss_mask; 107 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 108 "RSS mask"); 109 110 static const u_int rss_maxbits = RSS_MAXBITS; 111 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 112 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 113 114 /* 115 * RSS's own count of the number of CPUs it could be using for processing. 116 * Bounded to 64 by RSS constants. 117 */ 118 static u_int rss_ncpus; 119 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 120 "Number of CPUs available to RSS"); 121 122 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 123 static const u_int rss_maxcpus = RSS_MAXCPUS; 124 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 125 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 126 127 /* 128 * Variable exists just for reporting rss_bits in a user-friendly way. 129 */ 130 static u_int rss_buckets; 131 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 132 "RSS buckets"); 133 134 /* 135 * Base CPU number; devices will add this to all CPU numbers returned by the 136 * RSS indirection table. Currently unmodifable in FreeBSD. 137 */ 138 static const u_int rss_basecpu; 139 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 140 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 141 142 #endif 143 /* 144 * Print verbose debugging messages. 145 * 0 - disable 146 * non-zero - enable 147 */ 148 int rss_debug = 0; 149 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0, 150 "RSS debug level"); 151 152 /* 153 * RSS secret key, intended to prevent attacks on load-balancing. Its 154 * effectiveness may be limited by algorithm choice and available entropy 155 * during the boot. 156 * 157 * XXXRW: And that we don't randomize it yet! 158 * 159 * This is the default Microsoft RSS specification key which is also 160 * the Chelsio T5 firmware default key. 161 */ 162 static uint8_t rss_key[RSS_KEYSIZE] = { 163 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 164 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 165 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 166 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 167 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 168 }; 169 170 #ifdef RSS 171 /* 172 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 173 * Drivers may supplement this table with a separate CPU<->queue table when 174 * programming devices. 175 */ 176 struct rss_table_entry { 177 uint8_t rte_cpu; /* CPU affinity of bucket. */ 178 }; 179 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 180 #endif 181 182 static void 183 rss_init(__unused void *arg) 184 { 185 #ifdef RSS 186 u_int i; 187 u_int cpuid; 188 #endif 189 /* 190 * Validate tunables, coerce to sensible values. 191 */ 192 switch (rss_hashalgo) { 193 case RSS_HASH_TOEPLITZ: 194 case RSS_HASH_NAIVE: 195 break; 196 197 default: 198 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", 199 rss_hashalgo, RSS_HASH_TOEPLITZ); 200 rss_hashalgo = RSS_HASH_TOEPLITZ; 201 } 202 203 #ifdef RSS 204 /* 205 * Count available CPUs. 206 * 207 * XXXRW: Note incorrect assumptions regarding contiguity of this set 208 * elsewhere. 209 */ 210 rss_ncpus = 0; 211 for (i = 0; i <= mp_maxid; i++) { 212 if (CPU_ABSENT(i)) 213 continue; 214 rss_ncpus++; 215 } 216 if (rss_ncpus > RSS_MAXCPUS) 217 rss_ncpus = RSS_MAXCPUS; 218 219 /* 220 * Tune RSS table entries to be no less than 2x the number of CPUs 221 * -- unless we're running uniprocessor, in which case there's not 222 * much point in having buckets to rearrange for load-balancing! 223 */ 224 if (rss_ncpus > 1) { 225 if (rss_bits == 0) 226 rss_bits = fls(rss_ncpus - 1) + 1; 227 228 /* 229 * Microsoft limits RSS table entries to 128, so apply that 230 * limit to both auto-detected CPU counts and user-configured 231 * ones. 232 */ 233 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 234 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", 235 rss_bits, RSS_MAXBITS); 236 rss_bits = RSS_MAXBITS; 237 } 238 239 /* 240 * Figure out how many buckets to use; warn if less than the 241 * number of configured CPUs, although this is not a fatal 242 * problem. 243 */ 244 rss_buckets = (1 << rss_bits); 245 if (rss_buckets < rss_ncpus) 246 RSS_DEBUG("WARNING: rss_buckets (%u) less than " 247 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); 248 rss_mask = rss_buckets - 1; 249 } else { 250 rss_bits = 0; 251 rss_buckets = 1; 252 rss_mask = 0; 253 } 254 255 /* 256 * Set up initial CPU assignments: round-robin by default. 257 */ 258 cpuid = CPU_FIRST(); 259 for (i = 0; i < rss_buckets; i++) { 260 rss_table[i].rte_cpu = cpuid; 261 cpuid = CPU_NEXT(cpuid); 262 } 263 #endif /* RSS */ 264 /* 265 * Randomize rrs_key. 266 * 267 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 268 * loop to check for "bad" RSS keys. 269 */ 270 } 271 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 272 273 static uint32_t 274 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 275 const uint8_t *data) 276 { 277 uint32_t v; 278 u_int i; 279 280 v = 0; 281 for (i = 0; i < keylen; i++) 282 v += key[i]; 283 for (i = 0; i < datalen; i++) 284 v += data[i]; 285 return (v); 286 } 287 288 uint32_t 289 rss_hash(u_int datalen, const uint8_t *data) 290 { 291 292 switch (rss_hashalgo) { 293 case RSS_HASH_TOEPLITZ: 294 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 295 data)); 296 297 case RSS_HASH_NAIVE: 298 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 299 data)); 300 301 default: 302 panic("%s: unsupported/unknown hashalgo %d", __func__, 303 rss_hashalgo); 304 } 305 } 306 307 /* 308 * Query the current RSS key; likely to be used by device drivers when 309 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 310 * 311 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 312 */ 313 void 314 rss_getkey(uint8_t *key) 315 { 316 317 bcopy(rss_key, key, sizeof(rss_key)); 318 } 319 320 /* 321 * Query the RSS hash algorithm. 322 */ 323 u_int 324 rss_gethashalgo(void) 325 { 326 327 return (rss_hashalgo); 328 } 329 330 #ifdef RSS 331 /* 332 * Query the number of RSS bits in use. 333 */ 334 u_int 335 rss_getbits(void) 336 { 337 338 return (rss_bits); 339 } 340 341 /* 342 * Query the RSS bucket associated with an RSS hash. 343 */ 344 u_int 345 rss_getbucket(u_int hash) 346 { 347 348 return (hash & rss_mask); 349 } 350 351 /* 352 * Query the RSS layer bucket associated with the given 353 * entry in the RSS hash space. 354 * 355 * The RSS indirection table is 0 .. rss_buckets-1, 356 * covering the low 'rss_bits' of the total 128 slot 357 * RSS indirection table. So just mask off rss_bits and 358 * return that. 359 * 360 * NIC drivers can then iterate over the 128 slot RSS 361 * indirection table and fetch which RSS bucket to 362 * map it to. This will typically be a CPU queue 363 */ 364 u_int 365 rss_get_indirection_to_bucket(u_int index) 366 { 367 368 return (index & rss_mask); 369 } 370 371 /* 372 * Query the RSS CPU associated with an RSS bucket. 373 */ 374 u_int 375 rss_getcpu(u_int bucket) 376 { 377 378 return (rss_table[bucket].rte_cpu); 379 } 380 381 /* 382 * netisr CPU affinity lookup given just the hash and hashtype. 383 */ 384 u_int 385 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 386 { 387 388 switch (hash_type) { 389 case M_HASHTYPE_RSS_IPV4: 390 case M_HASHTYPE_RSS_TCP_IPV4: 391 case M_HASHTYPE_RSS_UDP_IPV4: 392 case M_HASHTYPE_RSS_IPV6: 393 case M_HASHTYPE_RSS_TCP_IPV6: 394 case M_HASHTYPE_RSS_UDP_IPV6: 395 return (rss_getcpu(rss_getbucket(hash_val))); 396 default: 397 return (NETISR_CPUID_NONE); 398 } 399 } 400 401 /* 402 * Query the RSS bucket associated with the given hash value and 403 * type. 404 */ 405 int 406 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 407 { 408 409 switch (hash_type) { 410 case M_HASHTYPE_RSS_IPV4: 411 case M_HASHTYPE_RSS_TCP_IPV4: 412 case M_HASHTYPE_RSS_UDP_IPV4: 413 case M_HASHTYPE_RSS_IPV6: 414 case M_HASHTYPE_RSS_TCP_IPV6: 415 case M_HASHTYPE_RSS_UDP_IPV6: 416 *bucket_id = rss_getbucket(hash_val); 417 return (0); 418 default: 419 return (-1); 420 } 421 } 422 423 /* 424 * netisr CPU affinity lookup routine for use by protocols. 425 */ 426 struct mbuf * 427 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 428 { 429 430 M_ASSERTPKTHDR(m); 431 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 432 return (m); 433 } 434 435 int 436 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 437 { 438 439 M_ASSERTPKTHDR(m); 440 441 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 442 bucket_id)); 443 } 444 445 /* 446 * Query the number of buckets; this may be used by both network device 447 * drivers, which will need to populate hardware shadows of the software 448 * indirection table, and the network stack itself (such as when deciding how 449 * many connection groups to allocate). 450 */ 451 u_int 452 rss_getnumbuckets(void) 453 { 454 455 return (rss_buckets); 456 } 457 458 /* 459 * Query the number of CPUs in use by RSS; may be useful to device drivers 460 * trying to figure out how to map a larger number of CPUs into a smaller 461 * number of receive queues. 462 */ 463 u_int 464 rss_getnumcpus(void) 465 { 466 467 return (rss_ncpus); 468 } 469 470 #endif 471 /* 472 * Return the supported RSS hash configuration. 473 * 474 * NICs should query this to determine what to configure in their redirection 475 * matching table. 476 */ 477 inline u_int 478 rss_gethashconfig(void) 479 { 480 481 /* Return 4-tuple for TCP; 2-tuple for others */ 482 /* 483 * UDP may fragment more often than TCP and thus we'll end up with 484 * NICs returning 2-tuple fragments. 485 * udp_init() and udplite_init() both currently initialise things 486 * as 2-tuple. 487 * So for now disable UDP 4-tuple hashing until all of the other 488 * pieces are in place. 489 */ 490 return ( 491 RSS_HASHTYPE_RSS_IPV4 492 | RSS_HASHTYPE_RSS_TCP_IPV4 493 | RSS_HASHTYPE_RSS_IPV6 494 | RSS_HASHTYPE_RSS_TCP_IPV6 495 | RSS_HASHTYPE_RSS_IPV6_EX 496 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 497 #if 0 498 | RSS_HASHTYPE_RSS_UDP_IPV4 499 | RSS_HASHTYPE_RSS_UDP_IPV6 500 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 501 #endif 502 ); 503 } 504 505 /* 506 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 507 * it appearing in debugging output unnecessarily. 508 */ 509 static int 510 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 511 { 512 uint8_t temp_rss_key[RSS_KEYSIZE]; 513 int error; 514 515 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 516 if (error) 517 return (error); 518 519 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 520 error = sysctl_handle_opaque(oidp, temp_rss_key, 521 sizeof(temp_rss_key), req); 522 if (error) 523 return (error); 524 if (req->newptr != NULL) { 525 /* XXXRW: Not yet. */ 526 return (EINVAL); 527 } 528 return (0); 529 } 530 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 531 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 532 "", "RSS keying material"); 533 534 #ifdef RSS 535 static int 536 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 537 { 538 struct sbuf *sb; 539 int error; 540 int i; 541 542 error = 0; 543 error = sysctl_wire_old_buffer(req, 0); 544 if (error != 0) 545 return (error); 546 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 547 if (sb == NULL) 548 return (ENOMEM); 549 for (i = 0; i < rss_buckets; i++) { 550 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 551 i, 552 rss_getcpu(i)); 553 } 554 error = sbuf_finish(sb); 555 sbuf_delete(sb); 556 557 return (error); 558 } 559 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 560 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 561 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 562 #endif 563