1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 31 #include "opt_inet6.h" 32 #include "opt_inet.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/mbuf.h> 37 #include <sys/socket.h> 38 #include <sys/priv.h> 39 #include <sys/kernel.h> 40 #include <sys/smp.h> 41 #include <sys/sysctl.h> 42 #include <sys/sbuf.h> 43 44 #include <net/if.h> 45 #include <net/if_var.h> 46 #include <net/netisr.h> 47 #include <net/rss_config.h> 48 #include <net/toeplitz.h> 49 50 /*- 51 * Operating system parts of receiver-side scaling (RSS), which allows 52 * network cards to direct flows to particular receive queues based on hashes 53 * of header tuples. This implementation aligns RSS buckets with connection 54 * groups at the TCP/IP layer, so each bucket is associated with exactly one 55 * group. As a result, the group lookup structures (and lock) should have an 56 * effective affinity with exactly one CPU. 57 * 58 * Network device drivers needing to configure RSS will query this framework 59 * for parameters, such as the current RSS key, hashing policies, number of 60 * bits, and indirection table mapping hashes to buckets and CPUs. They may 61 * provide their own supplementary information, such as queue<->CPU bindings. 62 * It is the responsibility of the network device driver to inject packets 63 * into the stack on as close to the right CPU as possible, if playing by RSS 64 * rules. 65 * 66 * TODO: 67 * 68 * - Synchronization for rss_key and other future-configurable parameters. 69 * - Event handler drivers can register to pick up RSS configuration changes. 70 * - Should we allow rss_basecpu to be configured? 71 * - Randomize key on boot. 72 * - IPv6 support. 73 * - Statistics on how often there's a misalignment between hardware 74 * placement and pcbgroup expectations. 75 */ 76 77 #if !defined(INET) && !defined(INET6) 78 #define _net_inet _net 79 #define _net_inet_rss _net_rss 80 #endif 81 SYSCTL_DECL(_net_inet); 82 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 83 "Receive-side steering"); 84 85 /* 86 * Toeplitz is the only required hash function in the RSS spec, so use it by 87 * default. 88 */ 89 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 90 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 91 "RSS hash algorithm"); 92 93 #ifdef RSS 94 FEATURE(rss, "Receiver-side scaling"); 95 /* 96 * Size of the indirection table; at most 128 entries per the RSS spec. We 97 * size it to at least 2 times the number of CPUs by default to allow useful 98 * rebalancing. If not set explicitly with a loader tunable, we tune based 99 * on the number of CPUs present. 100 * 101 * XXXRW: buckets might be better to use for the tunable than bits. 102 */ 103 static u_int rss_bits; 104 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 105 "RSS bits"); 106 107 static u_int rss_mask; 108 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 109 "RSS mask"); 110 111 static const u_int rss_maxbits = RSS_MAXBITS; 112 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 113 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 114 115 /* 116 * RSS's own count of the number of CPUs it could be using for processing. 117 * Bounded to 64 by RSS constants. 118 */ 119 static u_int rss_ncpus; 120 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 121 "Number of CPUs available to RSS"); 122 123 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 124 static const u_int rss_maxcpus = RSS_MAXCPUS; 125 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 126 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 127 128 /* 129 * Variable exists just for reporting rss_bits in a user-friendly way. 130 */ 131 static u_int rss_buckets; 132 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 133 "RSS buckets"); 134 135 /* 136 * Base CPU number; devices will add this to all CPU numbers returned by the 137 * RSS indirection table. Currently unmodifable in FreeBSD. 138 */ 139 static const u_int rss_basecpu; 140 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 141 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 142 143 #endif 144 /* 145 * Print verbose debugging messages. 146 * 0 - disable 147 * non-zero - enable 148 */ 149 int rss_debug = 0; 150 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0, 151 "RSS debug level"); 152 153 /* 154 * RSS secret key, intended to prevent attacks on load-balancing. Its 155 * effectiveness may be limited by algorithm choice and available entropy 156 * during the boot. 157 * 158 * XXXRW: And that we don't randomize it yet! 159 * 160 * This is the default Microsoft RSS specification key which is also 161 * the Chelsio T5 firmware default key. 162 */ 163 static uint8_t rss_key[RSS_KEYSIZE] = { 164 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 165 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 166 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 167 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 168 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 169 }; 170 171 #ifdef RSS 172 /* 173 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 174 * Drivers may supplement this table with a separate CPU<->queue table when 175 * programming devices. 176 */ 177 struct rss_table_entry { 178 uint8_t rte_cpu; /* CPU affinity of bucket. */ 179 }; 180 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 181 #endif 182 183 static void 184 rss_init(__unused void *arg) 185 { 186 #ifdef RSS 187 u_int i; 188 u_int cpuid; 189 #endif 190 /* 191 * Validate tunables, coerce to sensible values. 192 */ 193 switch (rss_hashalgo) { 194 case RSS_HASH_TOEPLITZ: 195 case RSS_HASH_NAIVE: 196 break; 197 198 default: 199 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", 200 rss_hashalgo, RSS_HASH_TOEPLITZ); 201 rss_hashalgo = RSS_HASH_TOEPLITZ; 202 } 203 204 #ifdef RSS 205 /* 206 * Count available CPUs. 207 * 208 * XXXRW: Note incorrect assumptions regarding contiguity of this set 209 * elsewhere. 210 */ 211 rss_ncpus = 0; 212 for (i = 0; i <= mp_maxid; i++) { 213 if (CPU_ABSENT(i)) 214 continue; 215 rss_ncpus++; 216 } 217 if (rss_ncpus > RSS_MAXCPUS) 218 rss_ncpus = RSS_MAXCPUS; 219 220 /* 221 * Tune RSS table entries to be no less than 2x the number of CPUs 222 * -- unless we're running uniprocessor, in which case there's not 223 * much point in having buckets to rearrange for load-balancing! 224 */ 225 if (rss_ncpus > 1) { 226 if (rss_bits == 0) 227 rss_bits = fls(rss_ncpus - 1) + 1; 228 229 /* 230 * Microsoft limits RSS table entries to 128, so apply that 231 * limit to both auto-detected CPU counts and user-configured 232 * ones. 233 */ 234 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 235 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", 236 rss_bits, RSS_MAXBITS); 237 rss_bits = RSS_MAXBITS; 238 } 239 240 /* 241 * Figure out how many buckets to use; warn if less than the 242 * number of configured CPUs, although this is not a fatal 243 * problem. 244 */ 245 rss_buckets = (1 << rss_bits); 246 if (rss_buckets < rss_ncpus) 247 RSS_DEBUG("WARNING: rss_buckets (%u) less than " 248 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); 249 rss_mask = rss_buckets - 1; 250 } else { 251 rss_bits = 0; 252 rss_buckets = 1; 253 rss_mask = 0; 254 } 255 256 /* 257 * Set up initial CPU assignments: round-robin by default. 258 */ 259 cpuid = CPU_FIRST(); 260 for (i = 0; i < rss_buckets; i++) { 261 rss_table[i].rte_cpu = cpuid; 262 cpuid = CPU_NEXT(cpuid); 263 } 264 #endif /* RSS */ 265 /* 266 * Randomize rrs_key. 267 * 268 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 269 * loop to check for "bad" RSS keys. 270 */ 271 } 272 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 273 274 static uint32_t 275 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 276 const uint8_t *data) 277 { 278 uint32_t v; 279 u_int i; 280 281 v = 0; 282 for (i = 0; i < keylen; i++) 283 v += key[i]; 284 for (i = 0; i < datalen; i++) 285 v += data[i]; 286 return (v); 287 } 288 289 uint32_t 290 rss_hash(u_int datalen, const uint8_t *data) 291 { 292 293 switch (rss_hashalgo) { 294 case RSS_HASH_TOEPLITZ: 295 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 296 data)); 297 298 case RSS_HASH_NAIVE: 299 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 300 data)); 301 302 default: 303 panic("%s: unsupported/unknown hashalgo %d", __func__, 304 rss_hashalgo); 305 } 306 } 307 308 /* 309 * Query the current RSS key; likely to be used by device drivers when 310 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 311 * 312 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 313 */ 314 void 315 rss_getkey(uint8_t *key) 316 { 317 318 bcopy(rss_key, key, sizeof(rss_key)); 319 } 320 321 /* 322 * Query the RSS hash algorithm. 323 */ 324 u_int 325 rss_gethashalgo(void) 326 { 327 328 return (rss_hashalgo); 329 } 330 331 #ifdef RSS 332 /* 333 * Query the number of RSS bits in use. 334 */ 335 u_int 336 rss_getbits(void) 337 { 338 339 return (rss_bits); 340 } 341 342 /* 343 * Query the RSS bucket associated with an RSS hash. 344 */ 345 u_int 346 rss_getbucket(u_int hash) 347 { 348 349 return (hash & rss_mask); 350 } 351 352 /* 353 * Query the RSS layer bucket associated with the given 354 * entry in the RSS hash space. 355 * 356 * The RSS indirection table is 0 .. rss_buckets-1, 357 * covering the low 'rss_bits' of the total 128 slot 358 * RSS indirection table. So just mask off rss_bits and 359 * return that. 360 * 361 * NIC drivers can then iterate over the 128 slot RSS 362 * indirection table and fetch which RSS bucket to 363 * map it to. This will typically be a CPU queue 364 */ 365 u_int 366 rss_get_indirection_to_bucket(u_int index) 367 { 368 369 return (index & rss_mask); 370 } 371 372 /* 373 * Query the RSS CPU associated with an RSS bucket. 374 */ 375 u_int 376 rss_getcpu(u_int bucket) 377 { 378 379 return (rss_table[bucket].rte_cpu); 380 } 381 382 /* 383 * netisr CPU affinity lookup given just the hash and hashtype. 384 */ 385 u_int 386 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 387 { 388 389 switch (hash_type) { 390 case M_HASHTYPE_RSS_IPV4: 391 case M_HASHTYPE_RSS_TCP_IPV4: 392 case M_HASHTYPE_RSS_UDP_IPV4: 393 case M_HASHTYPE_RSS_IPV6: 394 case M_HASHTYPE_RSS_TCP_IPV6: 395 case M_HASHTYPE_RSS_UDP_IPV6: 396 return (rss_getcpu(rss_getbucket(hash_val))); 397 default: 398 return (NETISR_CPUID_NONE); 399 } 400 } 401 402 /* 403 * Query the RSS bucket associated with the given hash value and 404 * type. 405 */ 406 int 407 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 408 { 409 410 switch (hash_type) { 411 case M_HASHTYPE_RSS_IPV4: 412 case M_HASHTYPE_RSS_TCP_IPV4: 413 case M_HASHTYPE_RSS_UDP_IPV4: 414 case M_HASHTYPE_RSS_IPV6: 415 case M_HASHTYPE_RSS_TCP_IPV6: 416 case M_HASHTYPE_RSS_UDP_IPV6: 417 *bucket_id = rss_getbucket(hash_val); 418 return (0); 419 default: 420 return (-1); 421 } 422 } 423 424 /* 425 * netisr CPU affinity lookup routine for use by protocols. 426 */ 427 struct mbuf * 428 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 429 { 430 431 M_ASSERTPKTHDR(m); 432 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 433 return (m); 434 } 435 436 int 437 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 438 { 439 440 M_ASSERTPKTHDR(m); 441 442 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 443 bucket_id)); 444 } 445 446 /* 447 * Query the number of buckets; this may be used by both network device 448 * drivers, which will need to populate hardware shadows of the software 449 * indirection table, and the network stack itself (such as when deciding how 450 * many connection groups to allocate). 451 */ 452 u_int 453 rss_getnumbuckets(void) 454 { 455 456 return (rss_buckets); 457 } 458 459 /* 460 * Query the number of CPUs in use by RSS; may be useful to device drivers 461 * trying to figure out how to map a larger number of CPUs into a smaller 462 * number of receive queues. 463 */ 464 u_int 465 rss_getnumcpus(void) 466 { 467 468 return (rss_ncpus); 469 } 470 471 #endif 472 /* 473 * Return the supported RSS hash configuration. 474 * 475 * NICs should query this to determine what to configure in their redirection 476 * matching table. 477 */ 478 inline u_int 479 rss_gethashconfig(void) 480 { 481 482 /* Return 4-tuple for TCP; 2-tuple for others */ 483 /* 484 * UDP may fragment more often than TCP and thus we'll end up with 485 * NICs returning 2-tuple fragments. 486 * udp_init() and udplite_init() both currently initialise things 487 * as 2-tuple. 488 * So for now disable UDP 4-tuple hashing until all of the other 489 * pieces are in place. 490 */ 491 return ( 492 RSS_HASHTYPE_RSS_IPV4 493 | RSS_HASHTYPE_RSS_TCP_IPV4 494 | RSS_HASHTYPE_RSS_IPV6 495 | RSS_HASHTYPE_RSS_TCP_IPV6 496 | RSS_HASHTYPE_RSS_IPV6_EX 497 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 498 #if 0 499 | RSS_HASHTYPE_RSS_UDP_IPV4 500 | RSS_HASHTYPE_RSS_UDP_IPV6 501 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 502 #endif 503 ); 504 } 505 506 /* 507 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 508 * it appearing in debugging output unnecessarily. 509 */ 510 static int 511 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 512 { 513 uint8_t temp_rss_key[RSS_KEYSIZE]; 514 int error; 515 516 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 517 if (error) 518 return (error); 519 520 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 521 error = sysctl_handle_opaque(oidp, temp_rss_key, 522 sizeof(temp_rss_key), req); 523 if (error) 524 return (error); 525 if (req->newptr != NULL) { 526 /* XXXRW: Not yet. */ 527 return (EINVAL); 528 } 529 return (0); 530 } 531 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 532 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 533 "", "RSS keying material"); 534 535 #ifdef RSS 536 static int 537 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 538 { 539 struct sbuf *sb; 540 int error; 541 int i; 542 543 error = 0; 544 error = sysctl_wire_old_buffer(req, 0); 545 if (error != 0) 546 return (error); 547 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 548 if (sb == NULL) 549 return (ENOMEM); 550 for (i = 0; i < rss_buckets; i++) { 551 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 552 i, 553 rss_getcpu(i)); 554 } 555 error = sbuf_finish(sb); 556 sbuf_delete(sb); 557 558 return (error); 559 } 560 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 561 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 562 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 563 #endif 564