1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 31 #include "opt_inet6.h" 32 33 #include <sys/param.h> 34 #include <sys/mbuf.h> 35 #include <sys/socket.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/sbuf.h> 41 42 #include <net/if.h> 43 #include <net/if_var.h> 44 #include <net/netisr.h> 45 #include <net/rss_config.h> 46 #include <net/toeplitz.h> 47 48 /*- 49 * Operating system parts of receiver-side scaling (RSS), which allows 50 * network cards to direct flows to particular receive queues based on hashes 51 * of header tuples. This implementation aligns RSS buckets with connection 52 * groups at the TCP/IP layer, so each bucket is associated with exactly one 53 * group. As a result, the group lookup structures (and lock) should have an 54 * effective affinity with exactly one CPU. 55 * 56 * Network device drivers needing to configure RSS will query this framework 57 * for parameters, such as the current RSS key, hashing policies, number of 58 * bits, and indirection table mapping hashes to buckets and CPUs. They may 59 * provide their own supplementary information, such as queue<->CPU bindings. 60 * It is the responsibility of the network device driver to inject packets 61 * into the stack on as close to the right CPU as possible, if playing by RSS 62 * rules. 63 * 64 * TODO: 65 * 66 * - Synchronization for rss_key and other future-configurable parameters. 67 * - Event handler drivers can register to pick up RSS configuration changes. 68 * - Should we allow rss_basecpu to be configured? 69 * - Randomize key on boot. 70 * - IPv6 support. 71 * - Statistics on how often there's a misalignment between hardware 72 * placement and pcbgroup expectations. 73 */ 74 75 SYSCTL_DECL(_net_inet); 76 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 77 "Receive-side steering"); 78 79 /* 80 * Toeplitz is the only required hash function in the RSS spec, so use it by 81 * default. 82 */ 83 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 84 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 85 "RSS hash algorithm"); 86 87 /* 88 * Size of the indirection table; at most 128 entries per the RSS spec. We 89 * size it to at least 2 times the number of CPUs by default to allow useful 90 * rebalancing. If not set explicitly with a loader tunable, we tune based 91 * on the number of CPUs present. 92 * 93 * XXXRW: buckets might be better to use for the tunable than bits. 94 */ 95 static u_int rss_bits; 96 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 97 "RSS bits"); 98 99 static u_int rss_mask; 100 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 101 "RSS mask"); 102 103 static const u_int rss_maxbits = RSS_MAXBITS; 104 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 105 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 106 107 /* 108 * RSS's own count of the number of CPUs it could be using for processing. 109 * Bounded to 64 by RSS constants. 110 */ 111 static u_int rss_ncpus; 112 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 113 "Number of CPUs available to RSS"); 114 115 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 116 static const u_int rss_maxcpus = RSS_MAXCPUS; 117 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 118 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 119 120 /* 121 * Variable exists just for reporting rss_bits in a user-friendly way. 122 */ 123 static u_int rss_buckets; 124 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 125 "RSS buckets"); 126 127 /* 128 * Base CPU number; devices will add this to all CPU numbers returned by the 129 * RSS indirection table. Currently unmodifable in FreeBSD. 130 */ 131 static const u_int rss_basecpu; 132 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 133 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 134 135 /* 136 * Print verbose debugging messages. 137 * 0 - disable 138 * non-zero - enable 139 */ 140 int rss_debug = 0; 141 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0, 142 "RSS debug level"); 143 144 /* 145 * RSS secret key, intended to prevent attacks on load-balancing. Its 146 * effectiveness may be limited by algorithm choice and available entropy 147 * during the boot. 148 * 149 * XXXRW: And that we don't randomize it yet! 150 * 151 * This is the default Microsoft RSS specification key which is also 152 * the Chelsio T5 firmware default key. 153 */ 154 static uint8_t rss_key[RSS_KEYSIZE] = { 155 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 156 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 157 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 158 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 159 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 160 }; 161 162 /* 163 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 164 * Drivers may supplement this table with a separate CPU<->queue table when 165 * programming devices. 166 */ 167 struct rss_table_entry { 168 uint8_t rte_cpu; /* CPU affinity of bucket. */ 169 }; 170 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 171 172 static void 173 rss_init(__unused void *arg) 174 { 175 u_int i; 176 u_int cpuid; 177 178 /* 179 * Validate tunables, coerce to sensible values. 180 */ 181 switch (rss_hashalgo) { 182 case RSS_HASH_TOEPLITZ: 183 case RSS_HASH_NAIVE: 184 break; 185 186 default: 187 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", 188 rss_hashalgo, RSS_HASH_TOEPLITZ); 189 rss_hashalgo = RSS_HASH_TOEPLITZ; 190 } 191 192 /* 193 * Count available CPUs. 194 * 195 * XXXRW: Note incorrect assumptions regarding contiguity of this set 196 * elsewhere. 197 */ 198 rss_ncpus = 0; 199 for (i = 0; i <= mp_maxid; i++) { 200 if (CPU_ABSENT(i)) 201 continue; 202 rss_ncpus++; 203 } 204 if (rss_ncpus > RSS_MAXCPUS) 205 rss_ncpus = RSS_MAXCPUS; 206 207 /* 208 * Tune RSS table entries to be no less than 2x the number of CPUs 209 * -- unless we're running uniprocessor, in which case there's not 210 * much point in having buckets to rearrange for load-balancing! 211 */ 212 if (rss_ncpus > 1) { 213 if (rss_bits == 0) 214 rss_bits = fls(rss_ncpus - 1) + 1; 215 216 /* 217 * Microsoft limits RSS table entries to 128, so apply that 218 * limit to both auto-detected CPU counts and user-configured 219 * ones. 220 */ 221 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 222 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", 223 rss_bits, RSS_MAXBITS); 224 rss_bits = RSS_MAXBITS; 225 } 226 227 /* 228 * Figure out how many buckets to use; warn if less than the 229 * number of configured CPUs, although this is not a fatal 230 * problem. 231 */ 232 rss_buckets = (1 << rss_bits); 233 if (rss_buckets < rss_ncpus) 234 RSS_DEBUG("WARNING: rss_buckets (%u) less than " 235 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); 236 rss_mask = rss_buckets - 1; 237 } else { 238 rss_bits = 0; 239 rss_buckets = 1; 240 rss_mask = 0; 241 } 242 243 /* 244 * Set up initial CPU assignments: round-robin by default. 245 */ 246 cpuid = CPU_FIRST(); 247 for (i = 0; i < rss_buckets; i++) { 248 rss_table[i].rte_cpu = cpuid; 249 cpuid = CPU_NEXT(cpuid); 250 } 251 252 /* 253 * Randomize rrs_key. 254 * 255 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 256 * loop to check for "bad" RSS keys. 257 */ 258 } 259 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 260 261 static uint32_t 262 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 263 const uint8_t *data) 264 { 265 uint32_t v; 266 u_int i; 267 268 v = 0; 269 for (i = 0; i < keylen; i++) 270 v += key[i]; 271 for (i = 0; i < datalen; i++) 272 v += data[i]; 273 return (v); 274 } 275 276 uint32_t 277 rss_hash(u_int datalen, const uint8_t *data) 278 { 279 280 switch (rss_hashalgo) { 281 case RSS_HASH_TOEPLITZ: 282 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 283 data)); 284 285 case RSS_HASH_NAIVE: 286 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 287 data)); 288 289 default: 290 panic("%s: unsupported/unknown hashalgo %d", __func__, 291 rss_hashalgo); 292 } 293 } 294 295 /* 296 * Query the number of RSS bits in use. 297 */ 298 u_int 299 rss_getbits(void) 300 { 301 302 return (rss_bits); 303 } 304 305 /* 306 * Query the RSS bucket associated with an RSS hash. 307 */ 308 u_int 309 rss_getbucket(u_int hash) 310 { 311 312 return (hash & rss_mask); 313 } 314 315 /* 316 * Query the RSS layer bucket associated with the given 317 * entry in the RSS hash space. 318 * 319 * The RSS indirection table is 0 .. rss_buckets-1, 320 * covering the low 'rss_bits' of the total 128 slot 321 * RSS indirection table. So just mask off rss_bits and 322 * return that. 323 * 324 * NIC drivers can then iterate over the 128 slot RSS 325 * indirection table and fetch which RSS bucket to 326 * map it to. This will typically be a CPU queue 327 */ 328 u_int 329 rss_get_indirection_to_bucket(u_int index) 330 { 331 332 return (index & rss_mask); 333 } 334 335 /* 336 * Query the RSS CPU associated with an RSS bucket. 337 */ 338 u_int 339 rss_getcpu(u_int bucket) 340 { 341 342 return (rss_table[bucket].rte_cpu); 343 } 344 345 /* 346 * netisr CPU affinity lookup given just the hash and hashtype. 347 */ 348 u_int 349 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 350 { 351 352 switch (hash_type) { 353 case M_HASHTYPE_RSS_IPV4: 354 case M_HASHTYPE_RSS_TCP_IPV4: 355 case M_HASHTYPE_RSS_UDP_IPV4: 356 case M_HASHTYPE_RSS_IPV6: 357 case M_HASHTYPE_RSS_TCP_IPV6: 358 case M_HASHTYPE_RSS_UDP_IPV6: 359 return (rss_getcpu(rss_getbucket(hash_val))); 360 default: 361 return (NETISR_CPUID_NONE); 362 } 363 } 364 365 /* 366 * Query the RSS bucket associated with the given hash value and 367 * type. 368 */ 369 int 370 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 371 { 372 373 switch (hash_type) { 374 case M_HASHTYPE_RSS_IPV4: 375 case M_HASHTYPE_RSS_TCP_IPV4: 376 case M_HASHTYPE_RSS_UDP_IPV4: 377 case M_HASHTYPE_RSS_IPV6: 378 case M_HASHTYPE_RSS_TCP_IPV6: 379 case M_HASHTYPE_RSS_UDP_IPV6: 380 *bucket_id = rss_getbucket(hash_val); 381 return (0); 382 default: 383 return (-1); 384 } 385 } 386 387 /* 388 * netisr CPU affinity lookup routine for use by protocols. 389 */ 390 struct mbuf * 391 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 392 { 393 394 M_ASSERTPKTHDR(m); 395 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 396 return (m); 397 } 398 399 int 400 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 401 { 402 403 M_ASSERTPKTHDR(m); 404 405 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 406 bucket_id)); 407 } 408 409 /* 410 * Query the RSS hash algorithm. 411 */ 412 u_int 413 rss_gethashalgo(void) 414 { 415 416 return (rss_hashalgo); 417 } 418 419 /* 420 * Query the current RSS key; likely to be used by device drivers when 421 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 422 * 423 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 424 */ 425 void 426 rss_getkey(uint8_t *key) 427 { 428 429 bcopy(rss_key, key, sizeof(rss_key)); 430 } 431 432 /* 433 * Query the number of buckets; this may be used by both network device 434 * drivers, which will need to populate hardware shadows of the software 435 * indirection table, and the network stack itself (such as when deciding how 436 * many connection groups to allocate). 437 */ 438 u_int 439 rss_getnumbuckets(void) 440 { 441 442 return (rss_buckets); 443 } 444 445 /* 446 * Query the number of CPUs in use by RSS; may be useful to device drivers 447 * trying to figure out how to map a larger number of CPUs into a smaller 448 * number of receive queues. 449 */ 450 u_int 451 rss_getnumcpus(void) 452 { 453 454 return (rss_ncpus); 455 } 456 457 /* 458 * Return the supported RSS hash configuration. 459 * 460 * NICs should query this to determine what to configure in their redirection 461 * matching table. 462 */ 463 inline u_int 464 rss_gethashconfig(void) 465 { 466 467 /* Return 4-tuple for TCP; 2-tuple for others */ 468 /* 469 * UDP may fragment more often than TCP and thus we'll end up with 470 * NICs returning 2-tuple fragments. 471 * udp_init() and udplite_init() both currently initialise things 472 * as 2-tuple. 473 * So for now disable UDP 4-tuple hashing until all of the other 474 * pieces are in place. 475 */ 476 return ( 477 RSS_HASHTYPE_RSS_IPV4 478 | RSS_HASHTYPE_RSS_TCP_IPV4 479 | RSS_HASHTYPE_RSS_IPV6 480 | RSS_HASHTYPE_RSS_TCP_IPV6 481 | RSS_HASHTYPE_RSS_IPV6_EX 482 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 483 #if 0 484 | RSS_HASHTYPE_RSS_UDP_IPV4 485 | RSS_HASHTYPE_RSS_UDP_IPV6 486 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 487 #endif 488 ); 489 } 490 491 /* 492 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 493 * it appearing in debugging output unnecessarily. 494 */ 495 static int 496 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 497 { 498 uint8_t temp_rss_key[RSS_KEYSIZE]; 499 int error; 500 501 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 502 if (error) 503 return (error); 504 505 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 506 error = sysctl_handle_opaque(oidp, temp_rss_key, 507 sizeof(temp_rss_key), req); 508 if (error) 509 return (error); 510 if (req->newptr != NULL) { 511 /* XXXRW: Not yet. */ 512 return (EINVAL); 513 } 514 return (0); 515 } 516 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 517 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 518 "", "RSS keying material"); 519 520 static int 521 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 522 { 523 struct sbuf *sb; 524 int error; 525 int i; 526 527 error = 0; 528 error = sysctl_wire_old_buffer(req, 0); 529 if (error != 0) 530 return (error); 531 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 532 if (sb == NULL) 533 return (ENOMEM); 534 for (i = 0; i < rss_buckets; i++) { 535 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 536 i, 537 rss_getcpu(i)); 538 } 539 error = sbuf_finish(sb); 540 sbuf_delete(sb); 541 542 return (error); 543 } 544 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 545 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, 546 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 547