xref: /freebsd/sys/netinet/in_rss.c (revision a0e793cbf1951d07fc47a0d9ea389d7dacba5213)
1 /*-
2  * Copyright (c) 2010-2011 Juniper Networks, Inc.
3  * All rights reserved.
4  *
5  * This software was developed by Robert N. M. Watson under contract
6  * to Juniper Networks, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_inet6.h"
35 #include "opt_pcbgroup.h"
36 
37 #ifndef PCBGROUP
38 #error "options RSS depends on options PCBGROUP"
39 #endif
40 
41 #include <sys/param.h>
42 #include <sys/mbuf.h>
43 #include <sys/socket.h>
44 #include <sys/priv.h>
45 #include <sys/kernel.h>
46 #include <sys/smp.h>
47 #include <sys/sysctl.h>
48 #include <sys/sbuf.h>
49 
50 #include <net/if.h>
51 #include <net/if_var.h>
52 #include <net/netisr.h>
53 
54 #include <netinet/in.h>
55 #include <netinet/in_pcb.h>
56 #include <netinet/in_rss.h>
57 #include <netinet/in_var.h>
58 #include <netinet/toeplitz.h>
59 
60 /*-
61  * Operating system parts of receiver-side scaling (RSS), which allows
62  * network cards to direct flows to particular receive queues based on hashes
63  * of header tuples.  This implementation aligns RSS buckets with connection
64  * groups at the TCP/IP layer, so each bucket is associated with exactly one
65  * group.  As a result, the group lookup structures (and lock) should have an
66  * effective affinity with exactly one CPU.
67  *
68  * Network device drivers needing to configure RSS will query this framework
69  * for parameters, such as the current RSS key, hashing policies, number of
70  * bits, and indirection table mapping hashes to buckets and CPUs.  They may
71  * provide their own supplementary information, such as queue<->CPU bindings.
72  * It is the responsibility of the network device driver to inject packets
73  * into the stack on as close to the right CPU as possible, if playing by RSS
74  * rules.
75  *
76  * TODO:
77  *
78  * - Synchronization for rss_key and other future-configurable parameters.
79  * - Event handler drivers can register to pick up RSS configuration changes.
80  * - Should we allow rss_basecpu to be configured?
81  * - Randomize key on boot.
82  * - IPv6 support.
83  * - Statistics on how often there's a misalignment between hardware
84  *   placement and pcbgroup expectations.
85  */
86 
87 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
88 
89 /*
90  * Toeplitz is the only required hash function in the RSS spec, so use it by
91  * default.
92  */
93 static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
94 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
95     "RSS hash algorithm");
96 
97 /*
98  * Size of the indirection table; at most 128 entries per the RSS spec.  We
99  * size it to at least 2 times the number of CPUs by default to allow useful
100  * rebalancing.  If not set explicitly with a loader tunable, we tune based
101  * on the number of CPUs present.
102  *
103  * XXXRW: buckets might be better to use for the tunable than bits.
104  */
105 static u_int	rss_bits;
106 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
107     "RSS bits");
108 
109 static u_int	rss_mask;
110 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
111     "RSS mask");
112 
113 static const u_int	rss_maxbits = RSS_MAXBITS;
114 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
115     __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
116 
117 /*
118  * RSS's own count of the number of CPUs it could be using for processing.
119  * Bounded to 64 by RSS constants.
120  */
121 static u_int	rss_ncpus;
122 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
123     "Number of CPUs available to RSS");
124 
125 #define	RSS_MAXCPUS	(1 << (RSS_MAXBITS - 1))
126 static const u_int	rss_maxcpus = RSS_MAXCPUS;
127 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
128     __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
129 
130 /*
131  * Variable exists just for reporting rss_bits in a user-friendly way.
132  */
133 static u_int	rss_buckets;
134 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
135     "RSS buckets");
136 
137 /*
138  * Base CPU number; devices will add this to all CPU numbers returned by the
139  * RSS indirection table.  Currently unmodifable in FreeBSD.
140  */
141 static const u_int	rss_basecpu;
142 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
143     __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
144 
145 /*
146  * RSS secret key, intended to prevent attacks on load-balancing.  Its
147  * effectiveness may be limited by algorithm choice and available entropy
148  * during the boot.
149  *
150  * XXXRW: And that we don't randomize it yet!
151  *
152  * This is the default Microsoft RSS specification key which is also
153  * the Chelsio T5 firmware default key.
154  */
155 static uint8_t rss_key[RSS_KEYSIZE] = {
156 	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
157 	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
158 	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
159 	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
160 	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
161 };
162 
163 /*
164  * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
165  * Drivers may supplement this table with a seperate CPU<->queue table when
166  * programming devices.
167  */
168 struct rss_table_entry {
169 	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
170 };
171 static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
172 
173 static void
174 rss_init(__unused void *arg)
175 {
176 	u_int i;
177 	u_int cpuid;
178 
179 	/*
180 	 * Validate tunables, coerce to sensible values.
181 	 */
182 	switch (rss_hashalgo) {
183 	case RSS_HASH_TOEPLITZ:
184 	case RSS_HASH_NAIVE:
185 		break;
186 
187 	default:
188 		printf("%s: invalid RSS hashalgo %u, coercing to %u",
189 		    __func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
190 		rss_hashalgo = RSS_HASH_TOEPLITZ;
191 	}
192 
193 	/*
194 	 * Count available CPUs.
195 	 *
196 	 * XXXRW: Note incorrect assumptions regarding contiguity of this set
197 	 * elsewhere.
198 	 */
199 	rss_ncpus = 0;
200 	for (i = 0; i <= mp_maxid; i++) {
201 		if (CPU_ABSENT(i))
202 			continue;
203 		rss_ncpus++;
204 	}
205 	if (rss_ncpus > RSS_MAXCPUS)
206 		rss_ncpus = RSS_MAXCPUS;
207 
208 	/*
209 	 * Tune RSS table entries to be no less than 2x the number of CPUs
210 	 * -- unless we're running uniprocessor, in which case there's not
211 	 * much point in having buckets to rearrange for load-balancing!
212 	 */
213 	if (rss_ncpus > 1) {
214 		if (rss_bits == 0)
215 			rss_bits = fls(rss_ncpus - 1) + 1;
216 
217 		/*
218 		 * Microsoft limits RSS table entries to 128, so apply that
219 		 * limit to both auto-detected CPU counts and user-configured
220 		 * ones.
221 		 */
222 		if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
223 			printf("%s: RSS bits %u not valid, coercing to  %u",
224 			    __func__, rss_bits, RSS_MAXBITS);
225 			rss_bits = RSS_MAXBITS;
226 		}
227 
228 		/*
229 		 * Figure out how many buckets to use; warn if less than the
230 		 * number of configured CPUs, although this is not a fatal
231 		 * problem.
232 		 */
233 		rss_buckets = (1 << rss_bits);
234 		if (rss_buckets < rss_ncpus)
235 			printf("%s: WARNING: rss_buckets (%u) less than "
236 			    "rss_ncpus (%u)\n", __func__, rss_buckets,
237 			    rss_ncpus);
238 		rss_mask = rss_buckets - 1;
239 	} else {
240 		rss_bits = 0;
241 		rss_buckets = 1;
242 		rss_mask = 0;
243 	}
244 
245 	/*
246 	 * Set up initial CPU assignments: round-robin by default.
247 	 */
248 	cpuid = CPU_FIRST();
249 	for (i = 0; i < rss_buckets; i++) {
250 		rss_table[i].rte_cpu = cpuid;
251 		cpuid = CPU_NEXT(cpuid);
252 	}
253 
254 	/*
255 	 * Randomize rrs_key.
256 	 *
257 	 * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
258 	 * loop to check for "bad" RSS keys.
259 	 */
260 }
261 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
262 
263 static uint32_t
264 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
265     const uint8_t *data)
266 {
267 	uint32_t v;
268 	u_int i;
269 
270 	v = 0;
271 	for (i = 0; i < keylen; i++)
272 		v += key[i];
273 	for (i = 0; i < datalen; i++)
274 		v += data[i];
275 	return (v);
276 }
277 
278 static uint32_t
279 rss_hash(u_int datalen, const uint8_t *data)
280 {
281 
282 	switch (rss_hashalgo) {
283 	case RSS_HASH_TOEPLITZ:
284 		return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
285 		    data));
286 
287 	case RSS_HASH_NAIVE:
288 		return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
289 		    data));
290 
291 	default:
292 		panic("%s: unsupported/unknown hashalgo %d", __func__,
293 		    rss_hashalgo);
294 	}
295 }
296 
297 /*
298  * Hash an IPv4 2-tuple.
299  */
300 uint32_t
301 rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst)
302 {
303 	uint8_t data[sizeof(src) + sizeof(dst)];
304 	u_int datalen;
305 
306 	datalen = 0;
307 	bcopy(&src, &data[datalen], sizeof(src));
308 	datalen += sizeof(src);
309 	bcopy(&dst, &data[datalen], sizeof(dst));
310 	datalen += sizeof(dst);
311 	return (rss_hash(datalen, data));
312 }
313 
314 /*
315  * Hash an IPv4 4-tuple.
316  */
317 uint32_t
318 rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst,
319     u_short dstport)
320 {
321 	uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) +
322 	    sizeof(dstport)];
323 	u_int datalen;
324 
325 	datalen = 0;
326 	bcopy(&src, &data[datalen], sizeof(src));
327 	datalen += sizeof(src);
328 	bcopy(&dst, &data[datalen], sizeof(dst));
329 	datalen += sizeof(dst);
330 	bcopy(&srcport, &data[datalen], sizeof(srcport));
331 	datalen += sizeof(srcport);
332 	bcopy(&dstport, &data[datalen], sizeof(dstport));
333 	datalen += sizeof(dstport);
334 	return (rss_hash(datalen, data));
335 }
336 
337 #ifdef INET6
338 /*
339  * Hash an IPv6 2-tuple.
340  */
341 uint32_t
342 rss_hash_ip6_2tuple(struct in6_addr src, struct in6_addr dst)
343 {
344 	uint8_t data[sizeof(src) + sizeof(dst)];
345 	u_int datalen;
346 
347 	datalen = 0;
348 	bcopy(&src, &data[datalen], sizeof(src));
349 	datalen += sizeof(src);
350 	bcopy(&dst, &data[datalen], sizeof(dst));
351 	datalen += sizeof(dst);
352 	return (rss_hash(datalen, data));
353 }
354 
355 /*
356  * Hash an IPv6 4-tuple.
357  */
358 uint32_t
359 rss_hash_ip6_4tuple(struct in6_addr src, u_short srcport,
360     struct in6_addr dst, u_short dstport)
361 {
362 	uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) +
363 	    sizeof(dstport)];
364 	u_int datalen;
365 
366 	datalen = 0;
367 	bcopy(&src, &data[datalen], sizeof(src));
368 	datalen += sizeof(src);
369 	bcopy(&dst, &data[datalen], sizeof(dst));
370 	datalen += sizeof(dst);
371 	bcopy(&srcport, &data[datalen], sizeof(srcport));
372 	datalen += sizeof(srcport);
373 	bcopy(&dstport, &data[datalen], sizeof(dstport));
374 	datalen += sizeof(dstport);
375 	return (rss_hash(datalen, data));
376 }
377 #endif /* INET6 */
378 
379 /*
380  * Query the number of RSS bits in use.
381  */
382 u_int
383 rss_getbits(void)
384 {
385 
386 	return (rss_bits);
387 }
388 
389 /*
390  * Query the RSS bucket associated with an RSS hash.
391  */
392 u_int
393 rss_getbucket(u_int hash)
394 {
395 
396 	return (hash & rss_mask);
397 }
398 
399 /*
400  * Query the RSS layer bucket associated with the given
401  * entry in the RSS hash space.
402  *
403  * The RSS indirection table is 0 .. rss_buckets-1,
404  * covering the low 'rss_bits' of the total 128 slot
405  * RSS indirection table.  So just mask off rss_bits and
406  * return that.
407  *
408  * NIC drivers can then iterate over the 128 slot RSS
409  * indirection table and fetch which RSS bucket to
410  * map it to.  This will typically be a CPU queue
411  */
412 u_int
413 rss_get_indirection_to_bucket(u_int index)
414 {
415 
416 	return (index & rss_mask);
417 }
418 
419 /*
420  * Query the RSS CPU associated with an RSS bucket.
421  */
422 u_int
423 rss_getcpu(u_int bucket)
424 {
425 
426 	return (rss_table[bucket].rte_cpu);
427 }
428 
429 /*
430  * netisr CPU affinity lookup given just the hash and hashtype.
431  */
432 u_int
433 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
434 {
435 
436 	switch (hash_type) {
437 	case M_HASHTYPE_RSS_IPV4:
438 	case M_HASHTYPE_RSS_TCP_IPV4:
439 	case M_HASHTYPE_RSS_UDP_IPV4:
440 	case M_HASHTYPE_RSS_IPV6:
441 	case M_HASHTYPE_RSS_TCP_IPV6:
442 	case M_HASHTYPE_RSS_UDP_IPV6:
443 		return (rss_getcpu(rss_getbucket(hash_val)));
444 	default:
445 		return (NETISR_CPUID_NONE);
446 	}
447 }
448 
449 /*
450  * Query the RSS bucket associated with the given hash value and
451  * type.
452  */
453 int
454 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
455 {
456 
457 	switch (hash_type) {
458 	case M_HASHTYPE_RSS_IPV4:
459 	case M_HASHTYPE_RSS_TCP_IPV4:
460 	case M_HASHTYPE_RSS_UDP_IPV4:
461 	case M_HASHTYPE_RSS_IPV6:
462 	case M_HASHTYPE_RSS_TCP_IPV6:
463 	case M_HASHTYPE_RSS_UDP_IPV6:
464 		*bucket_id = rss_getbucket(hash_val);
465 		return (0);
466 	default:
467 		return (-1);
468 	}
469 }
470 
471 /*
472  * netisr CPU affinity lookup routine for use by protocols.
473  */
474 struct mbuf *
475 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
476 {
477 
478 	M_ASSERTPKTHDR(m);
479 	*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
480 	return (m);
481 }
482 
483 int
484 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
485 {
486 
487 	M_ASSERTPKTHDR(m);
488 
489 	return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
490 	    bucket_id));
491 }
492 
493 /*
494  * Query the RSS hash algorithm.
495  */
496 u_int
497 rss_gethashalgo(void)
498 {
499 
500 	return (rss_hashalgo);
501 }
502 
503 /*
504  * Query the current RSS key; likely to be used by device drivers when
505  * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
506  *
507  * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
508  */
509 void
510 rss_getkey(uint8_t *key)
511 {
512 
513 	bcopy(rss_key, key, sizeof(rss_key));
514 }
515 
516 /*
517  * Query the number of buckets; this may be used by both network device
518  * drivers, which will need to populate hardware shadows of the software
519  * indirection table, and the network stack itself (such as when deciding how
520  * many connection groups to allocate).
521  */
522 u_int
523 rss_getnumbuckets(void)
524 {
525 
526 	return (rss_buckets);
527 }
528 
529 /*
530  * Query the number of CPUs in use by RSS; may be useful to device drivers
531  * trying to figure out how to map a larger number of CPUs into a smaller
532  * number of receive queues.
533  */
534 u_int
535 rss_getnumcpus(void)
536 {
537 
538 	return (rss_ncpus);
539 }
540 
541 /*
542  * Return the supported RSS hash configuration.
543  *
544  * NICs should query this to determine what to configure in their redirection
545  * matching table.
546  */
547 u_int
548 rss_gethashconfig(void)
549 {
550 	/* Return 4-tuple for TCP; 2-tuple for others */
551 	/*
552 	 * UDP may fragment more often than TCP and thus we'll end up with
553 	 * NICs returning 2-tuple fragments.
554 	 * udp_init() and udplite_init() both currently initialise things
555 	 * as 2-tuple.
556 	 * So for now disable UDP 4-tuple hashing until all of the other
557 	 * pieces are in place.
558 	 */
559 	return (
560 	    RSS_HASHTYPE_RSS_IPV4
561 	|    RSS_HASHTYPE_RSS_TCP_IPV4
562 	|    RSS_HASHTYPE_RSS_IPV6
563 	|    RSS_HASHTYPE_RSS_TCP_IPV6
564 	|    RSS_HASHTYPE_RSS_IPV6_EX
565 	|    RSS_HASHTYPE_RSS_TCP_IPV6_EX
566 #if 0
567 	|    RSS_HASHTYPE_RSS_UDP_IPV4
568 	|    RSS_HASHTYPE_RSS_UDP_IPV4_EX
569 	|    RSS_HASHTYPE_RSS_UDP_IPV6
570 	|    RSS_HASHTYPE_RSS_UDP_IPV6_EX
571 #endif
572 	);
573 }
574 
575 /*
576  * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
577  * it appearing in debugging output unnecessarily.
578  */
579 static int
580 sysctl_rss_key(SYSCTL_HANDLER_ARGS)
581 {
582 	uint8_t temp_rss_key[RSS_KEYSIZE];
583 	int error;
584 
585 	error = priv_check(req->td, PRIV_NETINET_HASHKEY);
586 	if (error)
587 		return (error);
588 
589 	bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
590 	error = sysctl_handle_opaque(oidp, temp_rss_key,
591 	    sizeof(temp_rss_key), req);
592 	if (error)
593 		return (error);
594 	if (req->newptr != NULL) {
595 		/* XXXRW: Not yet. */
596 		return (EINVAL);
597 	}
598 	return (0);
599 }
600 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
601     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
602     "", "RSS keying material");
603 
604 static int
605 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
606 {
607 	struct sbuf *sb;
608 	int error;
609 	int i;
610 
611 	error = 0;
612 	error = sysctl_wire_old_buffer(req, 0);
613 	if (error != 0)
614 		return (error);
615 	sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
616 	if (sb == NULL)
617 		return (ENOMEM);
618 	for (i = 0; i < rss_buckets; i++) {
619 		sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
620 		    i,
621 		    rss_getcpu(i));
622 	}
623 	error = sbuf_finish(sb);
624 	sbuf_delete(sb);
625 
626 	return (error);
627 }
628 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
629     CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
630     sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
631