xref: /freebsd/sys/netinet/in_rss.c (revision cc3f4b99653c34ae64f8a1fddea370abefef680e)
1 /*-
2  * Copyright (c) 2010-2011 Juniper Networks, Inc.
3  * All rights reserved.
4  *
5  * This software was developed by Robert N. M. Watson under contract
6  * to Juniper Networks, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_inet6.h"
35 #include "opt_pcbgroup.h"
36 
37 #ifndef PCBGROUP
38 #error "options RSS depends on options PCBGROUP"
39 #endif
40 
41 #include <sys/param.h>
42 #include <sys/mbuf.h>
43 #include <sys/socket.h>
44 #include <sys/priv.h>
45 #include <sys/kernel.h>
46 #include <sys/smp.h>
47 #include <sys/sysctl.h>
48 
49 #include <net/if.h>
50 #include <net/if_var.h>
51 #include <net/netisr.h>
52 
53 #include <netinet/in.h>
54 #include <netinet/in_pcb.h>
55 #include <netinet/in_rss.h>
56 #include <netinet/in_var.h>
57 #include <netinet/toeplitz.h>
58 
59 /*-
60  * Operating system parts of receiver-side scaling (RSS), which allows
61  * network cards to direct flows to particular receive queues based on hashes
62  * of header tuples.  This implementation aligns RSS buckets with connection
63  * groups at the TCP/IP layer, so each bucket is associated with exactly one
64  * group.  As a result, the group lookup structures (and lock) should have an
65  * effective affinity with exactly one CPU.
66  *
67  * Network device drivers needing to configure RSS will query this framework
68  * for parameters, such as the current RSS key, hashing policies, number of
69  * bits, and indirection table mapping hashes to buckets and CPUs.  They may
70  * provide their own supplementary information, such as queue<->CPU bindings.
71  * It is the responsibility of the network device driver to inject packets
72  * into the stack on as close to the right CPU as possible, if playing by RSS
73  * rules.
74  *
75  * TODO:
76  *
77  * - Synchronization for rss_key and other future-configurable parameters.
78  * - Event handler drivers can register to pick up RSS configuration changes.
79  * - Should we allow rss_basecpu to be configured?
80  * - Randomize key on boot.
81  * - IPv6 support.
82  * - Statistics on how often there's a misalignment between hardware
83  *   placement and pcbgroup expectations.
84  */
85 
86 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
87 
88 /*
89  * Toeplitz is the only required hash function in the RSS spec, so use it by
90  * default.
91  */
92 static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
93 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RD, &rss_hashalgo, 0,
94     "RSS hash algorithm");
95 TUNABLE_INT("net.inet.rss.hashalgo", &rss_hashalgo);
96 
97 /*
98  * Size of the indirection table; at most 128 entries per the RSS spec.  We
99  * size it to at least 2 times the number of CPUs by default to allow useful
100  * rebalancing.  If not set explicitly with a loader tunable, we tune based
101  * on the number of CPUs present.
102  *
103  * XXXRW: buckets might be better to use for the tunable than bits.
104  */
105 static u_int	rss_bits;
106 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RD, &rss_bits, 0,
107     "RSS bits");
108 TUNABLE_INT("net.inet.rss.bits", &rss_bits);
109 
110 static u_int	rss_mask;
111 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
112     "RSS mask");
113 
114 static const u_int	rss_maxbits = RSS_MAXBITS;
115 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
116     __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
117 
118 /*
119  * RSS's own count of the number of CPUs it could be using for processing.
120  * Bounded to 64 by RSS constants.
121  */
122 static u_int	rss_ncpus;
123 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
124     "Number of CPUs available to RSS");
125 
126 #define	RSS_MAXCPUS	(1 << (RSS_MAXBITS - 1))
127 static const u_int	rss_maxcpus = RSS_MAXCPUS;
128 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
129     __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
130 
131 /*
132  * Variable exists just for reporting rss_bits in a user-friendly way.
133  */
134 static u_int	rss_buckets;
135 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
136     "RSS buckets");
137 
138 /*
139  * Base CPU number; devices will add this to all CPU numbers returned by the
140  * RSS indirection table.  Currently unmodifable in FreeBSD.
141  */
142 static const u_int	rss_basecpu;
143 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
144     __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
145 
146 /*
147  * RSS secret key, intended to prevent attacks on load-balancing.  Its
148  * effectiveness may be limited by algorithm choice and available entropy
149  * during the boot.
150  *
151  * XXXRW: And that we don't randomize it yet!
152  *
153  * XXXRW: This default is actually the default key from Chelsio T3 cards, as
154  * it offers reasonable distribution, unlike all-0 keys which always
155  * generate a hash of 0 (upsettingly).
156  */
157 static uint8_t	rss_key[RSS_KEYSIZE] = {
158 	0x43, 0xa3, 0x8f, 0xb0, 0x41, 0x67, 0x25, 0x3d,
159 	0x25, 0x5b, 0x0e, 0xc2, 0x6d, 0x5a, 0x56, 0xda,
160 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
161 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
162 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
163 };
164 
165 /*
166  * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
167  * Drivers may supplement this table with a seperate CPU<->queue table when
168  * programming devices.
169  */
170 struct rss_table_entry {
171 	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
172 };
173 static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
174 
175 static void
176 rss_init(__unused void *arg)
177 {
178 	u_int i;
179 
180 	/*
181 	 * Validate tunables, coerce to sensible values.
182 	 */
183 	switch (rss_hashalgo) {
184 	case RSS_HASH_TOEPLITZ:
185 	case RSS_HASH_NAIVE:
186 		break;
187 
188 	default:
189 		printf("%s: invalid RSS hashalgo %u, coercing to %u",
190 		    __func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
191 		rss_hashalgo = RSS_HASH_TOEPLITZ;
192 	}
193 
194 	/*
195 	 * Count available CPUs.
196 	 *
197 	 * XXXRW: Note incorrect assumptions regarding contiguity of this set
198 	 * elsewhere.
199 	 */
200 	rss_ncpus = 0;
201 	for (i = 0; i <= mp_maxid; i++) {
202 		if (CPU_ABSENT(i))
203 			continue;
204 		rss_ncpus++;
205 	}
206 	if (rss_ncpus > RSS_MAXCPUS)
207 		rss_ncpus = RSS_MAXCPUS;
208 
209 	/*
210 	 * Tune RSS table entries to be no less than 2x the number of CPUs
211 	 * -- unless we're running uniprocessor, in which case there's not
212 	 * much point in having buckets to rearrange for load-balancing!
213 	 */
214 	if (rss_ncpus > 1) {
215 		if (rss_bits == 0)
216 			rss_bits = fls(rss_ncpus - 1) + 1;
217 
218 		/*
219 		 * Microsoft limits RSS table entries to 128, so apply that
220 		 * limit to both auto-detected CPU counts and user-configured
221 		 * ones.
222 		 */
223 		if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
224 			printf("%s: RSS bits %u not valid, coercing to  %u",
225 			    __func__, rss_bits, RSS_MAXBITS);
226 			rss_bits = RSS_MAXBITS;
227 		}
228 
229 		/*
230 		 * Figure out how many buckets to use; warn if less than the
231 		 * number of configured CPUs, although this is not a fatal
232 		 * problem.
233 		 */
234 		rss_buckets = (1 << rss_bits);
235 		if (rss_buckets < rss_ncpus)
236 			printf("%s: WARNING: rss_buckets (%u) less than "
237 			    "rss_ncpus (%u)\n", __func__, rss_buckets,
238 			    rss_ncpus);
239 		rss_mask = rss_buckets - 1;
240 	} else {
241 		rss_bits = 0;
242 		rss_buckets = 1;
243 		rss_mask = 0;
244 	}
245 
246 	/*
247 	 * Set up initial CPU assignments: round-robin by default.
248 	 *
249 	 * XXXRW: Need a mapping to non-contiguous IDs here.
250 	 */
251 	for (i = 0; i < rss_buckets; i++)
252 		rss_table[i].rte_cpu = i % rss_ncpus;
253 
254 	/*
255 	 * Randomize rrs_key.
256 	 *
257 	 * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
258 	 * loop to check for "bad" RSS keys.
259 	 */
260 }
261 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
262 
263 static uint32_t
264 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
265     const uint8_t *data)
266 {
267 	uint32_t v;
268 	u_int i;
269 
270 	v = 0;
271 	for (i = 0; i < keylen; i++)
272 		v += key[i];
273 	for (i = 0; i < datalen; i++)
274 		v += data[i];
275 	return (v);
276 }
277 
278 static uint32_t
279 rss_hash(u_int datalen, const uint8_t *data)
280 {
281 
282 	switch (rss_hashalgo) {
283 	case RSS_HASH_TOEPLITZ:
284 		return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
285 		    data));
286 
287 	case RSS_HASH_NAIVE:
288 		return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
289 		    data));
290 
291 	default:
292 		panic("%s: unsupported/unknown hashalgo %d", __func__,
293 		    rss_hashalgo);
294 	}
295 }
296 
297 /*
298  * Hash an IPv4 2-tuple.
299  */
300 uint32_t
301 rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst)
302 {
303 	uint8_t data[sizeof(src) + sizeof(dst)];
304 	u_int datalen;
305 
306 	datalen = 0;
307 	bcopy(&src, &data[datalen], sizeof(src));
308 	datalen += sizeof(src);
309 	bcopy(&dst, &data[datalen], sizeof(dst));
310 	datalen += sizeof(dst);
311 	return (rss_hash(datalen, data));
312 }
313 
314 /*
315  * Hash an IPv4 4-tuple.
316  */
317 uint32_t
318 rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst,
319     u_short dstport)
320 {
321 	uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) +
322 	    sizeof(dstport)];
323 	u_int datalen;
324 
325 	datalen = 0;
326 	bcopy(&src, &data[datalen], sizeof(src));
327 	datalen += sizeof(src);
328 	bcopy(&dst, &data[datalen], sizeof(dst));
329 	datalen += sizeof(dst);
330 	bcopy(&srcport, &data[datalen], sizeof(srcport));
331 	datalen += sizeof(srcport);
332 	bcopy(&dstport, &data[datalen], sizeof(dstport));
333 	datalen += sizeof(dstport);
334 	return (rss_hash(datalen, data));
335 }
336 
337 #ifdef INET6
338 /*
339  * Hash an IPv6 2-tuple.
340  */
341 uint32_t
342 rss_hash_ip6_2tuple(struct in6_addr src, struct in6_addr dst)
343 {
344 	uint8_t data[sizeof(src) + sizeof(dst)];
345 	u_int datalen;
346 
347 	datalen = 0;
348 	bcopy(&src, &data[datalen], sizeof(src));
349 	datalen += sizeof(src);
350 	bcopy(&dst, &data[datalen], sizeof(dst));
351 	datalen += sizeof(dst);
352 	return (rss_hash(datalen, data));
353 }
354 
355 /*
356  * Hash an IPv6 4-tuple.
357  */
358 uint32_t
359 rss_hash_ip6_4tuple(struct in6_addr src, u_short srcport,
360     struct in6_addr dst, u_short dstport)
361 {
362 	uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) +
363 	    sizeof(dstport)];
364 	u_int datalen;
365 
366 	datalen = 0;
367 	bcopy(&src, &data[datalen], sizeof(src));
368 	datalen += sizeof(src);
369 	bcopy(&dst, &data[datalen], sizeof(dst));
370 	datalen += sizeof(dst);
371 	bcopy(&srcport, &data[datalen], sizeof(srcport));
372 	datalen += sizeof(srcport);
373 	bcopy(&dstport, &data[datalen], sizeof(dstport));
374 	datalen += sizeof(dstport);
375 	return (rss_hash(datalen, data));
376 }
377 #endif /* INET6 */
378 
379 /*
380  * Query the number of RSS bits in use.
381  */
382 u_int
383 rss_getbits(void)
384 {
385 
386 	return (rss_bits);
387 }
388 
389 /*
390  * Query the RSS bucket associated with an RSS hash.
391  */
392 u_int
393 rss_getbucket(u_int hash)
394 {
395 
396 	return (hash & rss_mask);
397 }
398 
399 /*
400  * Query the RSS CPU associated with an RSS bucket.
401  */
402 u_int
403 rss_getcpu(u_int bucket)
404 {
405 
406 	return (rss_table[bucket].rte_cpu);
407 }
408 
409 /*
410  * netisr CPU affinity lookup routine for use by protocols.
411  */
412 struct mbuf *
413 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
414 {
415 
416 	M_ASSERTPKTHDR(m);
417 
418 	switch (M_HASHTYPE_GET(m)) {
419 	case M_HASHTYPE_RSS_IPV4:
420 	case M_HASHTYPE_RSS_TCP_IPV4:
421 		*cpuid = rss_getcpu(rss_getbucket(m->m_pkthdr.flowid));
422 		return (m);
423 
424 	default:
425 		*cpuid = NETISR_CPUID_NONE;
426 		return (m);
427 	}
428 }
429 
430 /*
431  * Query the RSS hash algorithm.
432  */
433 u_int
434 rss_gethashalgo(void)
435 {
436 
437 	return (rss_hashalgo);
438 }
439 
440 /*
441  * Query the current RSS key; likely to be used by device drivers when
442  * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
443  *
444  * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
445  */
446 void
447 rss_getkey(uint8_t *key)
448 {
449 
450 	bcopy(rss_key, key, sizeof(rss_key));
451 }
452 
453 /*
454  * Query the number of buckets; this may be used by both network device
455  * drivers, which will need to populate hardware shadows of the software
456  * indirection table, and the network stack itself (such as when deciding how
457  * many connection groups to allocate).
458  */
459 u_int
460 rss_getnumbuckets(void)
461 {
462 
463 	return (rss_buckets);
464 }
465 
466 /*
467  * Query the number of CPUs in use by RSS; may be useful to device drivers
468  * trying to figure out how to map a larger number of CPUs into a smaller
469  * number of receive queues.
470  */
471 u_int
472 rss_getnumcpus(void)
473 {
474 
475 	return (rss_ncpus);
476 }
477 
478 /*
479  * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
480  * it appearing in debugging output unnecessarily.
481  */
482 static int
483 sysctl_rss_key(SYSCTL_HANDLER_ARGS)
484 {
485 	uint8_t temp_rss_key[RSS_KEYSIZE];
486 	int error;
487 
488 	error = priv_check(req->td, PRIV_NETINET_HASHKEY);
489 	if (error)
490 		return (error);
491 
492 	bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
493 	error = sysctl_handle_opaque(oidp, temp_rss_key,
494 	    sizeof(temp_rss_key), req);
495 	if (error)
496 		return (error);
497 	if (req->newptr != NULL) {
498 		/* XXXRW: Not yet. */
499 		return (EINVAL);
500 	}
501 	return (0);
502 }
503 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
504     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
505     "", "RSS keying material");
506