1 /*-
2 * Copyright (c) 2010-2011 Juniper Networks, Inc.
3 * All rights reserved.
4 *
5 * This software was developed by Robert N. M. Watson under contract
6 * to Juniper Networks, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30
31 #include "opt_inet6.h"
32 #include "opt_inet.h"
33 #include "opt_rss.h"
34
35 #include <sys/param.h>
36 #include <sys/mbuf.h>
37 #include <sys/socket.h>
38 #include <sys/priv.h>
39 #include <sys/kernel.h>
40 #include <sys/smp.h>
41 #include <sys/sysctl.h>
42 #include <sys/sbuf.h>
43
44 #include <net/if.h>
45 #include <net/if_var.h>
46 #include <net/netisr.h>
47 #include <net/rss_config.h>
48 #include <net/toeplitz.h>
49
50 /*-
51 * Operating system parts of receiver-side scaling (RSS), which allows
52 * network cards to direct flows to particular receive queues based on hashes
53 * of header tuples. This implementation aligns RSS buckets with connection
54 * groups at the TCP/IP layer, so each bucket is associated with exactly one
55 * group. As a result, the group lookup structures (and lock) should have an
56 * effective affinity with exactly one CPU.
57 *
58 * Network device drivers needing to configure RSS will query this framework
59 * for parameters, such as the current RSS key, hashing policies, number of
60 * bits, and indirection table mapping hashes to buckets and CPUs. They may
61 * provide their own supplementary information, such as queue<->CPU bindings.
62 * It is the responsibility of the network device driver to inject packets
63 * into the stack on as close to the right CPU as possible, if playing by RSS
64 * rules.
65 *
66 * TODO:
67 *
68 * - Synchronization for rss_key and other future-configurable parameters.
69 * - Event handler drivers can register to pick up RSS configuration changes.
70 * - Should we allow rss_basecpu to be configured?
71 * - Randomize key on boot.
72 * - IPv6 support.
73 * - Statistics on how often there's a misalignment between hardware
74 * placement and pcbgroup expectations.
75 */
76
77 #if !defined(INET) && !defined(INET6)
78 #define _net_inet _net
79 #define _net_inet_rss _net_rss
80 #endif
81 SYSCTL_DECL(_net_inet);
82 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
83 "Receive-side steering");
84
85 /*
86 * Toeplitz is the only required hash function in the RSS spec, so use it by
87 * default.
88 */
89 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ;
90 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
91 "RSS hash algorithm");
92
93 #ifdef RSS
94 /*
95 * Size of the indirection table; at most 128 entries per the RSS spec. We
96 * size it to at least 2 times the number of CPUs by default to allow useful
97 * rebalancing. If not set explicitly with a loader tunable, we tune based
98 * on the number of CPUs present.
99 *
100 * XXXRW: buckets might be better to use for the tunable than bits.
101 */
102 static u_int rss_bits;
103 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
104 "RSS bits");
105
106 static u_int rss_mask;
107 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
108 "RSS mask");
109
110 static const u_int rss_maxbits = RSS_MAXBITS;
111 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
112 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
113
114 /*
115 * RSS's own count of the number of CPUs it could be using for processing.
116 * Bounded to 64 by RSS constants.
117 */
118 static u_int rss_ncpus;
119 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
120 "Number of CPUs available to RSS");
121
122 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1))
123 static const u_int rss_maxcpus = RSS_MAXCPUS;
124 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
125 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
126
127 /*
128 * Variable exists just for reporting rss_bits in a user-friendly way.
129 */
130 static u_int rss_buckets;
131 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
132 "RSS buckets");
133
134 /*
135 * Base CPU number; devices will add this to all CPU numbers returned by the
136 * RSS indirection table. Currently unmodifable in FreeBSD.
137 */
138 static const u_int rss_basecpu;
139 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
140 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
141
142 #endif
143 /*
144 * Print verbose debugging messages.
145 * 0 - disable
146 * non-zero - enable
147 */
148 int rss_debug = 0;
149 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0,
150 "RSS debug level");
151
152 /*
153 * RSS secret key, intended to prevent attacks on load-balancing. Its
154 * effectiveness may be limited by algorithm choice and available entropy
155 * during the boot.
156 *
157 * XXXRW: And that we don't randomize it yet!
158 *
159 * This is the default Microsoft RSS specification key which is also
160 * the Chelsio T5 firmware default key.
161 */
162 static uint8_t rss_key[RSS_KEYSIZE] = {
163 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
164 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
165 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
166 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
167 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
168 };
169
170 #ifdef RSS
171 /*
172 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
173 * Drivers may supplement this table with a separate CPU<->queue table when
174 * programming devices.
175 */
176 struct rss_table_entry {
177 uint8_t rte_cpu; /* CPU affinity of bucket. */
178 };
179 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
180 #endif
181
182 static void
rss_init(__unused void * arg)183 rss_init(__unused void *arg)
184 {
185 #ifdef RSS
186 u_int i;
187 u_int cpuid;
188 #endif
189 /*
190 * Validate tunables, coerce to sensible values.
191 */
192 switch (rss_hashalgo) {
193 case RSS_HASH_TOEPLITZ:
194 case RSS_HASH_NAIVE:
195 break;
196
197 default:
198 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n",
199 rss_hashalgo, RSS_HASH_TOEPLITZ);
200 rss_hashalgo = RSS_HASH_TOEPLITZ;
201 }
202
203 #ifdef RSS
204 /*
205 * Count available CPUs.
206 *
207 * XXXRW: Note incorrect assumptions regarding contiguity of this set
208 * elsewhere.
209 */
210 rss_ncpus = 0;
211 for (i = 0; i <= mp_maxid; i++) {
212 if (CPU_ABSENT(i))
213 continue;
214 rss_ncpus++;
215 }
216 if (rss_ncpus > RSS_MAXCPUS)
217 rss_ncpus = RSS_MAXCPUS;
218
219 /*
220 * Tune RSS table entries to be no less than 2x the number of CPUs
221 * -- unless we're running uniprocessor, in which case there's not
222 * much point in having buckets to rearrange for load-balancing!
223 */
224 if (rss_ncpus > 1) {
225 if (rss_bits == 0)
226 rss_bits = fls(rss_ncpus - 1) + 1;
227
228 /*
229 * Microsoft limits RSS table entries to 128, so apply that
230 * limit to both auto-detected CPU counts and user-configured
231 * ones.
232 */
233 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
234 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n",
235 rss_bits, RSS_MAXBITS);
236 rss_bits = RSS_MAXBITS;
237 }
238
239 /*
240 * Figure out how many buckets to use; warn if less than the
241 * number of configured CPUs, although this is not a fatal
242 * problem.
243 */
244 rss_buckets = (1 << rss_bits);
245 if (rss_buckets < rss_ncpus)
246 RSS_DEBUG("WARNING: rss_buckets (%u) less than "
247 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus);
248 rss_mask = rss_buckets - 1;
249 } else {
250 rss_bits = 0;
251 rss_buckets = 1;
252 rss_mask = 0;
253 }
254
255 /*
256 * Set up initial CPU assignments: round-robin by default.
257 */
258 cpuid = CPU_FIRST();
259 for (i = 0; i < rss_buckets; i++) {
260 rss_table[i].rte_cpu = cpuid;
261 cpuid = CPU_NEXT(cpuid);
262 }
263 #endif /* RSS */
264 /*
265 * Randomize rrs_key.
266 *
267 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey()
268 * loop to check for "bad" RSS keys.
269 */
270 }
271 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
272
273 static uint32_t
rss_naive_hash(u_int keylen,const uint8_t * key,u_int datalen,const uint8_t * data)274 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
275 const uint8_t *data)
276 {
277 uint32_t v;
278 u_int i;
279
280 v = 0;
281 for (i = 0; i < keylen; i++)
282 v += key[i];
283 for (i = 0; i < datalen; i++)
284 v += data[i];
285 return (v);
286 }
287
288 uint32_t
rss_hash(u_int datalen,const uint8_t * data)289 rss_hash(u_int datalen, const uint8_t *data)
290 {
291
292 switch (rss_hashalgo) {
293 case RSS_HASH_TOEPLITZ:
294 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
295 data));
296
297 case RSS_HASH_NAIVE:
298 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
299 data));
300
301 default:
302 panic("%s: unsupported/unknown hashalgo %d", __func__,
303 rss_hashalgo);
304 }
305 }
306
307 /*
308 * Query the current RSS key; likely to be used by device drivers when
309 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE.
310 *
311 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
312 */
313 void
rss_getkey(uint8_t * key)314 rss_getkey(uint8_t *key)
315 {
316
317 bcopy(rss_key, key, sizeof(rss_key));
318 }
319
320 /*
321 * Query the RSS hash algorithm.
322 */
323 u_int
rss_gethashalgo(void)324 rss_gethashalgo(void)
325 {
326
327 return (rss_hashalgo);
328 }
329
330 #ifdef RSS
331 /*
332 * Query the number of RSS bits in use.
333 */
334 u_int
rss_getbits(void)335 rss_getbits(void)
336 {
337
338 return (rss_bits);
339 }
340
341 /*
342 * Query the RSS bucket associated with an RSS hash.
343 */
344 u_int
rss_getbucket(u_int hash)345 rss_getbucket(u_int hash)
346 {
347
348 return (hash & rss_mask);
349 }
350
351 /*
352 * Query the RSS layer bucket associated with the given
353 * entry in the RSS hash space.
354 *
355 * The RSS indirection table is 0 .. rss_buckets-1,
356 * covering the low 'rss_bits' of the total 128 slot
357 * RSS indirection table. So just mask off rss_bits and
358 * return that.
359 *
360 * NIC drivers can then iterate over the 128 slot RSS
361 * indirection table and fetch which RSS bucket to
362 * map it to. This will typically be a CPU queue
363 */
364 u_int
rss_get_indirection_to_bucket(u_int index)365 rss_get_indirection_to_bucket(u_int index)
366 {
367
368 return (index & rss_mask);
369 }
370
371 /*
372 * Query the RSS CPU associated with an RSS bucket.
373 */
374 u_int
rss_getcpu(u_int bucket)375 rss_getcpu(u_int bucket)
376 {
377
378 return (rss_table[bucket].rte_cpu);
379 }
380
381 /*
382 * netisr CPU affinity lookup given just the hash and hashtype.
383 */
384 u_int
rss_hash2cpuid(uint32_t hash_val,uint32_t hash_type)385 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
386 {
387
388 switch (hash_type) {
389 case M_HASHTYPE_RSS_IPV4:
390 case M_HASHTYPE_RSS_TCP_IPV4:
391 case M_HASHTYPE_RSS_UDP_IPV4:
392 case M_HASHTYPE_RSS_IPV6:
393 case M_HASHTYPE_RSS_TCP_IPV6:
394 case M_HASHTYPE_RSS_UDP_IPV6:
395 return (rss_getcpu(rss_getbucket(hash_val)));
396 default:
397 return (NETISR_CPUID_NONE);
398 }
399 }
400
401 /*
402 * Query the RSS bucket associated with the given hash value and
403 * type.
404 */
405 int
rss_hash2bucket(uint32_t hash_val,uint32_t hash_type,uint32_t * bucket_id)406 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
407 {
408
409 switch (hash_type) {
410 case M_HASHTYPE_RSS_IPV4:
411 case M_HASHTYPE_RSS_TCP_IPV4:
412 case M_HASHTYPE_RSS_UDP_IPV4:
413 case M_HASHTYPE_RSS_IPV6:
414 case M_HASHTYPE_RSS_TCP_IPV6:
415 case M_HASHTYPE_RSS_UDP_IPV6:
416 *bucket_id = rss_getbucket(hash_val);
417 return (0);
418 default:
419 return (-1);
420 }
421 }
422
423 /*
424 * netisr CPU affinity lookup routine for use by protocols.
425 */
426 struct mbuf *
rss_m2cpuid(struct mbuf * m,uintptr_t source,u_int * cpuid)427 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
428 {
429
430 M_ASSERTPKTHDR(m);
431 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
432 return (m);
433 }
434
435 int
rss_m2bucket(struct mbuf * m,uint32_t * bucket_id)436 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
437 {
438
439 M_ASSERTPKTHDR(m);
440
441 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
442 bucket_id));
443 }
444
445 /*
446 * Query the number of buckets; this may be used by both network device
447 * drivers, which will need to populate hardware shadows of the software
448 * indirection table, and the network stack itself (such as when deciding how
449 * many connection groups to allocate).
450 */
451 u_int
rss_getnumbuckets(void)452 rss_getnumbuckets(void)
453 {
454
455 return (rss_buckets);
456 }
457
458 /*
459 * Query the number of CPUs in use by RSS; may be useful to device drivers
460 * trying to figure out how to map a larger number of CPUs into a smaller
461 * number of receive queues.
462 */
463 u_int
rss_getnumcpus(void)464 rss_getnumcpus(void)
465 {
466
467 return (rss_ncpus);
468 }
469
470 #endif
471 /*
472 * Return the supported RSS hash configuration.
473 *
474 * NICs should query this to determine what to configure in their redirection
475 * matching table.
476 */
477 inline u_int
rss_gethashconfig(void)478 rss_gethashconfig(void)
479 {
480
481 /* Return 4-tuple for TCP; 2-tuple for others */
482 /*
483 * UDP may fragment more often than TCP and thus we'll end up with
484 * NICs returning 2-tuple fragments.
485 * udp_init() and udplite_init() both currently initialise things
486 * as 2-tuple.
487 * So for now disable UDP 4-tuple hashing until all of the other
488 * pieces are in place.
489 */
490 return (
491 RSS_HASHTYPE_RSS_IPV4
492 | RSS_HASHTYPE_RSS_TCP_IPV4
493 | RSS_HASHTYPE_RSS_IPV6
494 | RSS_HASHTYPE_RSS_TCP_IPV6
495 | RSS_HASHTYPE_RSS_IPV6_EX
496 | RSS_HASHTYPE_RSS_TCP_IPV6_EX
497 #if 0
498 | RSS_HASHTYPE_RSS_UDP_IPV4
499 | RSS_HASHTYPE_RSS_UDP_IPV6
500 | RSS_HASHTYPE_RSS_UDP_IPV6_EX
501 #endif
502 );
503 }
504
505 /*
506 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
507 * it appearing in debugging output unnecessarily.
508 */
509 static int
sysctl_rss_key(SYSCTL_HANDLER_ARGS)510 sysctl_rss_key(SYSCTL_HANDLER_ARGS)
511 {
512 uint8_t temp_rss_key[RSS_KEYSIZE];
513 int error;
514
515 error = priv_check(req->td, PRIV_NETINET_HASHKEY);
516 if (error)
517 return (error);
518
519 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
520 error = sysctl_handle_opaque(oidp, temp_rss_key,
521 sizeof(temp_rss_key), req);
522 if (error)
523 return (error);
524 if (req->newptr != NULL) {
525 /* XXXRW: Not yet. */
526 return (EINVAL);
527 }
528 return (0);
529 }
530 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
531 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
532 "", "RSS keying material");
533
534 #ifdef RSS
535 static int
sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)536 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
537 {
538 struct sbuf *sb;
539 int error;
540 int i;
541
542 error = 0;
543 error = sysctl_wire_old_buffer(req, 0);
544 if (error != 0)
545 return (error);
546 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
547 if (sb == NULL)
548 return (ENOMEM);
549 for (i = 0; i < rss_buckets; i++) {
550 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
551 i,
552 rss_getcpu(i));
553 }
554 error = sbuf_finish(sb);
555 sbuf_delete(sb);
556
557 return (error);
558 }
559 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
560 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
561 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
562 #endif
563