1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Authors:
4 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se>
5 * Uppsala University and
6 * Swedish University of Agricultural Sciences
7 *
8 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
9 * Ben Greear <greearb@candelatech.com>
10 * Jens Låås <jens.laas@data.slu.se>
11 *
12 * A tool for loading the network with preconfigurated packets.
13 * The tool is implemented as a linux module. Parameters are output
14 * device, delay (to hard_xmit), number of packets, and whether
15 * to use multiple SKBs or just the same one.
16 * pktgen uses the installed interface's output routine.
17 *
18 * Additional hacking by:
19 *
20 * Jens.Laas@data.slu.se
21 * Improved by ANK. 010120.
22 * Improved by ANK even more. 010212.
23 * MAC address typo fixed. 010417 --ro
24 * Integrated. 020301 --DaveM
25 * Added multiskb option 020301 --DaveM
26 * Scaling of results. 020417--sigurdur@linpro.no
27 * Significant re-work of the module:
28 * * Convert to threaded model to more efficiently be able to transmit
29 * and receive on multiple interfaces at once.
30 * * Converted many counters to __u64 to allow longer runs.
31 * * Allow configuration of ranges, like min/max IP address, MACs,
32 * and UDP-ports, for both source and destination, and can
33 * set to use a random distribution or sequentially walk the range.
34 * * Can now change most values after starting.
35 * * Place 12-byte packet in UDP payload with magic number,
36 * sequence number, and timestamp.
37 * * Add receiver code that detects dropped pkts, re-ordered pkts, and
38 * latencies (with micro-second) precision.
39 * * Add IOCTL interface to easily get counters & configuration.
40 * --Ben Greear <greearb@candelatech.com>
41 *
42 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct
43 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0
44 * as a "fastpath" with a configurable number of clones after alloc's.
45 * clone_skb=0 means all packets are allocated this also means ranges time
46 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100
47 * clones.
48 *
49 * Also moved to /proc/net/pktgen/
50 * --ro
51 *
52 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever
53 * mistakes. Also merged in DaveM's patch in the -pre6 patch.
54 * --Ben Greear <greearb@candelatech.com>
55 *
56 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br)
57 *
58 * 021124 Finished major redesign and rewrite for new functionality.
59 * See Documentation/networking/pktgen.rst for how to use this.
60 *
61 * The new operation:
62 * For each CPU one thread/process is created at start. This process checks
63 * for running devices in the if_list and sends packets until count is 0 it
64 * also the thread checks the thread->control which is used for inter-process
65 * communication. controlling process "posts" operations to the threads this
66 * way.
67 * The if_list is RCU protected, and the if_lock remains to protect updating
68 * of if_list, from "add_device" as it invoked from userspace (via proc write).
69 *
70 * By design there should only be *one* "controlling" process. In practice
71 * multiple write accesses gives unpredictable result. Understood by "write"
72 * to /proc gives result code that should be read be the "writer".
73 * For practical use this should be no problem.
74 *
75 * Note when adding devices to a specific CPU there good idea to also assign
76 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU.
77 * --ro
78 *
79 * Fix refcount off by one if first packet fails, potential null deref,
80 * memleak 030710- KJP
81 *
82 * First "ranges" functionality for ipv6 030726 --ro
83 *
84 * Included flow support. 030802 ANK.
85 *
86 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org>
87 *
88 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419
89 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604
90 *
91 * New xmit() return, do_div and misc clean up by Stephen Hemminger
92 * <shemminger@osdl.org> 040923
93 *
94 * Randy Dunlap fixed u64 printk compiler warning
95 *
96 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org>
97 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213
98 *
99 * Corrections from Nikolai Malykh (nmalykh@bilim.com)
100 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230
101 *
102 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com>
103 * 050103
104 *
105 * MPLS support by Steven Whitehouse <steve@chygwyn.com>
106 *
107 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com>
108 *
109 * Fixed src_mac command to set source mac of packet to value specified in
110 * command by Adit Ranadive <adit.262@gmail.com>
111 */
112
113 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
114
115 #include <linux/sys.h>
116 #include <linux/types.h>
117 #include <linux/module.h>
118 #include <linux/moduleparam.h>
119 #include <linux/kernel.h>
120 #include <linux/mutex.h>
121 #include <linux/sched.h>
122 #include <linux/slab.h>
123 #include <linux/vmalloc.h>
124 #include <linux/unistd.h>
125 #include <linux/string.h>
126 #include <linux/ptrace.h>
127 #include <linux/errno.h>
128 #include <linux/ioport.h>
129 #include <linux/interrupt.h>
130 #include <linux/capability.h>
131 #include <linux/hrtimer.h>
132 #include <linux/freezer.h>
133 #include <linux/delay.h>
134 #include <linux/timer.h>
135 #include <linux/list.h>
136 #include <linux/init.h>
137 #include <linux/skbuff.h>
138 #include <linux/netdevice.h>
139 #include <linux/inet.h>
140 #include <linux/inetdevice.h>
141 #include <linux/rtnetlink.h>
142 #include <linux/if_arp.h>
143 #include <linux/if_vlan.h>
144 #include <linux/in.h>
145 #include <linux/ip.h>
146 #include <linux/ipv6.h>
147 #include <linux/udp.h>
148 #include <linux/proc_fs.h>
149 #include <linux/seq_file.h>
150 #include <linux/wait.h>
151 #include <linux/etherdevice.h>
152 #include <linux/kthread.h>
153 #include <linux/prefetch.h>
154 #include <linux/mmzone.h>
155 #include <net/net_namespace.h>
156 #include <net/checksum.h>
157 #include <net/ipv6.h>
158 #include <net/udp.h>
159 #include <net/ip6_checksum.h>
160 #include <net/addrconf.h>
161 #include <net/xfrm.h>
162 #include <net/netns/generic.h>
163 #include <asm/byteorder.h>
164 #include <linux/rcupdate.h>
165 #include <linux/bitops.h>
166 #include <linux/io.h>
167 #include <linux/timex.h>
168 #include <linux/uaccess.h>
169 #include <asm/dma.h>
170 #include <asm/div64.h> /* do_div */
171
172 #define VERSION "2.75"
173 #define IP_NAME_SZ 32
174 #define MAX_MPLS_LABELS 16 /* This is the max label stack depth */
175 #define MPLS_STACK_BOTTOM htonl(0x00000100)
176 /* Max number of internet mix entries that can be specified in imix_weights. */
177 #define MAX_IMIX_ENTRIES 20
178 #define IMIX_PRECISION 100 /* Precision of IMIX distribution */
179
180 #define func_enter() pr_debug("entering %s\n", __func__);
181
182 #define PKT_FLAGS \
183 pf(IPV6) /* Interface in IPV6 Mode */ \
184 pf(IPSRC_RND) /* IP-Src Random */ \
185 pf(IPDST_RND) /* IP-Dst Random */ \
186 pf(TXSIZE_RND) /* Transmit size is random */ \
187 pf(UDPSRC_RND) /* UDP-Src Random */ \
188 pf(UDPDST_RND) /* UDP-Dst Random */ \
189 pf(UDPCSUM) /* Include UDP checksum */ \
190 pf(NO_TIMESTAMP) /* Don't timestamp packets (default TS) */ \
191 pf(MPLS_RND) /* Random MPLS labels */ \
192 pf(QUEUE_MAP_RND) /* queue map Random */ \
193 pf(QUEUE_MAP_CPU) /* queue map mirrors smp_processor_id() */ \
194 pf(FLOW_SEQ) /* Sequential flows */ \
195 pf(IPSEC) /* ipsec on for flows */ \
196 pf(MACSRC_RND) /* MAC-Src Random */ \
197 pf(MACDST_RND) /* MAC-Dst Random */ \
198 pf(VID_RND) /* Random VLAN ID */ \
199 pf(SVID_RND) /* Random SVLAN ID */ \
200 pf(NODE) /* Node memory alloc*/ \
201 pf(SHARED) /* Shared SKB */ \
202
203 #define pf(flag) flag##_SHIFT,
204 enum pkt_flags {
205 PKT_FLAGS
206 };
207 #undef pf
208
209 /* Device flag bits */
210 #define pf(flag) static const __u32 F_##flag = (1<<flag##_SHIFT);
211 PKT_FLAGS
212 #undef pf
213
214 #define pf(flag) __stringify(flag),
215 static char *pkt_flag_names[] = {
216 PKT_FLAGS
217 };
218 #undef pf
219
220 #define NR_PKT_FLAGS ARRAY_SIZE(pkt_flag_names)
221
222 /* Thread control flag bits */
223 #define T_STOP (1<<0) /* Stop run */
224 #define T_RUN (1<<1) /* Start run */
225 #define T_REMDEVALL (1<<2) /* Remove all devs */
226 #define T_REMDEV (1<<3) /* Remove one dev */
227
228 /* Xmit modes */
229 #define M_START_XMIT 0 /* Default normal TX */
230 #define M_NETIF_RECEIVE 1 /* Inject packets into stack */
231 #define M_QUEUE_XMIT 2 /* Inject packet into qdisc */
232
233 /* If lock -- protects updating of if_list */
234 #define if_lock(t) mutex_lock(&(t->if_lock));
235 #define if_unlock(t) mutex_unlock(&(t->if_lock));
236
237 /* Used to help with determining the pkts on receive */
238 #define PKTGEN_MAGIC 0xbe9be955
239 #define PG_PROC_DIR "pktgen"
240 #define PGCTRL "pgctrl"
241
242 #define MAX_CFLOWS 65536
243
244 #define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4)
245 #define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4)
246
247 struct imix_pkt {
248 u64 size;
249 u64 weight;
250 u64 count_so_far;
251 };
252
253 struct flow_state {
254 __be32 cur_daddr;
255 int count;
256 #ifdef CONFIG_XFRM
257 struct xfrm_state *x;
258 #endif
259 __u32 flags;
260 };
261
262 /* flow flag bits */
263 #define F_INIT (1<<0) /* flow has been initialized */
264
265 struct pktgen_dev {
266 /*
267 * Try to keep frequent/infrequent used vars. separated.
268 */
269 struct proc_dir_entry *entry; /* proc file */
270 struct pktgen_thread *pg_thread;/* the owner */
271 struct list_head list; /* chaining in the thread's run-queue */
272 struct rcu_head rcu; /* freed by RCU */
273
274 int running; /* if false, the test will stop */
275
276 /* If min != max, then we will either do a linear iteration, or
277 * we will do a random selection from within the range.
278 */
279 __u32 flags;
280 int xmit_mode;
281 int min_pkt_size;
282 int max_pkt_size;
283 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */
284 int nfrags;
285 int removal_mark; /* non-zero => the device is marked for
286 * removal by worker thread */
287
288 struct page *page;
289 u64 delay; /* nano-seconds */
290
291 __u64 count; /* Default No packets to send */
292 __u64 sofar; /* How many pkts we've sent so far */
293 __u64 tx_bytes; /* How many bytes we've transmitted */
294 __u64 errors; /* Errors when trying to transmit, */
295
296 /* runtime counters relating to clone_skb */
297
298 __u32 clone_count;
299 int last_ok; /* Was last skb sent?
300 * Or a failed transmit of some sort?
301 * This will keep sequence numbers in order
302 */
303 ktime_t next_tx;
304 ktime_t started_at;
305 ktime_t stopped_at;
306 u64 idle_acc; /* nano-seconds */
307
308 __u32 seq_num;
309
310 int clone_skb; /*
311 * Use multiple SKBs during packet gen.
312 * If this number is greater than 1, then
313 * that many copies of the same packet will be
314 * sent before a new packet is allocated.
315 * If you want to send 1024 identical packets
316 * before creating a new packet,
317 * set clone_skb to 1024.
318 */
319
320 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
321 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
322 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
323 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
324
325 struct in6_addr in6_saddr;
326 struct in6_addr in6_daddr;
327 struct in6_addr cur_in6_daddr;
328 struct in6_addr cur_in6_saddr;
329 /* For ranges */
330 struct in6_addr min_in6_daddr;
331 struct in6_addr max_in6_daddr;
332 struct in6_addr min_in6_saddr;
333 struct in6_addr max_in6_saddr;
334
335 /* If we're doing ranges, random or incremental, then this
336 * defines the min/max for those ranges.
337 */
338 __be32 saddr_min; /* inclusive, source IP address */
339 __be32 saddr_max; /* exclusive, source IP address */
340 __be32 daddr_min; /* inclusive, dest IP address */
341 __be32 daddr_max; /* exclusive, dest IP address */
342
343 __u16 udp_src_min; /* inclusive, source UDP port */
344 __u16 udp_src_max; /* exclusive, source UDP port */
345 __u16 udp_dst_min; /* inclusive, dest UDP port */
346 __u16 udp_dst_max; /* exclusive, dest UDP port */
347
348 /* DSCP + ECN */
349 __u8 tos; /* six MSB of (former) IPv4 TOS
350 are for dscp codepoint */
351 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6
352 (see RFC 3260, sec. 4) */
353
354 /* IMIX */
355 unsigned int n_imix_entries;
356 struct imix_pkt imix_entries[MAX_IMIX_ENTRIES];
357 /* Maps 0-IMIX_PRECISION range to imix_entry based on probability*/
358 __u8 imix_distribution[IMIX_PRECISION];
359
360 /* MPLS */
361 unsigned int nr_labels; /* Depth of stack, 0 = no MPLS */
362 __be32 labels[MAX_MPLS_LABELS];
363
364 /* VLAN/SVLAN (802.1Q/Q-in-Q) */
365 __u8 vlan_p;
366 __u8 vlan_cfi;
367 __u16 vlan_id; /* 0xffff means no vlan tag */
368
369 __u8 svlan_p;
370 __u8 svlan_cfi;
371 __u16 svlan_id; /* 0xffff means no svlan tag */
372
373 __u32 src_mac_count; /* How many MACs to iterate through */
374 __u32 dst_mac_count; /* How many MACs to iterate through */
375
376 unsigned char dst_mac[ETH_ALEN];
377 unsigned char src_mac[ETH_ALEN];
378
379 __u32 cur_dst_mac_offset;
380 __u32 cur_src_mac_offset;
381 __be32 cur_saddr;
382 __be32 cur_daddr;
383 __u16 ip_id;
384 __u16 cur_udp_dst;
385 __u16 cur_udp_src;
386 __u16 cur_queue_map;
387 __u32 cur_pkt_size;
388 __u32 last_pkt_size;
389
390 __u8 hh[14];
391 /* = {
392 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB,
393
394 We fill in SRC address later
395 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
396 0x08, 0x00
397 };
398 */
399 __u16 pad; /* pad out the hh struct to an even 16 bytes */
400
401 struct sk_buff *skb; /* skb we are to transmit next, used for when we
402 * are transmitting the same one multiple times
403 */
404 struct net_device *odev; /* The out-going device.
405 * Note that the device should have it's
406 * pg_info pointer pointing back to this
407 * device.
408 * Set when the user specifies the out-going
409 * device name (not when the inject is
410 * started as it used to do.)
411 */
412 netdevice_tracker dev_tracker;
413 char odevname[32];
414 struct flow_state *flows;
415 unsigned int cflows; /* Concurrent flows (config) */
416 unsigned int lflow; /* Flow length (config) */
417 unsigned int nflows; /* accumulated flows (stats) */
418 unsigned int curfl; /* current sequenced flow (state)*/
419
420 u16 queue_map_min;
421 u16 queue_map_max;
422 __u32 skb_priority; /* skb priority field */
423 unsigned int burst; /* number of duplicated packets to burst */
424 int node; /* Memory node */
425
426 #ifdef CONFIG_XFRM
427 __u8 ipsmode; /* IPSEC mode (config) */
428 __u8 ipsproto; /* IPSEC type (config) */
429 __u32 spi;
430 struct xfrm_dst xdst;
431 struct dst_ops dstops;
432 #endif
433 char result[512];
434 };
435
436 struct pktgen_hdr {
437 __be32 pgh_magic;
438 __be32 seq_num;
439 __be32 tv_sec;
440 __be32 tv_usec;
441 };
442
443
444 static unsigned int pg_net_id __read_mostly;
445
446 struct pktgen_net {
447 struct net *net;
448 struct proc_dir_entry *proc_dir;
449 struct list_head pktgen_threads;
450 bool pktgen_exiting;
451 };
452
453 struct pktgen_thread {
454 struct mutex if_lock; /* for list of devices */
455 struct list_head if_list; /* All device here */
456 struct list_head th_list;
457 struct task_struct *tsk;
458 char result[512];
459
460 /* Field for thread to receive "posted" events terminate,
461 stop ifs etc. */
462
463 u32 control;
464 int cpu;
465
466 wait_queue_head_t queue;
467 struct completion start_done;
468 struct pktgen_net *net;
469 };
470
471 #define REMOVE 1
472 #define FIND 0
473
474 static const char version[] =
475 "Packet Generator for packet performance testing. "
476 "Version: " VERSION "\n";
477
478 static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i);
479 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname);
480 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
481 const char *ifname, bool exact);
482 static int pktgen_device_event(struct notifier_block *, unsigned long, void *);
483 static void pktgen_run_all_threads(struct pktgen_net *pn);
484 static void pktgen_reset_all_threads(struct pktgen_net *pn);
485 static void pktgen_stop_all_threads(struct pktgen_net *pn);
486
487 static void pktgen_stop(struct pktgen_thread *t);
488 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev);
489 static void fill_imix_distribution(struct pktgen_dev *pkt_dev);
490
491 /* Module parameters, defaults. */
492 static int pg_count_d __read_mostly = 1000;
493 static int pg_delay_d __read_mostly;
494 static int pg_clone_skb_d __read_mostly;
495 static int debug __read_mostly;
496
497 static DEFINE_MUTEX(pktgen_thread_lock);
498
499 static struct notifier_block pktgen_notifier_block = {
500 .notifier_call = pktgen_device_event,
501 };
502
503 /*
504 * /proc handling functions
505 *
506 */
507
pgctrl_show(struct seq_file * seq,void * v)508 static int pgctrl_show(struct seq_file *seq, void *v)
509 {
510 seq_puts(seq, version);
511 return 0;
512 }
513
pgctrl_write(struct file * file,const char __user * buf,size_t count,loff_t * ppos)514 static ssize_t pgctrl_write(struct file *file, const char __user *buf,
515 size_t count, loff_t *ppos)
516 {
517 char data[128];
518 size_t max;
519 struct pktgen_net *pn = net_generic(current->nsproxy->net_ns, pg_net_id);
520
521 if (!capable(CAP_NET_ADMIN))
522 return -EPERM;
523
524 if (count < 1)
525 return -EINVAL;
526
527 max = min(count, sizeof(data) - 1);
528 if (copy_from_user(data, buf, max))
529 return -EFAULT;
530
531 if (data[max - 1] == '\n')
532 data[max - 1] = 0; /* strip trailing '\n', terminate string */
533 else
534 data[max] = 0; /* terminate string */
535
536 if (!strcmp(data, "stop"))
537 pktgen_stop_all_threads(pn);
538 else if (!strcmp(data, "start"))
539 pktgen_run_all_threads(pn);
540 else if (!strcmp(data, "reset"))
541 pktgen_reset_all_threads(pn);
542 else
543 return -EINVAL;
544
545 return count;
546 }
547
pgctrl_open(struct inode * inode,struct file * file)548 static int pgctrl_open(struct inode *inode, struct file *file)
549 {
550 return single_open(file, pgctrl_show, pde_data(inode));
551 }
552
553 static const struct proc_ops pktgen_proc_ops = {
554 .proc_open = pgctrl_open,
555 .proc_read = seq_read,
556 .proc_lseek = seq_lseek,
557 .proc_write = pgctrl_write,
558 .proc_release = single_release,
559 };
560
pktgen_if_show(struct seq_file * seq,void * v)561 static int pktgen_if_show(struct seq_file *seq, void *v)
562 {
563 const struct pktgen_dev *pkt_dev = seq->private;
564 ktime_t stopped;
565 unsigned int i;
566 u64 idle;
567
568 seq_printf(seq,
569 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n",
570 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size,
571 pkt_dev->max_pkt_size);
572
573 if (pkt_dev->n_imix_entries > 0) {
574 seq_puts(seq, " imix_weights: ");
575 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
576 seq_printf(seq, "%llu,%llu ",
577 pkt_dev->imix_entries[i].size,
578 pkt_dev->imix_entries[i].weight);
579 }
580 seq_puts(seq, "\n");
581 }
582
583 seq_printf(seq,
584 " frags: %d delay: %llu clone_skb: %d ifname: %s\n",
585 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay,
586 pkt_dev->clone_skb, pkt_dev->odevname);
587
588 seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows,
589 pkt_dev->lflow);
590
591 seq_printf(seq,
592 " queue_map_min: %u queue_map_max: %u\n",
593 pkt_dev->queue_map_min,
594 pkt_dev->queue_map_max);
595
596 if (pkt_dev->skb_priority)
597 seq_printf(seq, " skb_priority: %u\n",
598 pkt_dev->skb_priority);
599
600 if (pkt_dev->flags & F_IPV6) {
601 seq_printf(seq,
602 " saddr: %pI6c min_saddr: %pI6c max_saddr: %pI6c\n"
603 " daddr: %pI6c min_daddr: %pI6c max_daddr: %pI6c\n",
604 &pkt_dev->in6_saddr,
605 &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr,
606 &pkt_dev->in6_daddr,
607 &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr);
608 } else {
609 seq_printf(seq,
610 " dst_min: %s dst_max: %s\n",
611 pkt_dev->dst_min, pkt_dev->dst_max);
612 seq_printf(seq,
613 " src_min: %s src_max: %s\n",
614 pkt_dev->src_min, pkt_dev->src_max);
615 }
616
617 seq_puts(seq, " src_mac: ");
618
619 seq_printf(seq, "%pM ",
620 is_zero_ether_addr(pkt_dev->src_mac) ?
621 pkt_dev->odev->dev_addr : pkt_dev->src_mac);
622
623 seq_puts(seq, "dst_mac: ");
624 seq_printf(seq, "%pM\n", pkt_dev->dst_mac);
625
626 seq_printf(seq,
627 " udp_src_min: %d udp_src_max: %d"
628 " udp_dst_min: %d udp_dst_max: %d\n",
629 pkt_dev->udp_src_min, pkt_dev->udp_src_max,
630 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max);
631
632 seq_printf(seq,
633 " src_mac_count: %d dst_mac_count: %d\n",
634 pkt_dev->src_mac_count, pkt_dev->dst_mac_count);
635
636 if (pkt_dev->nr_labels) {
637 seq_puts(seq, " mpls: ");
638 for (i = 0; i < pkt_dev->nr_labels; i++)
639 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]),
640 i == pkt_dev->nr_labels-1 ? "\n" : ", ");
641 }
642
643 if (pkt_dev->vlan_id != 0xffff)
644 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n",
645 pkt_dev->vlan_id, pkt_dev->vlan_p,
646 pkt_dev->vlan_cfi);
647
648 if (pkt_dev->svlan_id != 0xffff)
649 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n",
650 pkt_dev->svlan_id, pkt_dev->svlan_p,
651 pkt_dev->svlan_cfi);
652
653 if (pkt_dev->tos)
654 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos);
655
656 if (pkt_dev->traffic_class)
657 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class);
658
659 if (pkt_dev->burst > 1)
660 seq_printf(seq, " burst: %d\n", pkt_dev->burst);
661
662 if (pkt_dev->node >= 0)
663 seq_printf(seq, " node: %d\n", pkt_dev->node);
664
665 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE)
666 seq_puts(seq, " xmit_mode: netif_receive\n");
667 else if (pkt_dev->xmit_mode == M_QUEUE_XMIT)
668 seq_puts(seq, " xmit_mode: xmit_queue\n");
669
670 seq_puts(seq, " Flags: ");
671
672 for (i = 0; i < NR_PKT_FLAGS; i++) {
673 if (i == FLOW_SEQ_SHIFT)
674 if (!pkt_dev->cflows)
675 continue;
676
677 if (pkt_dev->flags & (1 << i)) {
678 seq_printf(seq, "%s ", pkt_flag_names[i]);
679 #ifdef CONFIG_XFRM
680 if (i == IPSEC_SHIFT && pkt_dev->spi)
681 seq_printf(seq, "spi:%u ", pkt_dev->spi);
682 #endif
683 } else if (i == FLOW_SEQ_SHIFT) {
684 seq_puts(seq, "FLOW_RND ");
685 }
686 }
687
688 seq_puts(seq, "\n");
689
690 /* not really stopped, more like last-running-at */
691 stopped = pkt_dev->running ? ktime_get() : pkt_dev->stopped_at;
692 idle = pkt_dev->idle_acc;
693 do_div(idle, NSEC_PER_USEC);
694
695 seq_printf(seq,
696 "Current:\n pkts-sofar: %llu errors: %llu\n",
697 (unsigned long long)pkt_dev->sofar,
698 (unsigned long long)pkt_dev->errors);
699
700 if (pkt_dev->n_imix_entries > 0) {
701 int i;
702
703 seq_puts(seq, " imix_size_counts: ");
704 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
705 seq_printf(seq, "%llu,%llu ",
706 pkt_dev->imix_entries[i].size,
707 pkt_dev->imix_entries[i].count_so_far);
708 }
709 seq_puts(seq, "\n");
710 }
711
712 seq_printf(seq,
713 " started: %lluus stopped: %lluus idle: %lluus\n",
714 (unsigned long long) ktime_to_us(pkt_dev->started_at),
715 (unsigned long long) ktime_to_us(stopped),
716 (unsigned long long) idle);
717
718 seq_printf(seq,
719 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n",
720 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset,
721 pkt_dev->cur_src_mac_offset);
722
723 if (pkt_dev->flags & F_IPV6) {
724 seq_printf(seq, " cur_saddr: %pI6c cur_daddr: %pI6c\n",
725 &pkt_dev->cur_in6_saddr,
726 &pkt_dev->cur_in6_daddr);
727 } else
728 seq_printf(seq, " cur_saddr: %pI4 cur_daddr: %pI4\n",
729 &pkt_dev->cur_saddr, &pkt_dev->cur_daddr);
730
731 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n",
732 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src);
733
734 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map);
735
736 seq_printf(seq, " flows: %u\n", pkt_dev->nflows);
737
738 if (pkt_dev->result[0])
739 seq_printf(seq, "Result: %s\n", pkt_dev->result);
740 else
741 seq_puts(seq, "Result: Idle\n");
742
743 return 0;
744 }
745
746
hex32_arg(const char __user * user_buffer,size_t maxlen,__u32 * num)747 static ssize_t hex32_arg(const char __user *user_buffer, size_t maxlen,
748 __u32 *num)
749 {
750 size_t i = 0;
751
752 *num = 0;
753
754 for (; i < maxlen; i++) {
755 int value;
756 char c;
757 if (get_user(c, &user_buffer[i]))
758 return -EFAULT;
759 value = hex_to_bin(c);
760 if (value >= 0) {
761 *num <<= 4;
762 *num |= value;
763 } else {
764 break;
765 }
766 }
767 return i;
768 }
769
count_trail_chars(const char __user * user_buffer,size_t maxlen)770 static ssize_t count_trail_chars(const char __user *user_buffer, size_t maxlen)
771 {
772 size_t i;
773
774 for (i = 0; i < maxlen; i++) {
775 char c;
776 if (get_user(c, &user_buffer[i]))
777 return -EFAULT;
778 switch (c) {
779 case '\"':
780 case '\n':
781 case '\r':
782 case '\t':
783 case ' ':
784 case '=':
785 break;
786 default:
787 goto done;
788 }
789 }
790 done:
791 return i;
792 }
793
num_arg(const char __user * user_buffer,size_t maxlen,unsigned long * num)794 static ssize_t num_arg(const char __user *user_buffer, size_t maxlen,
795 unsigned long *num)
796 {
797 size_t i;
798 *num = 0;
799
800 for (i = 0; i < maxlen; i++) {
801 char c;
802 if (get_user(c, &user_buffer[i]))
803 return -EFAULT;
804 if ((c >= '0') && (c <= '9')) {
805 *num *= 10;
806 *num += c - '0';
807 } else
808 break;
809 }
810 return i;
811 }
812
strn_len(const char __user * user_buffer,size_t maxlen)813 static ssize_t strn_len(const char __user *user_buffer, size_t maxlen)
814 {
815 size_t i;
816
817 for (i = 0; i < maxlen; i++) {
818 char c;
819 if (get_user(c, &user_buffer[i]))
820 return -EFAULT;
821 switch (c) {
822 case '\"':
823 case '\n':
824 case '\r':
825 case '\t':
826 case ' ':
827 case '=':
828 goto done_str;
829 default:
830 break;
831 }
832 }
833 done_str:
834 return i;
835 }
836
837 /* Parses imix entries from user buffer.
838 * The user buffer should consist of imix entries separated by spaces
839 * where each entry consists of size and weight delimited by commas.
840 * "size1,weight_1 size2,weight_2 ... size_n,weight_n" for example.
841 */
get_imix_entries(const char __user * buffer,size_t maxlen,struct pktgen_dev * pkt_dev)842 static ssize_t get_imix_entries(const char __user *buffer,
843 size_t maxlen,
844 struct pktgen_dev *pkt_dev)
845 {
846 size_t i = 0, max;
847 ssize_t len;
848 char c;
849
850 pkt_dev->n_imix_entries = 0;
851
852 do {
853 unsigned long weight;
854 unsigned long size;
855
856 if (pkt_dev->n_imix_entries >= MAX_IMIX_ENTRIES)
857 return -E2BIG;
858
859 if (i >= maxlen)
860 return -EINVAL;
861
862 max = min(10, maxlen - i);
863 len = num_arg(&buffer[i], max, &size);
864 if (len < 0)
865 return len;
866 i += len;
867 if (i >= maxlen)
868 return -EINVAL;
869 if (get_user(c, &buffer[i]))
870 return -EFAULT;
871 /* Check for comma between size_i and weight_i */
872 if (c != ',')
873 return -EINVAL;
874 i++;
875 if (i >= maxlen)
876 return -EINVAL;
877
878 if (size < 14 + 20 + 8)
879 size = 14 + 20 + 8;
880
881 max = min(10, maxlen - i);
882 len = num_arg(&buffer[i], max, &weight);
883 if (len < 0)
884 return len;
885 if (weight <= 0)
886 return -EINVAL;
887
888 pkt_dev->imix_entries[pkt_dev->n_imix_entries].size = size;
889 pkt_dev->imix_entries[pkt_dev->n_imix_entries].weight = weight;
890
891 i += len;
892 pkt_dev->n_imix_entries++;
893
894 if (i >= maxlen)
895 break;
896 if (get_user(c, &buffer[i]))
897 return -EFAULT;
898 i++;
899 } while (c == ' ');
900
901 return i;
902 }
903
get_labels(const char __user * buffer,size_t maxlen,struct pktgen_dev * pkt_dev)904 static ssize_t get_labels(const char __user *buffer,
905 size_t maxlen, struct pktgen_dev *pkt_dev)
906 {
907 unsigned int n = 0;
908 size_t i = 0, max;
909 ssize_t len;
910 char c;
911
912 pkt_dev->nr_labels = 0;
913 do {
914 __u32 tmp;
915
916 if (n >= MAX_MPLS_LABELS)
917 return -E2BIG;
918
919 if (i >= maxlen)
920 return -EINVAL;
921
922 max = min(8, maxlen - i);
923 len = hex32_arg(&buffer[i], max, &tmp);
924 if (len < 0)
925 return len;
926
927 /* return empty list in case of invalid input or zero value */
928 if (len == 0 || tmp == 0)
929 return maxlen;
930
931 pkt_dev->labels[n] = htonl(tmp);
932 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM)
933 pkt_dev->flags |= F_MPLS_RND;
934 i += len;
935 n++;
936 if (i >= maxlen)
937 break;
938 if (get_user(c, &buffer[i]))
939 return -EFAULT;
940 i++;
941 } while (c == ',');
942
943 pkt_dev->nr_labels = n;
944 return i;
945 }
946
pktgen_read_flag(const char * f,bool * disable)947 static __u32 pktgen_read_flag(const char *f, bool *disable)
948 {
949 __u32 i;
950
951 if (f[0] == '!') {
952 *disable = true;
953 f++;
954 }
955
956 for (i = 0; i < NR_PKT_FLAGS; i++) {
957 if (!IS_ENABLED(CONFIG_XFRM) && i == IPSEC_SHIFT)
958 continue;
959
960 /* allow only disabling ipv6 flag */
961 if (!*disable && i == IPV6_SHIFT)
962 continue;
963
964 if (strcmp(f, pkt_flag_names[i]) == 0)
965 return 1 << i;
966 }
967
968 if (strcmp(f, "FLOW_RND") == 0) {
969 *disable = !*disable;
970 return F_FLOW_SEQ;
971 }
972
973 return 0;
974 }
975
pktgen_if_write(struct file * file,const char __user * user_buffer,size_t count,loff_t * offset)976 static ssize_t pktgen_if_write(struct file *file,
977 const char __user * user_buffer, size_t count,
978 loff_t * offset)
979 {
980 struct seq_file *seq = file->private_data;
981 struct pktgen_dev *pkt_dev = seq->private;
982 size_t i, max;
983 ssize_t len;
984 char name[16], valstr[32];
985 unsigned long value = 0;
986 char *pg_result = NULL;
987 char buf[128];
988
989 pg_result = &(pkt_dev->result[0]);
990
991 if (count < 1) {
992 pr_warn("wrong command format\n");
993 return -EINVAL;
994 }
995
996 max = count;
997 len = count_trail_chars(user_buffer, max);
998 if (len < 0) {
999 pr_warn("illegal format\n");
1000 return len;
1001 }
1002 i = len;
1003
1004 /* Read variable name */
1005 max = min(sizeof(name) - 1, count - i);
1006 len = strn_len(&user_buffer[i], max);
1007 if (len < 0)
1008 return len;
1009
1010 memset(name, 0, sizeof(name));
1011 if (copy_from_user(name, &user_buffer[i], len))
1012 return -EFAULT;
1013 i += len;
1014
1015 max = count - i;
1016 len = count_trail_chars(&user_buffer[i], max);
1017 if (len < 0)
1018 return len;
1019
1020 i += len;
1021
1022 if (debug) {
1023 size_t copy = min_t(size_t, count + 1, 1024);
1024 char *tp = strndup_user(user_buffer, copy);
1025
1026 if (IS_ERR(tp))
1027 return PTR_ERR(tp);
1028
1029 pr_debug("%s,%zu buffer -:%s:-\n", name, count, tp);
1030 kfree(tp);
1031 }
1032
1033 if (!strcmp(name, "min_pkt_size")) {
1034 max = min(10, count - i);
1035 len = num_arg(&user_buffer[i], max, &value);
1036 if (len < 0)
1037 return len;
1038
1039 if (value < 14 + 20 + 8)
1040 value = 14 + 20 + 8;
1041 if (value != pkt_dev->min_pkt_size) {
1042 pkt_dev->min_pkt_size = value;
1043 pkt_dev->cur_pkt_size = value;
1044 }
1045 sprintf(pg_result, "OK: min_pkt_size=%d",
1046 pkt_dev->min_pkt_size);
1047 return count;
1048 }
1049
1050 if (!strcmp(name, "max_pkt_size")) {
1051 max = min(10, count - i);
1052 len = num_arg(&user_buffer[i], max, &value);
1053 if (len < 0)
1054 return len;
1055
1056 if (value < 14 + 20 + 8)
1057 value = 14 + 20 + 8;
1058 if (value != pkt_dev->max_pkt_size) {
1059 pkt_dev->max_pkt_size = value;
1060 pkt_dev->cur_pkt_size = value;
1061 }
1062 sprintf(pg_result, "OK: max_pkt_size=%d",
1063 pkt_dev->max_pkt_size);
1064 return count;
1065 }
1066
1067 /* Shortcut for min = max */
1068
1069 if (!strcmp(name, "pkt_size")) {
1070 max = min(10, count - i);
1071 len = num_arg(&user_buffer[i], max, &value);
1072 if (len < 0)
1073 return len;
1074
1075 if (value < 14 + 20 + 8)
1076 value = 14 + 20 + 8;
1077 if (value != pkt_dev->min_pkt_size) {
1078 pkt_dev->min_pkt_size = value;
1079 pkt_dev->max_pkt_size = value;
1080 pkt_dev->cur_pkt_size = value;
1081 }
1082 sprintf(pg_result, "OK: pkt_size=%d", pkt_dev->min_pkt_size);
1083 return count;
1084 }
1085
1086 if (!strcmp(name, "imix_weights")) {
1087 if (pkt_dev->clone_skb > 0)
1088 return -EINVAL;
1089
1090 max = count - i;
1091 len = get_imix_entries(&user_buffer[i], max, pkt_dev);
1092 if (len < 0)
1093 return len;
1094
1095 fill_imix_distribution(pkt_dev);
1096
1097 return count;
1098 }
1099
1100 if (!strcmp(name, "debug")) {
1101 max = min(10, count - i);
1102 len = num_arg(&user_buffer[i], max, &value);
1103 if (len < 0)
1104 return len;
1105
1106 debug = value;
1107 sprintf(pg_result, "OK: debug=%u", debug);
1108 return count;
1109 }
1110
1111 if (!strcmp(name, "frags")) {
1112 max = min(10, count - i);
1113 len = num_arg(&user_buffer[i], max, &value);
1114 if (len < 0)
1115 return len;
1116
1117 pkt_dev->nfrags = value;
1118 sprintf(pg_result, "OK: frags=%d", pkt_dev->nfrags);
1119 return count;
1120 }
1121 if (!strcmp(name, "delay")) {
1122 max = min(10, count - i);
1123 len = num_arg(&user_buffer[i], max, &value);
1124 if (len < 0)
1125 return len;
1126
1127 if (value == 0x7FFFFFFF)
1128 pkt_dev->delay = ULLONG_MAX;
1129 else
1130 pkt_dev->delay = (u64)value;
1131
1132 sprintf(pg_result, "OK: delay=%llu",
1133 (unsigned long long) pkt_dev->delay);
1134 return count;
1135 }
1136 if (!strcmp(name, "rate")) {
1137 max = min(10, count - i);
1138 len = num_arg(&user_buffer[i], max, &value);
1139 if (len < 0)
1140 return len;
1141
1142 if (!value)
1143 return -EINVAL;
1144 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value;
1145 if (debug)
1146 pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1147
1148 sprintf(pg_result, "OK: rate=%lu", value);
1149 return count;
1150 }
1151 if (!strcmp(name, "ratep")) {
1152 max = min(10, count - i);
1153 len = num_arg(&user_buffer[i], max, &value);
1154 if (len < 0)
1155 return len;
1156
1157 if (!value)
1158 return -EINVAL;
1159 pkt_dev->delay = NSEC_PER_SEC/value;
1160 if (debug)
1161 pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1162
1163 sprintf(pg_result, "OK: rate=%lu", value);
1164 return count;
1165 }
1166 if (!strcmp(name, "udp_src_min")) {
1167 max = min(10, count - i);
1168 len = num_arg(&user_buffer[i], max, &value);
1169 if (len < 0)
1170 return len;
1171
1172 if (value != pkt_dev->udp_src_min) {
1173 pkt_dev->udp_src_min = value;
1174 pkt_dev->cur_udp_src = value;
1175 }
1176 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min);
1177 return count;
1178 }
1179 if (!strcmp(name, "udp_dst_min")) {
1180 max = min(10, count - i);
1181 len = num_arg(&user_buffer[i], max, &value);
1182 if (len < 0)
1183 return len;
1184
1185 if (value != pkt_dev->udp_dst_min) {
1186 pkt_dev->udp_dst_min = value;
1187 pkt_dev->cur_udp_dst = value;
1188 }
1189 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min);
1190 return count;
1191 }
1192 if (!strcmp(name, "udp_src_max")) {
1193 max = min(10, count - i);
1194 len = num_arg(&user_buffer[i], max, &value);
1195 if (len < 0)
1196 return len;
1197
1198 if (value != pkt_dev->udp_src_max) {
1199 pkt_dev->udp_src_max = value;
1200 pkt_dev->cur_udp_src = value;
1201 }
1202 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max);
1203 return count;
1204 }
1205 if (!strcmp(name, "udp_dst_max")) {
1206 max = min(10, count - i);
1207 len = num_arg(&user_buffer[i], max, &value);
1208 if (len < 0)
1209 return len;
1210
1211 if (value != pkt_dev->udp_dst_max) {
1212 pkt_dev->udp_dst_max = value;
1213 pkt_dev->cur_udp_dst = value;
1214 }
1215 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max);
1216 return count;
1217 }
1218 if (!strcmp(name, "clone_skb")) {
1219 max = min(10, count - i);
1220 len = num_arg(&user_buffer[i], max, &value);
1221 if (len < 0)
1222 return len;
1223 /* clone_skb is not supported for netif_receive xmit_mode and
1224 * IMIX mode.
1225 */
1226 if ((value > 0) &&
1227 ((pkt_dev->xmit_mode == M_NETIF_RECEIVE) ||
1228 !(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))
1229 return -EOPNOTSUPP;
1230 if (value > 0 && (pkt_dev->n_imix_entries > 0 ||
1231 !(pkt_dev->flags & F_SHARED)))
1232 return -EINVAL;
1233
1234 pkt_dev->clone_skb = value;
1235
1236 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb);
1237 return count;
1238 }
1239 if (!strcmp(name, "count")) {
1240 max = min(10, count - i);
1241 len = num_arg(&user_buffer[i], max, &value);
1242 if (len < 0)
1243 return len;
1244
1245 pkt_dev->count = value;
1246 sprintf(pg_result, "OK: count=%llu",
1247 (unsigned long long)pkt_dev->count);
1248 return count;
1249 }
1250 if (!strcmp(name, "src_mac_count")) {
1251 max = min(10, count - i);
1252 len = num_arg(&user_buffer[i], max, &value);
1253 if (len < 0)
1254 return len;
1255
1256 if (pkt_dev->src_mac_count != value) {
1257 pkt_dev->src_mac_count = value;
1258 pkt_dev->cur_src_mac_offset = 0;
1259 }
1260 sprintf(pg_result, "OK: src_mac_count=%d",
1261 pkt_dev->src_mac_count);
1262 return count;
1263 }
1264 if (!strcmp(name, "dst_mac_count")) {
1265 max = min(10, count - i);
1266 len = num_arg(&user_buffer[i], max, &value);
1267 if (len < 0)
1268 return len;
1269
1270 if (pkt_dev->dst_mac_count != value) {
1271 pkt_dev->dst_mac_count = value;
1272 pkt_dev->cur_dst_mac_offset = 0;
1273 }
1274 sprintf(pg_result, "OK: dst_mac_count=%d",
1275 pkt_dev->dst_mac_count);
1276 return count;
1277 }
1278 if (!strcmp(name, "burst")) {
1279 max = min(10, count - i);
1280 len = num_arg(&user_buffer[i], max, &value);
1281 if (len < 0)
1282 return len;
1283
1284 if ((value > 1) &&
1285 ((pkt_dev->xmit_mode == M_QUEUE_XMIT) ||
1286 ((pkt_dev->xmit_mode == M_START_XMIT) &&
1287 (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))))
1288 return -EOPNOTSUPP;
1289
1290 if (value > 1 && !(pkt_dev->flags & F_SHARED))
1291 return -EINVAL;
1292
1293 pkt_dev->burst = value < 1 ? 1 : value;
1294 sprintf(pg_result, "OK: burst=%u", pkt_dev->burst);
1295 return count;
1296 }
1297 if (!strcmp(name, "node")) {
1298 max = min(10, count - i);
1299 len = num_arg(&user_buffer[i], max, &value);
1300 if (len < 0)
1301 return len;
1302
1303 if (node_possible(value)) {
1304 pkt_dev->node = value;
1305 sprintf(pg_result, "OK: node=%d", pkt_dev->node);
1306 if (pkt_dev->page) {
1307 put_page(pkt_dev->page);
1308 pkt_dev->page = NULL;
1309 }
1310 }
1311 else
1312 sprintf(pg_result, "ERROR: node not possible");
1313 return count;
1314 }
1315 if (!strcmp(name, "xmit_mode")) {
1316 char f[32];
1317
1318 max = min(sizeof(f) - 1, count - i);
1319 len = strn_len(&user_buffer[i], max);
1320 if (len < 0)
1321 return len;
1322
1323 memset(f, 0, sizeof(f));
1324 if (copy_from_user(f, &user_buffer[i], len))
1325 return -EFAULT;
1326
1327 if (strcmp(f, "start_xmit") == 0) {
1328 pkt_dev->xmit_mode = M_START_XMIT;
1329 } else if (strcmp(f, "netif_receive") == 0) {
1330 /* clone_skb set earlier, not supported in this mode */
1331 if (pkt_dev->clone_skb > 0)
1332 return -EOPNOTSUPP;
1333
1334 pkt_dev->xmit_mode = M_NETIF_RECEIVE;
1335
1336 /* make sure new packet is allocated every time
1337 * pktgen_xmit() is called
1338 */
1339 pkt_dev->last_ok = 1;
1340 } else if (strcmp(f, "queue_xmit") == 0) {
1341 pkt_dev->xmit_mode = M_QUEUE_XMIT;
1342 pkt_dev->last_ok = 1;
1343 } else {
1344 sprintf(pg_result,
1345 "xmit_mode -:%s:- unknown\nAvailable modes: %s",
1346 f, "start_xmit, netif_receive\n");
1347 return count;
1348 }
1349 sprintf(pg_result, "OK: xmit_mode=%s", f);
1350 return count;
1351 }
1352 if (!strcmp(name, "flag")) {
1353 bool disable = false;
1354 __u32 flag;
1355 char f[32];
1356 char *end;
1357
1358 max = min(sizeof(f) - 1, count - i);
1359 len = strn_len(&user_buffer[i], max);
1360 if (len < 0)
1361 return len;
1362
1363 memset(f, 0, 32);
1364 if (copy_from_user(f, &user_buffer[i], len))
1365 return -EFAULT;
1366
1367 flag = pktgen_read_flag(f, &disable);
1368 if (flag) {
1369 if (disable) {
1370 /* If "clone_skb", or "burst" parameters are
1371 * configured, it means that the skb still
1372 * needs to be referenced by the pktgen, so
1373 * the skb must be shared.
1374 */
1375 if (flag == F_SHARED && (pkt_dev->clone_skb ||
1376 pkt_dev->burst > 1))
1377 return -EINVAL;
1378 pkt_dev->flags &= ~flag;
1379 } else {
1380 pkt_dev->flags |= flag;
1381 }
1382
1383 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags);
1384 return count;
1385 }
1386
1387 /* Unknown flag */
1388 end = pkt_dev->result + sizeof(pkt_dev->result);
1389 pg_result += sprintf(pg_result,
1390 "Flag -:%s:- unknown\n"
1391 "Available flags, (prepend ! to un-set flag):\n", f);
1392
1393 for (int n = 0; n < NR_PKT_FLAGS && pg_result < end; n++) {
1394 if (!IS_ENABLED(CONFIG_XFRM) && n == IPSEC_SHIFT)
1395 continue;
1396 pg_result += snprintf(pg_result, end - pg_result,
1397 "%s, ", pkt_flag_names[n]);
1398 }
1399 if (!WARN_ON_ONCE(pg_result >= end)) {
1400 /* Remove the comma and whitespace at the end */
1401 *(pg_result - 2) = '\0';
1402 }
1403
1404 return count;
1405 }
1406 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) {
1407 max = min(sizeof(pkt_dev->dst_min) - 1, count - i);
1408 len = strn_len(&user_buffer[i], max);
1409 if (len < 0)
1410 return len;
1411
1412 if (copy_from_user(buf, &user_buffer[i], len))
1413 return -EFAULT;
1414 buf[len] = 0;
1415 if (strcmp(buf, pkt_dev->dst_min) != 0) {
1416 memset(pkt_dev->dst_min, 0, sizeof(pkt_dev->dst_min));
1417 strcpy(pkt_dev->dst_min, buf);
1418 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
1419 pkt_dev->cur_daddr = pkt_dev->daddr_min;
1420 }
1421 if (debug)
1422 pr_debug("dst_min set to: %s\n", pkt_dev->dst_min);
1423
1424 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min);
1425 return count;
1426 }
1427 if (!strcmp(name, "dst_max")) {
1428 max = min(sizeof(pkt_dev->dst_max) - 1, count - i);
1429 len = strn_len(&user_buffer[i], max);
1430 if (len < 0)
1431 return len;
1432
1433 if (copy_from_user(buf, &user_buffer[i], len))
1434 return -EFAULT;
1435 buf[len] = 0;
1436 if (strcmp(buf, pkt_dev->dst_max) != 0) {
1437 memset(pkt_dev->dst_max, 0, sizeof(pkt_dev->dst_max));
1438 strcpy(pkt_dev->dst_max, buf);
1439 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
1440 pkt_dev->cur_daddr = pkt_dev->daddr_max;
1441 }
1442 if (debug)
1443 pr_debug("dst_max set to: %s\n", pkt_dev->dst_max);
1444
1445 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max);
1446 return count;
1447 }
1448 if (!strcmp(name, "dst6")) {
1449 max = min(sizeof(buf) - 1, count - i);
1450 len = strn_len(&user_buffer[i], max);
1451 if (len < 0)
1452 return len;
1453
1454 pkt_dev->flags |= F_IPV6;
1455
1456 if (copy_from_user(buf, &user_buffer[i], len))
1457 return -EFAULT;
1458 buf[len] = 0;
1459
1460 in6_pton(buf, -1, pkt_dev->in6_daddr.s6_addr, -1, NULL);
1461 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr);
1462
1463 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr;
1464
1465 if (debug)
1466 pr_debug("dst6 set to: %s\n", buf);
1467
1468 sprintf(pg_result, "OK: dst6=%s", buf);
1469 return count;
1470 }
1471 if (!strcmp(name, "dst6_min")) {
1472 max = min(sizeof(buf) - 1, count - i);
1473 len = strn_len(&user_buffer[i], max);
1474 if (len < 0)
1475 return len;
1476
1477 pkt_dev->flags |= F_IPV6;
1478
1479 if (copy_from_user(buf, &user_buffer[i], len))
1480 return -EFAULT;
1481 buf[len] = 0;
1482
1483 in6_pton(buf, -1, pkt_dev->min_in6_daddr.s6_addr, -1, NULL);
1484 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr);
1485
1486 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr;
1487 if (debug)
1488 pr_debug("dst6_min set to: %s\n", buf);
1489
1490 sprintf(pg_result, "OK: dst6_min=%s", buf);
1491 return count;
1492 }
1493 if (!strcmp(name, "dst6_max")) {
1494 max = min(sizeof(buf) - 1, count - i);
1495 len = strn_len(&user_buffer[i], max);
1496 if (len < 0)
1497 return len;
1498
1499 pkt_dev->flags |= F_IPV6;
1500
1501 if (copy_from_user(buf, &user_buffer[i], len))
1502 return -EFAULT;
1503 buf[len] = 0;
1504
1505 in6_pton(buf, -1, pkt_dev->max_in6_daddr.s6_addr, -1, NULL);
1506 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr);
1507
1508 if (debug)
1509 pr_debug("dst6_max set to: %s\n", buf);
1510
1511 sprintf(pg_result, "OK: dst6_max=%s", buf);
1512 return count;
1513 }
1514 if (!strcmp(name, "src6")) {
1515 max = min(sizeof(buf) - 1, count - i);
1516 len = strn_len(&user_buffer[i], max);
1517 if (len < 0)
1518 return len;
1519
1520 pkt_dev->flags |= F_IPV6;
1521
1522 if (copy_from_user(buf, &user_buffer[i], len))
1523 return -EFAULT;
1524 buf[len] = 0;
1525
1526 in6_pton(buf, -1, pkt_dev->in6_saddr.s6_addr, -1, NULL);
1527 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr);
1528
1529 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr;
1530
1531 if (debug)
1532 pr_debug("src6 set to: %s\n", buf);
1533
1534 sprintf(pg_result, "OK: src6=%s", buf);
1535 return count;
1536 }
1537 if (!strcmp(name, "src_min")) {
1538 max = min(sizeof(pkt_dev->src_min) - 1, count - i);
1539 len = strn_len(&user_buffer[i], max);
1540 if (len < 0)
1541 return len;
1542
1543 if (copy_from_user(buf, &user_buffer[i], len))
1544 return -EFAULT;
1545 buf[len] = 0;
1546 if (strcmp(buf, pkt_dev->src_min) != 0) {
1547 memset(pkt_dev->src_min, 0, sizeof(pkt_dev->src_min));
1548 strcpy(pkt_dev->src_min, buf);
1549 pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
1550 pkt_dev->cur_saddr = pkt_dev->saddr_min;
1551 }
1552 if (debug)
1553 pr_debug("src_min set to: %s\n", pkt_dev->src_min);
1554
1555 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min);
1556 return count;
1557 }
1558 if (!strcmp(name, "src_max")) {
1559 max = min(sizeof(pkt_dev->src_max) - 1, count - i);
1560 len = strn_len(&user_buffer[i], max);
1561 if (len < 0)
1562 return len;
1563
1564 if (copy_from_user(buf, &user_buffer[i], len))
1565 return -EFAULT;
1566 buf[len] = 0;
1567 if (strcmp(buf, pkt_dev->src_max) != 0) {
1568 memset(pkt_dev->src_max, 0, sizeof(pkt_dev->src_max));
1569 strcpy(pkt_dev->src_max, buf);
1570 pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
1571 pkt_dev->cur_saddr = pkt_dev->saddr_max;
1572 }
1573 if (debug)
1574 pr_debug("src_max set to: %s\n", pkt_dev->src_max);
1575
1576 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max);
1577 return count;
1578 }
1579 if (!strcmp(name, "dst_mac")) {
1580 max = min(sizeof(valstr) - 1, count - i);
1581 len = strn_len(&user_buffer[i], max);
1582 if (len < 0)
1583 return len;
1584
1585 memset(valstr, 0, sizeof(valstr));
1586 if (copy_from_user(valstr, &user_buffer[i], len))
1587 return -EFAULT;
1588
1589 if (!mac_pton(valstr, pkt_dev->dst_mac))
1590 return -EINVAL;
1591 /* Set up Dest MAC */
1592 ether_addr_copy(&pkt_dev->hh[0], pkt_dev->dst_mac);
1593
1594 sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac);
1595 return count;
1596 }
1597 if (!strcmp(name, "src_mac")) {
1598 max = min(sizeof(valstr) - 1, count - i);
1599 len = strn_len(&user_buffer[i], max);
1600 if (len < 0)
1601 return len;
1602
1603 memset(valstr, 0, sizeof(valstr));
1604 if (copy_from_user(valstr, &user_buffer[i], len))
1605 return -EFAULT;
1606
1607 if (!mac_pton(valstr, pkt_dev->src_mac))
1608 return -EINVAL;
1609 /* Set up Src MAC */
1610 ether_addr_copy(&pkt_dev->hh[6], pkt_dev->src_mac);
1611
1612 sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac);
1613 return count;
1614 }
1615
1616 if (!strcmp(name, "clear_counters")) {
1617 pktgen_clear_counters(pkt_dev);
1618 sprintf(pg_result, "OK: Clearing counters.\n");
1619 return count;
1620 }
1621
1622 if (!strcmp(name, "flows")) {
1623 max = min(10, count - i);
1624 len = num_arg(&user_buffer[i], max, &value);
1625 if (len < 0)
1626 return len;
1627
1628 if (value > MAX_CFLOWS)
1629 value = MAX_CFLOWS;
1630
1631 pkt_dev->cflows = value;
1632 sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows);
1633 return count;
1634 }
1635 #ifdef CONFIG_XFRM
1636 if (!strcmp(name, "spi")) {
1637 max = min(10, count - i);
1638 len = num_arg(&user_buffer[i], max, &value);
1639 if (len < 0)
1640 return len;
1641
1642 pkt_dev->spi = value;
1643 sprintf(pg_result, "OK: spi=%u", pkt_dev->spi);
1644 return count;
1645 }
1646 #endif
1647 if (!strcmp(name, "flowlen")) {
1648 max = min(10, count - i);
1649 len = num_arg(&user_buffer[i], max, &value);
1650 if (len < 0)
1651 return len;
1652
1653 pkt_dev->lflow = value;
1654 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow);
1655 return count;
1656 }
1657
1658 if (!strcmp(name, "queue_map_min")) {
1659 max = min(5, count - i);
1660 len = num_arg(&user_buffer[i], max, &value);
1661 if (len < 0)
1662 return len;
1663
1664 pkt_dev->queue_map_min = value;
1665 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min);
1666 return count;
1667 }
1668
1669 if (!strcmp(name, "queue_map_max")) {
1670 max = min(5, count - i);
1671 len = num_arg(&user_buffer[i], max, &value);
1672 if (len < 0)
1673 return len;
1674
1675 pkt_dev->queue_map_max = value;
1676 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max);
1677 return count;
1678 }
1679
1680 if (!strcmp(name, "mpls")) {
1681 unsigned int n, cnt;
1682
1683 max = count - i;
1684 len = get_labels(&user_buffer[i], max, pkt_dev);
1685 if (len < 0)
1686 return len;
1687
1688 cnt = sprintf(pg_result, "OK: mpls=");
1689 for (n = 0; n < pkt_dev->nr_labels; n++)
1690 cnt += sprintf(pg_result + cnt,
1691 "%08x%s", ntohl(pkt_dev->labels[n]),
1692 n == pkt_dev->nr_labels-1 ? "" : ",");
1693
1694 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) {
1695 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1696 pkt_dev->svlan_id = 0xffff;
1697
1698 if (debug)
1699 pr_debug("VLAN/SVLAN auto turned off\n");
1700 }
1701 return count;
1702 }
1703
1704 if (!strcmp(name, "vlan_id")) {
1705 max = min(4, count - i);
1706 len = num_arg(&user_buffer[i], max, &value);
1707 if (len < 0)
1708 return len;
1709
1710 if (value <= 4095) {
1711 pkt_dev->vlan_id = value; /* turn on VLAN */
1712
1713 if (debug)
1714 pr_debug("VLAN turned on\n");
1715
1716 if (debug && pkt_dev->nr_labels)
1717 pr_debug("MPLS auto turned off\n");
1718
1719 pkt_dev->nr_labels = 0; /* turn off MPLS */
1720 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id);
1721 } else {
1722 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1723 pkt_dev->svlan_id = 0xffff;
1724
1725 if (debug)
1726 pr_debug("VLAN/SVLAN turned off\n");
1727 }
1728 return count;
1729 }
1730
1731 if (!strcmp(name, "vlan_p")) {
1732 max = min(1, count - i);
1733 len = num_arg(&user_buffer[i], max, &value);
1734 if (len < 0)
1735 return len;
1736
1737 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) {
1738 pkt_dev->vlan_p = value;
1739 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p);
1740 } else {
1741 sprintf(pg_result, "ERROR: vlan_p must be 0-7");
1742 }
1743 return count;
1744 }
1745
1746 if (!strcmp(name, "vlan_cfi")) {
1747 max = min(1, count - i);
1748 len = num_arg(&user_buffer[i], max, &value);
1749 if (len < 0)
1750 return len;
1751
1752 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) {
1753 pkt_dev->vlan_cfi = value;
1754 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi);
1755 } else {
1756 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1");
1757 }
1758 return count;
1759 }
1760
1761 if (!strcmp(name, "svlan_id")) {
1762 max = min(4, count - i);
1763 len = num_arg(&user_buffer[i], max, &value);
1764 if (len < 0)
1765 return len;
1766
1767 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) {
1768 pkt_dev->svlan_id = value; /* turn on SVLAN */
1769
1770 if (debug)
1771 pr_debug("SVLAN turned on\n");
1772
1773 if (debug && pkt_dev->nr_labels)
1774 pr_debug("MPLS auto turned off\n");
1775
1776 pkt_dev->nr_labels = 0; /* turn off MPLS */
1777 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id);
1778 } else {
1779 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1780 pkt_dev->svlan_id = 0xffff;
1781
1782 if (debug)
1783 pr_debug("VLAN/SVLAN turned off\n");
1784 }
1785 return count;
1786 }
1787
1788 if (!strcmp(name, "svlan_p")) {
1789 max = min(1, count - i);
1790 len = num_arg(&user_buffer[i], max, &value);
1791 if (len < 0)
1792 return len;
1793
1794 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) {
1795 pkt_dev->svlan_p = value;
1796 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p);
1797 } else {
1798 sprintf(pg_result, "ERROR: svlan_p must be 0-7");
1799 }
1800 return count;
1801 }
1802
1803 if (!strcmp(name, "svlan_cfi")) {
1804 max = min(1, count - i);
1805 len = num_arg(&user_buffer[i], max, &value);
1806 if (len < 0)
1807 return len;
1808
1809 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) {
1810 pkt_dev->svlan_cfi = value;
1811 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi);
1812 } else {
1813 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1");
1814 }
1815 return count;
1816 }
1817
1818 if (!strcmp(name, "tos")) {
1819 __u32 tmp_value;
1820
1821 max = min(2, count - i);
1822 len = hex32_arg(&user_buffer[i], max, &tmp_value);
1823 if (len < 0)
1824 return len;
1825
1826 if (len == 2) {
1827 pkt_dev->tos = tmp_value;
1828 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos);
1829 } else {
1830 sprintf(pg_result, "ERROR: tos must be 00-ff");
1831 }
1832 return count;
1833 }
1834
1835 if (!strcmp(name, "traffic_class")) {
1836 __u32 tmp_value;
1837
1838 max = min(2, count - i);
1839 len = hex32_arg(&user_buffer[i], max, &tmp_value);
1840 if (len < 0)
1841 return len;
1842
1843 if (len == 2) {
1844 pkt_dev->traffic_class = tmp_value;
1845 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class);
1846 } else {
1847 sprintf(pg_result, "ERROR: traffic_class must be 00-ff");
1848 }
1849 return count;
1850 }
1851
1852 if (!strcmp(name, "skb_priority")) {
1853 max = min(9, count - i);
1854 len = num_arg(&user_buffer[i], max, &value);
1855 if (len < 0)
1856 return len;
1857
1858 pkt_dev->skb_priority = value;
1859 sprintf(pg_result, "OK: skb_priority=%i",
1860 pkt_dev->skb_priority);
1861 return count;
1862 }
1863
1864 sprintf(pkt_dev->result, "No such parameter \"%s\"", name);
1865 return -EINVAL;
1866 }
1867
pktgen_if_open(struct inode * inode,struct file * file)1868 static int pktgen_if_open(struct inode *inode, struct file *file)
1869 {
1870 return single_open(file, pktgen_if_show, pde_data(inode));
1871 }
1872
1873 static const struct proc_ops pktgen_if_proc_ops = {
1874 .proc_open = pktgen_if_open,
1875 .proc_read = seq_read,
1876 .proc_lseek = seq_lseek,
1877 .proc_write = pktgen_if_write,
1878 .proc_release = single_release,
1879 };
1880
pktgen_thread_show(struct seq_file * seq,void * v)1881 static int pktgen_thread_show(struct seq_file *seq, void *v)
1882 {
1883 struct pktgen_thread *t = seq->private;
1884 const struct pktgen_dev *pkt_dev;
1885
1886 BUG_ON(!t);
1887
1888 seq_puts(seq, "Running: ");
1889
1890 rcu_read_lock();
1891 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
1892 if (pkt_dev->running)
1893 seq_printf(seq, "%s ", pkt_dev->odevname);
1894
1895 seq_puts(seq, "\nStopped: ");
1896
1897 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
1898 if (!pkt_dev->running)
1899 seq_printf(seq, "%s ", pkt_dev->odevname);
1900
1901 if (t->result[0])
1902 seq_printf(seq, "\nResult: %s\n", t->result);
1903 else
1904 seq_puts(seq, "\nResult: NA\n");
1905
1906 rcu_read_unlock();
1907
1908 return 0;
1909 }
1910
pktgen_thread_write(struct file * file,const char __user * user_buffer,size_t count,loff_t * offset)1911 static ssize_t pktgen_thread_write(struct file *file,
1912 const char __user * user_buffer,
1913 size_t count, loff_t * offset)
1914 {
1915 struct seq_file *seq = file->private_data;
1916 struct pktgen_thread *t = seq->private;
1917 size_t i, max;
1918 ssize_t len, ret;
1919 char name[40];
1920 char *pg_result;
1921
1922 if (count < 1) {
1923 // sprintf(pg_result, "Wrong command format");
1924 return -EINVAL;
1925 }
1926
1927 max = count;
1928 len = count_trail_chars(user_buffer, max);
1929 if (len < 0)
1930 return len;
1931
1932 i = len;
1933
1934 /* Read variable name */
1935 max = min(sizeof(name) - 1, count - i);
1936 len = strn_len(&user_buffer[i], max);
1937 if (len < 0)
1938 return len;
1939
1940 memset(name, 0, sizeof(name));
1941 if (copy_from_user(name, &user_buffer[i], len))
1942 return -EFAULT;
1943 i += len;
1944
1945 max = count - i;
1946 len = count_trail_chars(&user_buffer[i], max);
1947 if (len < 0)
1948 return len;
1949
1950 i += len;
1951
1952 if (debug)
1953 pr_debug("t=%s, count=%lu\n", name, (unsigned long)count);
1954
1955 if (!t) {
1956 pr_err("ERROR: No thread\n");
1957 ret = -EINVAL;
1958 goto out;
1959 }
1960
1961 pg_result = &(t->result[0]);
1962
1963 if (!strcmp(name, "add_device")) {
1964 char f[32];
1965 memset(f, 0, 32);
1966 max = min(sizeof(f) - 1, count - i);
1967 len = strn_len(&user_buffer[i], max);
1968 if (len < 0) {
1969 ret = len;
1970 goto out;
1971 }
1972 if (copy_from_user(f, &user_buffer[i], len))
1973 return -EFAULT;
1974
1975 mutex_lock(&pktgen_thread_lock);
1976 ret = pktgen_add_device(t, f);
1977 mutex_unlock(&pktgen_thread_lock);
1978 if (!ret) {
1979 ret = count;
1980 sprintf(pg_result, "OK: add_device=%s", f);
1981 } else
1982 sprintf(pg_result, "ERROR: can not add device %s", f);
1983 goto out;
1984 }
1985
1986 if (!strcmp(name, "rem_device_all")) {
1987 mutex_lock(&pktgen_thread_lock);
1988 t->control |= T_REMDEVALL;
1989 mutex_unlock(&pktgen_thread_lock);
1990 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */
1991 ret = count;
1992 sprintf(pg_result, "OK: rem_device_all");
1993 goto out;
1994 }
1995
1996 if (!strcmp(name, "max_before_softirq")) {
1997 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use");
1998 ret = count;
1999 goto out;
2000 }
2001
2002 ret = -EINVAL;
2003 out:
2004 return ret;
2005 }
2006
pktgen_thread_open(struct inode * inode,struct file * file)2007 static int pktgen_thread_open(struct inode *inode, struct file *file)
2008 {
2009 return single_open(file, pktgen_thread_show, pde_data(inode));
2010 }
2011
2012 static const struct proc_ops pktgen_thread_proc_ops = {
2013 .proc_open = pktgen_thread_open,
2014 .proc_read = seq_read,
2015 .proc_lseek = seq_lseek,
2016 .proc_write = pktgen_thread_write,
2017 .proc_release = single_release,
2018 };
2019
2020 /* Think find or remove for NN */
__pktgen_NN_threads(const struct pktgen_net * pn,const char * ifname,int remove)2021 static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn,
2022 const char *ifname, int remove)
2023 {
2024 struct pktgen_thread *t;
2025 struct pktgen_dev *pkt_dev = NULL;
2026 bool exact = (remove == FIND);
2027
2028 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
2029 pkt_dev = pktgen_find_dev(t, ifname, exact);
2030 if (pkt_dev) {
2031 if (remove) {
2032 pkt_dev->removal_mark = 1;
2033 t->control |= T_REMDEV;
2034 }
2035 break;
2036 }
2037 }
2038 return pkt_dev;
2039 }
2040
2041 /*
2042 * mark a device for removal
2043 */
pktgen_mark_device(const struct pktgen_net * pn,const char * ifname)2044 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
2045 {
2046 struct pktgen_dev *pkt_dev = NULL;
2047 const int max_tries = 10, msec_per_try = 125;
2048 int i = 0;
2049
2050 mutex_lock(&pktgen_thread_lock);
2051 pr_debug("%s: marking %s for removal\n", __func__, ifname);
2052
2053 while (1) {
2054
2055 pkt_dev = __pktgen_NN_threads(pn, ifname, REMOVE);
2056 if (pkt_dev == NULL)
2057 break; /* success */
2058
2059 mutex_unlock(&pktgen_thread_lock);
2060 pr_debug("%s: waiting for %s to disappear....\n",
2061 __func__, ifname);
2062 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try));
2063 mutex_lock(&pktgen_thread_lock);
2064
2065 if (++i >= max_tries) {
2066 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n",
2067 __func__, msec_per_try * i, ifname);
2068 break;
2069 }
2070
2071 }
2072
2073 mutex_unlock(&pktgen_thread_lock);
2074 }
2075
pktgen_change_name(const struct pktgen_net * pn,struct net_device * dev)2076 static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *dev)
2077 {
2078 struct pktgen_thread *t;
2079
2080 mutex_lock(&pktgen_thread_lock);
2081
2082 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
2083 struct pktgen_dev *pkt_dev;
2084
2085 if_lock(t);
2086 list_for_each_entry(pkt_dev, &t->if_list, list) {
2087 if (pkt_dev->odev != dev)
2088 continue;
2089
2090 proc_remove(pkt_dev->entry);
2091
2092 pkt_dev->entry = proc_create_data(dev->name, 0600,
2093 pn->proc_dir,
2094 &pktgen_if_proc_ops,
2095 pkt_dev);
2096 if (!pkt_dev->entry)
2097 pr_err("can't move proc entry for '%s'\n",
2098 dev->name);
2099 break;
2100 }
2101 if_unlock(t);
2102 }
2103 mutex_unlock(&pktgen_thread_lock);
2104 }
2105
pktgen_device_event(struct notifier_block * unused,unsigned long event,void * ptr)2106 static int pktgen_device_event(struct notifier_block *unused,
2107 unsigned long event, void *ptr)
2108 {
2109 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2110 struct pktgen_net *pn = net_generic(dev_net(dev), pg_net_id);
2111
2112 if (pn->pktgen_exiting)
2113 return NOTIFY_DONE;
2114
2115 /* It is OK that we do not hold the group lock right now,
2116 * as we run under the RTNL lock.
2117 */
2118
2119 switch (event) {
2120 case NETDEV_CHANGENAME:
2121 pktgen_change_name(pn, dev);
2122 break;
2123
2124 case NETDEV_UNREGISTER:
2125 pktgen_mark_device(pn, dev->name);
2126 break;
2127 }
2128
2129 return NOTIFY_DONE;
2130 }
2131
pktgen_dev_get_by_name(const struct pktgen_net * pn,struct pktgen_dev * pkt_dev,const char * ifname)2132 static struct net_device *pktgen_dev_get_by_name(const struct pktgen_net *pn,
2133 struct pktgen_dev *pkt_dev,
2134 const char *ifname)
2135 {
2136 char b[IFNAMSIZ+5];
2137 int i;
2138
2139 for (i = 0; ifname[i] != '@'; i++) {
2140 if (i == IFNAMSIZ)
2141 break;
2142
2143 b[i] = ifname[i];
2144 }
2145 b[i] = 0;
2146
2147 return dev_get_by_name(pn->net, b);
2148 }
2149
2150
2151 /* Associate pktgen_dev with a device. */
2152
pktgen_setup_dev(const struct pktgen_net * pn,struct pktgen_dev * pkt_dev,const char * ifname)2153 static int pktgen_setup_dev(const struct pktgen_net *pn,
2154 struct pktgen_dev *pkt_dev, const char *ifname)
2155 {
2156 struct net_device *odev;
2157 int err;
2158
2159 /* Clean old setups */
2160 if (pkt_dev->odev) {
2161 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
2162 pkt_dev->odev = NULL;
2163 }
2164
2165 odev = pktgen_dev_get_by_name(pn, pkt_dev, ifname);
2166 if (!odev) {
2167 pr_err("no such netdevice: \"%s\"\n", ifname);
2168 return -ENODEV;
2169 }
2170
2171 if (odev->type != ARPHRD_ETHER && odev->type != ARPHRD_LOOPBACK) {
2172 pr_err("not an ethernet or loopback device: \"%s\"\n", ifname);
2173 err = -EINVAL;
2174 } else if (!netif_running(odev)) {
2175 pr_err("device is down: \"%s\"\n", ifname);
2176 err = -ENETDOWN;
2177 } else {
2178 pkt_dev->odev = odev;
2179 netdev_tracker_alloc(odev, &pkt_dev->dev_tracker, GFP_KERNEL);
2180 return 0;
2181 }
2182
2183 dev_put(odev);
2184 return err;
2185 }
2186
2187 /* Read pkt_dev from the interface and set up internal pktgen_dev
2188 * structure to have the right information to create/send packets
2189 */
pktgen_setup_inject(struct pktgen_dev * pkt_dev)2190 static void pktgen_setup_inject(struct pktgen_dev *pkt_dev)
2191 {
2192 int ntxq;
2193
2194 if (!pkt_dev->odev) {
2195 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n");
2196 sprintf(pkt_dev->result,
2197 "ERROR: pkt_dev->odev == NULL in setup_inject.\n");
2198 return;
2199 }
2200
2201 /* make sure that we don't pick a non-existing transmit queue */
2202 ntxq = pkt_dev->odev->real_num_tx_queues;
2203
2204 if (ntxq <= pkt_dev->queue_map_min) {
2205 pr_warn("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
2206 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq,
2207 pkt_dev->odevname);
2208 pkt_dev->queue_map_min = (ntxq ?: 1) - 1;
2209 }
2210 if (pkt_dev->queue_map_max >= ntxq) {
2211 pr_warn("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
2212 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq,
2213 pkt_dev->odevname);
2214 pkt_dev->queue_map_max = (ntxq ?: 1) - 1;
2215 }
2216
2217 /* Default to the interface's mac if not explicitly set. */
2218
2219 if (is_zero_ether_addr(pkt_dev->src_mac))
2220 ether_addr_copy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr);
2221
2222 /* Set up Dest MAC */
2223 ether_addr_copy(&(pkt_dev->hh[0]), pkt_dev->dst_mac);
2224
2225 if (pkt_dev->flags & F_IPV6) {
2226 int i, set = 0, err = 1;
2227 struct inet6_dev *idev;
2228
2229 if (pkt_dev->min_pkt_size == 0) {
2230 pkt_dev->min_pkt_size = 14 + sizeof(struct ipv6hdr)
2231 + sizeof(struct udphdr)
2232 + sizeof(struct pktgen_hdr)
2233 + pkt_dev->pkt_overhead;
2234 }
2235
2236 for (i = 0; i < sizeof(struct in6_addr); i++)
2237 if (pkt_dev->cur_in6_saddr.s6_addr[i]) {
2238 set = 1;
2239 break;
2240 }
2241
2242 if (!set) {
2243
2244 /*
2245 * Use linklevel address if unconfigured.
2246 *
2247 * use ipv6_get_lladdr if/when it's get exported
2248 */
2249
2250 rcu_read_lock();
2251 idev = __in6_dev_get(pkt_dev->odev);
2252 if (idev) {
2253 struct inet6_ifaddr *ifp;
2254
2255 read_lock_bh(&idev->lock);
2256 list_for_each_entry(ifp, &idev->addr_list, if_list) {
2257 if ((ifp->scope & IFA_LINK) &&
2258 !(ifp->flags & IFA_F_TENTATIVE)) {
2259 pkt_dev->cur_in6_saddr = ifp->addr;
2260 err = 0;
2261 break;
2262 }
2263 }
2264 read_unlock_bh(&idev->lock);
2265 }
2266 rcu_read_unlock();
2267 if (err)
2268 pr_err("ERROR: IPv6 link address not available\n");
2269 }
2270 } else {
2271 if (pkt_dev->min_pkt_size == 0) {
2272 pkt_dev->min_pkt_size = 14 + sizeof(struct iphdr)
2273 + sizeof(struct udphdr)
2274 + sizeof(struct pktgen_hdr)
2275 + pkt_dev->pkt_overhead;
2276 }
2277
2278 pkt_dev->saddr_min = 0;
2279 pkt_dev->saddr_max = 0;
2280 if (strlen(pkt_dev->src_min) == 0) {
2281
2282 struct in_device *in_dev;
2283
2284 rcu_read_lock();
2285 in_dev = __in_dev_get_rcu(pkt_dev->odev);
2286 if (in_dev) {
2287 const struct in_ifaddr *ifa;
2288
2289 ifa = rcu_dereference(in_dev->ifa_list);
2290 if (ifa) {
2291 pkt_dev->saddr_min = ifa->ifa_address;
2292 pkt_dev->saddr_max = pkt_dev->saddr_min;
2293 }
2294 }
2295 rcu_read_unlock();
2296 } else {
2297 pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
2298 pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
2299 }
2300
2301 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
2302 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
2303 }
2304 /* Initialize current values. */
2305 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size;
2306 if (pkt_dev->min_pkt_size > pkt_dev->max_pkt_size)
2307 pkt_dev->max_pkt_size = pkt_dev->min_pkt_size;
2308
2309 pkt_dev->cur_dst_mac_offset = 0;
2310 pkt_dev->cur_src_mac_offset = 0;
2311 pkt_dev->cur_saddr = pkt_dev->saddr_min;
2312 pkt_dev->cur_daddr = pkt_dev->daddr_min;
2313 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
2314 pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
2315 pkt_dev->nflows = 0;
2316 }
2317
2318
spin(struct pktgen_dev * pkt_dev,ktime_t spin_until)2319 static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
2320 {
2321 ktime_t start_time, end_time;
2322 s64 remaining;
2323 struct hrtimer_sleeper t;
2324
2325 hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2326 hrtimer_set_expires(&t.timer, spin_until);
2327
2328 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
2329 if (remaining <= 0)
2330 goto out;
2331
2332 start_time = ktime_get();
2333 if (remaining < 100000) {
2334 /* for small delays (<100us), just loop until limit is reached */
2335 do {
2336 end_time = ktime_get();
2337 } while (ktime_compare(end_time, spin_until) < 0);
2338 } else {
2339 do {
2340 set_current_state(TASK_INTERRUPTIBLE);
2341 hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
2342
2343 if (likely(t.task))
2344 schedule();
2345
2346 hrtimer_cancel(&t.timer);
2347 } while (t.task && pkt_dev->running && !signal_pending(current));
2348 __set_current_state(TASK_RUNNING);
2349 end_time = ktime_get();
2350 }
2351
2352 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time));
2353 out:
2354 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay);
2355 destroy_hrtimer_on_stack(&t.timer);
2356 }
2357
set_pkt_overhead(struct pktgen_dev * pkt_dev)2358 static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev)
2359 {
2360 pkt_dev->pkt_overhead = 0;
2361 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32);
2362 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev);
2363 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev);
2364 }
2365
f_seen(const struct pktgen_dev * pkt_dev,int flow)2366 static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow)
2367 {
2368 return !!(pkt_dev->flows[flow].flags & F_INIT);
2369 }
2370
f_pick(struct pktgen_dev * pkt_dev)2371 static inline int f_pick(struct pktgen_dev *pkt_dev)
2372 {
2373 int flow = pkt_dev->curfl;
2374
2375 if (pkt_dev->flags & F_FLOW_SEQ) {
2376 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) {
2377 /* reset time */
2378 pkt_dev->flows[flow].count = 0;
2379 pkt_dev->flows[flow].flags = 0;
2380 pkt_dev->curfl += 1;
2381 if (pkt_dev->curfl >= pkt_dev->cflows)
2382 pkt_dev->curfl = 0; /*reset */
2383 }
2384 } else {
2385 flow = get_random_u32_below(pkt_dev->cflows);
2386 pkt_dev->curfl = flow;
2387
2388 if (pkt_dev->flows[flow].count > pkt_dev->lflow) {
2389 pkt_dev->flows[flow].count = 0;
2390 pkt_dev->flows[flow].flags = 0;
2391 }
2392 }
2393
2394 return pkt_dev->curfl;
2395 }
2396
2397
2398 /* If there was already an IPSEC SA, we keep it as is, else
2399 * we go look for it ...
2400 */
2401 #define DUMMY_MARK 0
get_ipsec_sa(struct pktgen_dev * pkt_dev,int flow)2402 static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow)
2403 {
2404 #ifdef CONFIG_XFRM
2405 struct xfrm_state *x = pkt_dev->flows[flow].x;
2406 struct pktgen_net *pn = net_generic(dev_net(pkt_dev->odev), pg_net_id);
2407 if (!x) {
2408
2409 if (pkt_dev->spi) {
2410 /* We need as quick as possible to find the right SA
2411 * Searching with minimum criteria to achieve, this.
2412 */
2413 x = xfrm_state_lookup_byspi(pn->net, htonl(pkt_dev->spi), AF_INET);
2414 } else {
2415 /* slow path: we don't already have xfrm_state */
2416 x = xfrm_stateonly_find(pn->net, DUMMY_MARK, 0,
2417 (xfrm_address_t *)&pkt_dev->cur_daddr,
2418 (xfrm_address_t *)&pkt_dev->cur_saddr,
2419 AF_INET,
2420 pkt_dev->ipsmode,
2421 pkt_dev->ipsproto, 0);
2422 }
2423 if (x) {
2424 pkt_dev->flows[flow].x = x;
2425 set_pkt_overhead(pkt_dev);
2426 pkt_dev->pkt_overhead += x->props.header_len;
2427 }
2428
2429 }
2430 #endif
2431 }
set_cur_queue_map(struct pktgen_dev * pkt_dev)2432 static void set_cur_queue_map(struct pktgen_dev *pkt_dev)
2433 {
2434 if (pkt_dev->flags & F_QUEUE_MAP_CPU)
2435 pkt_dev->cur_queue_map = smp_processor_id();
2436
2437 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) {
2438 __u16 t;
2439 if (pkt_dev->flags & F_QUEUE_MAP_RND) {
2440 t = get_random_u32_inclusive(pkt_dev->queue_map_min,
2441 pkt_dev->queue_map_max);
2442 } else {
2443 t = pkt_dev->cur_queue_map + 1;
2444 if (t > pkt_dev->queue_map_max)
2445 t = pkt_dev->queue_map_min;
2446 }
2447 pkt_dev->cur_queue_map = t;
2448 }
2449 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues;
2450 }
2451
2452 /* Increment/randomize headers according to flags and current values
2453 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst
2454 */
mod_cur_headers(struct pktgen_dev * pkt_dev)2455 static void mod_cur_headers(struct pktgen_dev *pkt_dev)
2456 {
2457 __u32 imn;
2458 __u32 imx;
2459 int flow = 0;
2460
2461 if (pkt_dev->cflows)
2462 flow = f_pick(pkt_dev);
2463
2464 /* Deal with source MAC */
2465 if (pkt_dev->src_mac_count > 1) {
2466 __u32 mc;
2467 __u32 tmp;
2468
2469 if (pkt_dev->flags & F_MACSRC_RND)
2470 mc = get_random_u32_below(pkt_dev->src_mac_count);
2471 else {
2472 mc = pkt_dev->cur_src_mac_offset++;
2473 if (pkt_dev->cur_src_mac_offset >=
2474 pkt_dev->src_mac_count)
2475 pkt_dev->cur_src_mac_offset = 0;
2476 }
2477
2478 tmp = pkt_dev->src_mac[5] + (mc & 0xFF);
2479 pkt_dev->hh[11] = tmp;
2480 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
2481 pkt_dev->hh[10] = tmp;
2482 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
2483 pkt_dev->hh[9] = tmp;
2484 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
2485 pkt_dev->hh[8] = tmp;
2486 tmp = (pkt_dev->src_mac[1] + (tmp >> 8));
2487 pkt_dev->hh[7] = tmp;
2488 }
2489
2490 /* Deal with Destination MAC */
2491 if (pkt_dev->dst_mac_count > 1) {
2492 __u32 mc;
2493 __u32 tmp;
2494
2495 if (pkt_dev->flags & F_MACDST_RND)
2496 mc = get_random_u32_below(pkt_dev->dst_mac_count);
2497
2498 else {
2499 mc = pkt_dev->cur_dst_mac_offset++;
2500 if (pkt_dev->cur_dst_mac_offset >=
2501 pkt_dev->dst_mac_count) {
2502 pkt_dev->cur_dst_mac_offset = 0;
2503 }
2504 }
2505
2506 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF);
2507 pkt_dev->hh[5] = tmp;
2508 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
2509 pkt_dev->hh[4] = tmp;
2510 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
2511 pkt_dev->hh[3] = tmp;
2512 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
2513 pkt_dev->hh[2] = tmp;
2514 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8));
2515 pkt_dev->hh[1] = tmp;
2516 }
2517
2518 if (pkt_dev->flags & F_MPLS_RND) {
2519 unsigned int i;
2520 for (i = 0; i < pkt_dev->nr_labels; i++)
2521 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM)
2522 pkt_dev->labels[i] = MPLS_STACK_BOTTOM |
2523 ((__force __be32)get_random_u32() &
2524 htonl(0x000fffff));
2525 }
2526
2527 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) {
2528 pkt_dev->vlan_id = get_random_u32_below(4096);
2529 }
2530
2531 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) {
2532 pkt_dev->svlan_id = get_random_u32_below(4096);
2533 }
2534
2535 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) {
2536 if (pkt_dev->flags & F_UDPSRC_RND)
2537 pkt_dev->cur_udp_src = get_random_u32_inclusive(pkt_dev->udp_src_min,
2538 pkt_dev->udp_src_max - 1);
2539
2540 else {
2541 pkt_dev->cur_udp_src++;
2542 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max)
2543 pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
2544 }
2545 }
2546
2547 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) {
2548 if (pkt_dev->flags & F_UDPDST_RND) {
2549 pkt_dev->cur_udp_dst = get_random_u32_inclusive(pkt_dev->udp_dst_min,
2550 pkt_dev->udp_dst_max - 1);
2551 } else {
2552 pkt_dev->cur_udp_dst++;
2553 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max)
2554 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
2555 }
2556 }
2557
2558 if (!(pkt_dev->flags & F_IPV6)) {
2559
2560 imn = ntohl(pkt_dev->saddr_min);
2561 imx = ntohl(pkt_dev->saddr_max);
2562 if (imn < imx) {
2563 __u32 t;
2564 if (pkt_dev->flags & F_IPSRC_RND)
2565 t = get_random_u32_inclusive(imn, imx - 1);
2566 else {
2567 t = ntohl(pkt_dev->cur_saddr);
2568 t++;
2569 if (t > imx)
2570 t = imn;
2571
2572 }
2573 pkt_dev->cur_saddr = htonl(t);
2574 }
2575
2576 if (pkt_dev->cflows && f_seen(pkt_dev, flow)) {
2577 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr;
2578 } else {
2579 imn = ntohl(pkt_dev->daddr_min);
2580 imx = ntohl(pkt_dev->daddr_max);
2581 if (imn < imx) {
2582 __u32 t;
2583 __be32 s;
2584 if (pkt_dev->flags & F_IPDST_RND) {
2585
2586 do {
2587 t = get_random_u32_inclusive(imn, imx - 1);
2588 s = htonl(t);
2589 } while (ipv4_is_loopback(s) ||
2590 ipv4_is_multicast(s) ||
2591 ipv4_is_lbcast(s) ||
2592 ipv4_is_zeronet(s) ||
2593 ipv4_is_local_multicast(s));
2594 pkt_dev->cur_daddr = s;
2595 } else {
2596 t = ntohl(pkt_dev->cur_daddr);
2597 t++;
2598 if (t > imx) {
2599 t = imn;
2600 }
2601 pkt_dev->cur_daddr = htonl(t);
2602 }
2603 }
2604 if (pkt_dev->cflows) {
2605 pkt_dev->flows[flow].flags |= F_INIT;
2606 pkt_dev->flows[flow].cur_daddr =
2607 pkt_dev->cur_daddr;
2608 if (pkt_dev->flags & F_IPSEC)
2609 get_ipsec_sa(pkt_dev, flow);
2610 pkt_dev->nflows++;
2611 }
2612 }
2613 } else { /* IPV6 * */
2614
2615 if (!ipv6_addr_any(&pkt_dev->min_in6_daddr)) {
2616 int i;
2617
2618 /* Only random destinations yet */
2619
2620 for (i = 0; i < 4; i++) {
2621 pkt_dev->cur_in6_daddr.s6_addr32[i] =
2622 (((__force __be32)get_random_u32() |
2623 pkt_dev->min_in6_daddr.s6_addr32[i]) &
2624 pkt_dev->max_in6_daddr.s6_addr32[i]);
2625 }
2626 }
2627 }
2628
2629 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) {
2630 __u32 t;
2631 if (pkt_dev->flags & F_TXSIZE_RND) {
2632 t = get_random_u32_inclusive(pkt_dev->min_pkt_size,
2633 pkt_dev->max_pkt_size - 1);
2634 } else {
2635 t = pkt_dev->cur_pkt_size + 1;
2636 if (t > pkt_dev->max_pkt_size)
2637 t = pkt_dev->min_pkt_size;
2638 }
2639 pkt_dev->cur_pkt_size = t;
2640 } else if (pkt_dev->n_imix_entries > 0) {
2641 struct imix_pkt *entry;
2642 __u32 t = get_random_u32_below(IMIX_PRECISION);
2643 __u8 entry_index = pkt_dev->imix_distribution[t];
2644
2645 entry = &pkt_dev->imix_entries[entry_index];
2646 entry->count_so_far++;
2647 pkt_dev->cur_pkt_size = entry->size;
2648 }
2649
2650 set_cur_queue_map(pkt_dev);
2651
2652 pkt_dev->flows[flow].count++;
2653 }
2654
fill_imix_distribution(struct pktgen_dev * pkt_dev)2655 static void fill_imix_distribution(struct pktgen_dev *pkt_dev)
2656 {
2657 int cumulative_probabilites[MAX_IMIX_ENTRIES];
2658 int j = 0;
2659 __u64 cumulative_prob = 0;
2660 __u64 total_weight = 0;
2661 int i = 0;
2662
2663 for (i = 0; i < pkt_dev->n_imix_entries; i++)
2664 total_weight += pkt_dev->imix_entries[i].weight;
2665
2666 /* Fill cumulative_probabilites with sum of normalized probabilities */
2667 for (i = 0; i < pkt_dev->n_imix_entries - 1; i++) {
2668 cumulative_prob += div64_u64(pkt_dev->imix_entries[i].weight *
2669 IMIX_PRECISION,
2670 total_weight);
2671 cumulative_probabilites[i] = cumulative_prob;
2672 }
2673 cumulative_probabilites[pkt_dev->n_imix_entries - 1] = 100;
2674
2675 for (i = 0; i < IMIX_PRECISION; i++) {
2676 if (i == cumulative_probabilites[j])
2677 j++;
2678 pkt_dev->imix_distribution[i] = j;
2679 }
2680 }
2681
2682 #ifdef CONFIG_XFRM
2683 static u32 pktgen_dst_metrics[RTAX_MAX + 1] = {
2684
2685 [RTAX_HOPLIMIT] = 0x5, /* Set a static hoplimit */
2686 };
2687
pktgen_output_ipsec(struct sk_buff * skb,struct pktgen_dev * pkt_dev)2688 static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev)
2689 {
2690 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
2691 int err = 0;
2692 struct net *net = dev_net(pkt_dev->odev);
2693
2694 if (!x)
2695 return 0;
2696 /* XXX: we dont support tunnel mode for now until
2697 * we resolve the dst issue */
2698 if ((x->props.mode != XFRM_MODE_TRANSPORT) && (pkt_dev->spi == 0))
2699 return 0;
2700
2701 /* But when user specify an valid SPI, transformation
2702 * supports both transport/tunnel mode + ESP/AH type.
2703 */
2704 if ((x->props.mode == XFRM_MODE_TUNNEL) && (pkt_dev->spi != 0))
2705 skb->_skb_refdst = (unsigned long)&pkt_dev->xdst.u.dst | SKB_DST_NOREF;
2706
2707 rcu_read_lock_bh();
2708 err = pktgen_xfrm_outer_mode_output(x, skb);
2709 rcu_read_unlock_bh();
2710 if (err) {
2711 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
2712 goto error;
2713 }
2714 err = x->type->output(x, skb);
2715 if (err) {
2716 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
2717 goto error;
2718 }
2719 spin_lock_bh(&x->lock);
2720 x->curlft.bytes += skb->len;
2721 x->curlft.packets++;
2722 spin_unlock_bh(&x->lock);
2723 error:
2724 return err;
2725 }
2726
free_SAs(struct pktgen_dev * pkt_dev)2727 static void free_SAs(struct pktgen_dev *pkt_dev)
2728 {
2729 if (pkt_dev->cflows) {
2730 /* let go of the SAs if we have them */
2731 int i;
2732 for (i = 0; i < pkt_dev->cflows; i++) {
2733 struct xfrm_state *x = pkt_dev->flows[i].x;
2734 if (x) {
2735 xfrm_state_put(x);
2736 pkt_dev->flows[i].x = NULL;
2737 }
2738 }
2739 }
2740 }
2741
process_ipsec(struct pktgen_dev * pkt_dev,struct sk_buff * skb,__be16 protocol)2742 static int process_ipsec(struct pktgen_dev *pkt_dev,
2743 struct sk_buff *skb, __be16 protocol)
2744 {
2745 if (pkt_dev->flags & F_IPSEC) {
2746 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
2747 int nhead = 0;
2748 if (x) {
2749 struct ethhdr *eth;
2750 struct iphdr *iph;
2751 int ret;
2752
2753 nhead = x->props.header_len - skb_headroom(skb);
2754 if (nhead > 0) {
2755 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC);
2756 if (ret < 0) {
2757 pr_err("Error expanding ipsec packet %d\n",
2758 ret);
2759 goto err;
2760 }
2761 }
2762
2763 /* ipsec is not expecting ll header */
2764 skb_pull(skb, ETH_HLEN);
2765 ret = pktgen_output_ipsec(skb, pkt_dev);
2766 if (ret) {
2767 pr_err("Error creating ipsec packet %d\n", ret);
2768 goto err;
2769 }
2770 /* restore ll */
2771 eth = skb_push(skb, ETH_HLEN);
2772 memcpy(eth, pkt_dev->hh, 2 * ETH_ALEN);
2773 eth->h_proto = protocol;
2774
2775 /* Update IPv4 header len as well as checksum value */
2776 iph = ip_hdr(skb);
2777 iph->tot_len = htons(skb->len - ETH_HLEN);
2778 ip_send_check(iph);
2779 }
2780 }
2781 return 1;
2782 err:
2783 kfree_skb(skb);
2784 return 0;
2785 }
2786 #endif
2787
mpls_push(__be32 * mpls,struct pktgen_dev * pkt_dev)2788 static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev)
2789 {
2790 unsigned int i;
2791 for (i = 0; i < pkt_dev->nr_labels; i++)
2792 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM;
2793
2794 mpls--;
2795 *mpls |= MPLS_STACK_BOTTOM;
2796 }
2797
build_tci(unsigned int id,unsigned int cfi,unsigned int prio)2798 static inline __be16 build_tci(unsigned int id, unsigned int cfi,
2799 unsigned int prio)
2800 {
2801 return htons(id | (cfi << 12) | (prio << 13));
2802 }
2803
pktgen_finalize_skb(struct pktgen_dev * pkt_dev,struct sk_buff * skb,int datalen)2804 static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb,
2805 int datalen)
2806 {
2807 struct timespec64 timestamp;
2808 struct pktgen_hdr *pgh;
2809
2810 pgh = skb_put(skb, sizeof(*pgh));
2811 datalen -= sizeof(*pgh);
2812
2813 if (pkt_dev->nfrags <= 0) {
2814 skb_put_zero(skb, datalen);
2815 } else {
2816 int frags = pkt_dev->nfrags;
2817 int i, len;
2818 int frag_len;
2819
2820
2821 if (frags > MAX_SKB_FRAGS)
2822 frags = MAX_SKB_FRAGS;
2823 len = datalen - frags * PAGE_SIZE;
2824 if (len > 0) {
2825 skb_put_zero(skb, len);
2826 datalen = frags * PAGE_SIZE;
2827 }
2828
2829 i = 0;
2830 frag_len = (datalen/frags) < PAGE_SIZE ?
2831 (datalen/frags) : PAGE_SIZE;
2832 while (datalen > 0) {
2833 if (unlikely(!pkt_dev->page)) {
2834 int node = numa_node_id();
2835
2836 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE))
2837 node = pkt_dev->node;
2838 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2839 if (!pkt_dev->page)
2840 break;
2841 }
2842 get_page(pkt_dev->page);
2843
2844 /*last fragment, fill rest of data*/
2845 if (i == (frags - 1))
2846 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i],
2847 pkt_dev->page, 0,
2848 (datalen < PAGE_SIZE ?
2849 datalen : PAGE_SIZE));
2850 else
2851 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i],
2852 pkt_dev->page, 0, frag_len);
2853
2854 datalen -= skb_frag_size(&skb_shinfo(skb)->frags[i]);
2855 skb->len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2856 skb->data_len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2857 i++;
2858 skb_shinfo(skb)->nr_frags = i;
2859 }
2860 }
2861
2862 /* Stamp the time, and sequence number,
2863 * convert them to network byte order
2864 */
2865 pgh->pgh_magic = htonl(PKTGEN_MAGIC);
2866 pgh->seq_num = htonl(pkt_dev->seq_num);
2867
2868 if (pkt_dev->flags & F_NO_TIMESTAMP) {
2869 pgh->tv_sec = 0;
2870 pgh->tv_usec = 0;
2871 } else {
2872 /*
2873 * pgh->tv_sec wraps in y2106 when interpreted as unsigned
2874 * as done by wireshark, or y2038 when interpreted as signed.
2875 * This is probably harmless, but if anyone wants to improve
2876 * it, we could introduce a variant that puts 64-bit nanoseconds
2877 * into the respective header bytes.
2878 * This would also be slightly faster to read.
2879 */
2880 ktime_get_real_ts64(×tamp);
2881 pgh->tv_sec = htonl(timestamp.tv_sec);
2882 pgh->tv_usec = htonl(timestamp.tv_nsec / NSEC_PER_USEC);
2883 }
2884 }
2885
pktgen_alloc_skb(struct net_device * dev,struct pktgen_dev * pkt_dev)2886 static struct sk_buff *pktgen_alloc_skb(struct net_device *dev,
2887 struct pktgen_dev *pkt_dev)
2888 {
2889 unsigned int extralen = LL_RESERVED_SPACE(dev);
2890 struct sk_buff *skb = NULL;
2891 unsigned int size;
2892
2893 size = pkt_dev->cur_pkt_size + 64 + extralen + pkt_dev->pkt_overhead;
2894 if (pkt_dev->flags & F_NODE) {
2895 int node = pkt_dev->node >= 0 ? pkt_dev->node : numa_node_id();
2896
2897 skb = __alloc_skb(NET_SKB_PAD + size, GFP_NOWAIT, 0, node);
2898 if (likely(skb)) {
2899 skb_reserve(skb, NET_SKB_PAD);
2900 skb->dev = dev;
2901 }
2902 } else {
2903 skb = __netdev_alloc_skb(dev, size, GFP_NOWAIT);
2904 }
2905
2906 /* the caller pre-fetches from skb->data and reserves for the mac hdr */
2907 if (likely(skb))
2908 skb_reserve(skb, extralen - 16);
2909
2910 return skb;
2911 }
2912
fill_packet_ipv4(struct net_device * odev,struct pktgen_dev * pkt_dev)2913 static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2914 struct pktgen_dev *pkt_dev)
2915 {
2916 struct sk_buff *skb = NULL;
2917 __u8 *eth;
2918 struct udphdr *udph;
2919 int datalen, iplen;
2920 struct iphdr *iph;
2921 __be16 protocol = htons(ETH_P_IP);
2922 __be32 *mpls;
2923 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
2924 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
2925 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
2926 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
2927 u16 queue_map;
2928
2929 if (pkt_dev->nr_labels)
2930 protocol = htons(ETH_P_MPLS_UC);
2931
2932 if (pkt_dev->vlan_id != 0xffff)
2933 protocol = htons(ETH_P_8021Q);
2934
2935 /* Update any of the values, used when we're incrementing various
2936 * fields.
2937 */
2938 mod_cur_headers(pkt_dev);
2939 queue_map = pkt_dev->cur_queue_map;
2940
2941 skb = pktgen_alloc_skb(odev, pkt_dev);
2942 if (!skb) {
2943 sprintf(pkt_dev->result, "No memory");
2944 return NULL;
2945 }
2946
2947 prefetchw(skb->data);
2948 skb_reserve(skb, 16);
2949
2950 /* Reserve for ethernet and IP header */
2951 eth = skb_push(skb, 14);
2952 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32));
2953 if (pkt_dev->nr_labels)
2954 mpls_push(mpls, pkt_dev);
2955
2956 if (pkt_dev->vlan_id != 0xffff) {
2957 if (pkt_dev->svlan_id != 0xffff) {
2958 svlan_tci = skb_put(skb, sizeof(__be16));
2959 *svlan_tci = build_tci(pkt_dev->svlan_id,
2960 pkt_dev->svlan_cfi,
2961 pkt_dev->svlan_p);
2962 svlan_encapsulated_proto = skb_put(skb,
2963 sizeof(__be16));
2964 *svlan_encapsulated_proto = htons(ETH_P_8021Q);
2965 }
2966 vlan_tci = skb_put(skb, sizeof(__be16));
2967 *vlan_tci = build_tci(pkt_dev->vlan_id,
2968 pkt_dev->vlan_cfi,
2969 pkt_dev->vlan_p);
2970 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16));
2971 *vlan_encapsulated_proto = htons(ETH_P_IP);
2972 }
2973
2974 skb_reset_mac_header(skb);
2975 skb_set_network_header(skb, skb->len);
2976 iph = skb_put(skb, sizeof(struct iphdr));
2977
2978 skb_set_transport_header(skb, skb->len);
2979 udph = skb_put(skb, sizeof(struct udphdr));
2980 skb_set_queue_mapping(skb, queue_map);
2981 skb->priority = pkt_dev->skb_priority;
2982
2983 memcpy(eth, pkt_dev->hh, 12);
2984 *(__be16 *) & eth[12] = protocol;
2985
2986 /* Eth + IPh + UDPh + mpls */
2987 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 -
2988 pkt_dev->pkt_overhead;
2989 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr))
2990 datalen = sizeof(struct pktgen_hdr);
2991
2992 udph->source = htons(pkt_dev->cur_udp_src);
2993 udph->dest = htons(pkt_dev->cur_udp_dst);
2994 udph->len = htons(datalen + 8); /* DATA + udphdr */
2995 udph->check = 0;
2996
2997 iph->ihl = 5;
2998 iph->version = 4;
2999 iph->ttl = 32;
3000 iph->tos = pkt_dev->tos;
3001 iph->protocol = IPPROTO_UDP; /* UDP */
3002 iph->saddr = pkt_dev->cur_saddr;
3003 iph->daddr = pkt_dev->cur_daddr;
3004 iph->id = htons(pkt_dev->ip_id);
3005 pkt_dev->ip_id++;
3006 iph->frag_off = 0;
3007 iplen = 20 + 8 + datalen;
3008 iph->tot_len = htons(iplen);
3009 ip_send_check(iph);
3010 skb->protocol = protocol;
3011 skb->dev = odev;
3012 skb->pkt_type = PACKET_HOST;
3013
3014 pktgen_finalize_skb(pkt_dev, skb, datalen);
3015
3016 if (!(pkt_dev->flags & F_UDPCSUM)) {
3017 skb->ip_summed = CHECKSUM_NONE;
3018 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)) {
3019 skb->ip_summed = CHECKSUM_PARTIAL;
3020 skb->csum = 0;
3021 udp4_hwcsum(skb, iph->saddr, iph->daddr);
3022 } else {
3023 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), datalen + 8, 0);
3024
3025 /* add protocol-dependent pseudo-header */
3026 udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
3027 datalen + 8, IPPROTO_UDP, csum);
3028
3029 if (udph->check == 0)
3030 udph->check = CSUM_MANGLED_0;
3031 }
3032
3033 #ifdef CONFIG_XFRM
3034 if (!process_ipsec(pkt_dev, skb, protocol))
3035 return NULL;
3036 #endif
3037
3038 return skb;
3039 }
3040
fill_packet_ipv6(struct net_device * odev,struct pktgen_dev * pkt_dev)3041 static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
3042 struct pktgen_dev *pkt_dev)
3043 {
3044 struct sk_buff *skb = NULL;
3045 __u8 *eth;
3046 struct udphdr *udph;
3047 int datalen, udplen;
3048 struct ipv6hdr *iph;
3049 __be16 protocol = htons(ETH_P_IPV6);
3050 __be32 *mpls;
3051 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
3052 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
3053 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
3054 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
3055 u16 queue_map;
3056
3057 if (pkt_dev->nr_labels)
3058 protocol = htons(ETH_P_MPLS_UC);
3059
3060 if (pkt_dev->vlan_id != 0xffff)
3061 protocol = htons(ETH_P_8021Q);
3062
3063 /* Update any of the values, used when we're incrementing various
3064 * fields.
3065 */
3066 mod_cur_headers(pkt_dev);
3067 queue_map = pkt_dev->cur_queue_map;
3068
3069 skb = pktgen_alloc_skb(odev, pkt_dev);
3070 if (!skb) {
3071 sprintf(pkt_dev->result, "No memory");
3072 return NULL;
3073 }
3074
3075 prefetchw(skb->data);
3076 skb_reserve(skb, 16);
3077
3078 /* Reserve for ethernet and IP header */
3079 eth = skb_push(skb, 14);
3080 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32));
3081 if (pkt_dev->nr_labels)
3082 mpls_push(mpls, pkt_dev);
3083
3084 if (pkt_dev->vlan_id != 0xffff) {
3085 if (pkt_dev->svlan_id != 0xffff) {
3086 svlan_tci = skb_put(skb, sizeof(__be16));
3087 *svlan_tci = build_tci(pkt_dev->svlan_id,
3088 pkt_dev->svlan_cfi,
3089 pkt_dev->svlan_p);
3090 svlan_encapsulated_proto = skb_put(skb,
3091 sizeof(__be16));
3092 *svlan_encapsulated_proto = htons(ETH_P_8021Q);
3093 }
3094 vlan_tci = skb_put(skb, sizeof(__be16));
3095 *vlan_tci = build_tci(pkt_dev->vlan_id,
3096 pkt_dev->vlan_cfi,
3097 pkt_dev->vlan_p);
3098 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16));
3099 *vlan_encapsulated_proto = htons(ETH_P_IPV6);
3100 }
3101
3102 skb_reset_mac_header(skb);
3103 skb_set_network_header(skb, skb->len);
3104 iph = skb_put(skb, sizeof(struct ipv6hdr));
3105
3106 skb_set_transport_header(skb, skb->len);
3107 udph = skb_put(skb, sizeof(struct udphdr));
3108 skb_set_queue_mapping(skb, queue_map);
3109 skb->priority = pkt_dev->skb_priority;
3110
3111 memcpy(eth, pkt_dev->hh, 12);
3112 *(__be16 *) ð[12] = protocol;
3113
3114 /* Eth + IPh + UDPh + mpls */
3115 datalen = pkt_dev->cur_pkt_size - 14 -
3116 sizeof(struct ipv6hdr) - sizeof(struct udphdr) -
3117 pkt_dev->pkt_overhead;
3118
3119 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) {
3120 datalen = sizeof(struct pktgen_hdr);
3121 net_info_ratelimited("increased datalen to %d\n", datalen);
3122 }
3123
3124 udplen = datalen + sizeof(struct udphdr);
3125 udph->source = htons(pkt_dev->cur_udp_src);
3126 udph->dest = htons(pkt_dev->cur_udp_dst);
3127 udph->len = htons(udplen);
3128 udph->check = 0;
3129
3130 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */
3131
3132 if (pkt_dev->traffic_class) {
3133 /* Version + traffic class + flow (0) */
3134 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20));
3135 }
3136
3137 iph->hop_limit = 32;
3138
3139 iph->payload_len = htons(udplen);
3140 iph->nexthdr = IPPROTO_UDP;
3141
3142 iph->daddr = pkt_dev->cur_in6_daddr;
3143 iph->saddr = pkt_dev->cur_in6_saddr;
3144
3145 skb->protocol = protocol;
3146 skb->dev = odev;
3147 skb->pkt_type = PACKET_HOST;
3148
3149 pktgen_finalize_skb(pkt_dev, skb, datalen);
3150
3151 if (!(pkt_dev->flags & F_UDPCSUM)) {
3152 skb->ip_summed = CHECKSUM_NONE;
3153 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM)) {
3154 skb->ip_summed = CHECKSUM_PARTIAL;
3155 skb->csum_start = skb_transport_header(skb) - skb->head;
3156 skb->csum_offset = offsetof(struct udphdr, check);
3157 udph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, 0);
3158 } else {
3159 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), udplen, 0);
3160
3161 /* add protocol-dependent pseudo-header */
3162 udph->check = csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, csum);
3163
3164 if (udph->check == 0)
3165 udph->check = CSUM_MANGLED_0;
3166 }
3167
3168 return skb;
3169 }
3170
fill_packet(struct net_device * odev,struct pktgen_dev * pkt_dev)3171 static struct sk_buff *fill_packet(struct net_device *odev,
3172 struct pktgen_dev *pkt_dev)
3173 {
3174 if (pkt_dev->flags & F_IPV6)
3175 return fill_packet_ipv6(odev, pkt_dev);
3176 else
3177 return fill_packet_ipv4(odev, pkt_dev);
3178 }
3179
pktgen_clear_counters(struct pktgen_dev * pkt_dev)3180 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev)
3181 {
3182 pkt_dev->seq_num = 1;
3183 pkt_dev->idle_acc = 0;
3184 pkt_dev->sofar = 0;
3185 pkt_dev->tx_bytes = 0;
3186 pkt_dev->errors = 0;
3187 }
3188
3189 /* Set up structure for sending pkts, clear counters */
3190
pktgen_run(struct pktgen_thread * t)3191 static void pktgen_run(struct pktgen_thread *t)
3192 {
3193 struct pktgen_dev *pkt_dev;
3194 int started = 0;
3195
3196 func_enter();
3197
3198 rcu_read_lock();
3199 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3200
3201 /*
3202 * setup odev and create initial packet.
3203 */
3204 pktgen_setup_inject(pkt_dev);
3205
3206 if (pkt_dev->odev) {
3207 pktgen_clear_counters(pkt_dev);
3208 pkt_dev->skb = NULL;
3209 pkt_dev->started_at = pkt_dev->next_tx = ktime_get();
3210
3211 set_pkt_overhead(pkt_dev);
3212
3213 strcpy(pkt_dev->result, "Starting");
3214 pkt_dev->running = 1; /* Cranke yeself! */
3215 started++;
3216 } else
3217 strcpy(pkt_dev->result, "Error starting");
3218 }
3219 rcu_read_unlock();
3220 if (started)
3221 t->control &= ~(T_STOP);
3222 }
3223
pktgen_handle_all_threads(struct pktgen_net * pn,u32 flags)3224 static void pktgen_handle_all_threads(struct pktgen_net *pn, u32 flags)
3225 {
3226 struct pktgen_thread *t;
3227
3228 mutex_lock(&pktgen_thread_lock);
3229
3230 list_for_each_entry(t, &pn->pktgen_threads, th_list)
3231 t->control |= (flags);
3232
3233 mutex_unlock(&pktgen_thread_lock);
3234 }
3235
pktgen_stop_all_threads(struct pktgen_net * pn)3236 static void pktgen_stop_all_threads(struct pktgen_net *pn)
3237 {
3238 func_enter();
3239
3240 pktgen_handle_all_threads(pn, T_STOP);
3241 }
3242
thread_is_running(const struct pktgen_thread * t)3243 static int thread_is_running(const struct pktgen_thread *t)
3244 {
3245 const struct pktgen_dev *pkt_dev;
3246
3247 rcu_read_lock();
3248 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
3249 if (pkt_dev->running) {
3250 rcu_read_unlock();
3251 return 1;
3252 }
3253 rcu_read_unlock();
3254 return 0;
3255 }
3256
pktgen_wait_thread_run(struct pktgen_thread * t)3257 static int pktgen_wait_thread_run(struct pktgen_thread *t)
3258 {
3259 while (thread_is_running(t)) {
3260
3261 /* note: 't' will still be around even after the unlock/lock
3262 * cycle because pktgen_thread threads are only cleared at
3263 * net exit
3264 */
3265 mutex_unlock(&pktgen_thread_lock);
3266 msleep_interruptible(100);
3267 mutex_lock(&pktgen_thread_lock);
3268
3269 if (signal_pending(current))
3270 goto signal;
3271 }
3272 return 1;
3273 signal:
3274 return 0;
3275 }
3276
pktgen_wait_all_threads_run(struct pktgen_net * pn)3277 static int pktgen_wait_all_threads_run(struct pktgen_net *pn)
3278 {
3279 struct pktgen_thread *t;
3280 int sig = 1;
3281
3282 /* prevent from racing with rmmod */
3283 if (!try_module_get(THIS_MODULE))
3284 return sig;
3285
3286 mutex_lock(&pktgen_thread_lock);
3287
3288 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
3289 sig = pktgen_wait_thread_run(t);
3290 if (sig == 0)
3291 break;
3292 }
3293
3294 if (sig == 0)
3295 list_for_each_entry(t, &pn->pktgen_threads, th_list)
3296 t->control |= (T_STOP);
3297
3298 mutex_unlock(&pktgen_thread_lock);
3299 module_put(THIS_MODULE);
3300 return sig;
3301 }
3302
pktgen_run_all_threads(struct pktgen_net * pn)3303 static void pktgen_run_all_threads(struct pktgen_net *pn)
3304 {
3305 func_enter();
3306
3307 pktgen_handle_all_threads(pn, T_RUN);
3308
3309 /* Propagate thread->control */
3310 schedule_timeout_interruptible(msecs_to_jiffies(125));
3311
3312 pktgen_wait_all_threads_run(pn);
3313 }
3314
pktgen_reset_all_threads(struct pktgen_net * pn)3315 static void pktgen_reset_all_threads(struct pktgen_net *pn)
3316 {
3317 func_enter();
3318
3319 pktgen_handle_all_threads(pn, T_REMDEVALL);
3320
3321 /* Propagate thread->control */
3322 schedule_timeout_interruptible(msecs_to_jiffies(125));
3323
3324 pktgen_wait_all_threads_run(pn);
3325 }
3326
show_results(struct pktgen_dev * pkt_dev,int nr_frags)3327 static void show_results(struct pktgen_dev *pkt_dev, int nr_frags)
3328 {
3329 __u64 bps, mbps, pps;
3330 char *p = pkt_dev->result;
3331 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at,
3332 pkt_dev->started_at);
3333 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc);
3334
3335 p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n",
3336 (unsigned long long)ktime_to_us(elapsed),
3337 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)),
3338 (unsigned long long)ktime_to_us(idle),
3339 (unsigned long long)pkt_dev->sofar,
3340 pkt_dev->cur_pkt_size, nr_frags);
3341
3342 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC,
3343 ktime_to_ns(elapsed));
3344
3345 if (pkt_dev->n_imix_entries > 0) {
3346 int i;
3347 struct imix_pkt *entry;
3348
3349 bps = 0;
3350 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
3351 entry = &pkt_dev->imix_entries[i];
3352 bps += entry->size * entry->count_so_far;
3353 }
3354 bps = div64_u64(bps * 8 * NSEC_PER_SEC, ktime_to_ns(elapsed));
3355 } else {
3356 bps = pps * 8 * pkt_dev->cur_pkt_size;
3357 }
3358
3359 mbps = bps;
3360 do_div(mbps, 1000000);
3361 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu",
3362 (unsigned long long)pps,
3363 (unsigned long long)mbps,
3364 (unsigned long long)bps,
3365 (unsigned long long)pkt_dev->errors);
3366 }
3367
3368 /* Set stopped-at timer, remove from running list, do counters & statistics */
pktgen_stop_device(struct pktgen_dev * pkt_dev)3369 static int pktgen_stop_device(struct pktgen_dev *pkt_dev)
3370 {
3371 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1;
3372
3373 if (!pkt_dev->running) {
3374 pr_warn("interface: %s is already stopped\n",
3375 pkt_dev->odevname);
3376 return -EINVAL;
3377 }
3378
3379 pkt_dev->running = 0;
3380 kfree_skb(pkt_dev->skb);
3381 pkt_dev->skb = NULL;
3382 pkt_dev->stopped_at = ktime_get();
3383
3384 show_results(pkt_dev, nr_frags);
3385
3386 return 0;
3387 }
3388
next_to_run(struct pktgen_thread * t)3389 static struct pktgen_dev *next_to_run(struct pktgen_thread *t)
3390 {
3391 struct pktgen_dev *pkt_dev, *best = NULL;
3392
3393 rcu_read_lock();
3394 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3395 if (!pkt_dev->running)
3396 continue;
3397 if (best == NULL)
3398 best = pkt_dev;
3399 else if (ktime_compare(pkt_dev->next_tx, best->next_tx) < 0)
3400 best = pkt_dev;
3401 }
3402 rcu_read_unlock();
3403
3404 return best;
3405 }
3406
pktgen_stop(struct pktgen_thread * t)3407 static void pktgen_stop(struct pktgen_thread *t)
3408 {
3409 struct pktgen_dev *pkt_dev;
3410
3411 func_enter();
3412
3413 rcu_read_lock();
3414
3415 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3416 pktgen_stop_device(pkt_dev);
3417 }
3418
3419 rcu_read_unlock();
3420 }
3421
3422 /*
3423 * one of our devices needs to be removed - find it
3424 * and remove it
3425 */
pktgen_rem_one_if(struct pktgen_thread * t)3426 static void pktgen_rem_one_if(struct pktgen_thread *t)
3427 {
3428 struct list_head *q, *n;
3429 struct pktgen_dev *cur;
3430
3431 func_enter();
3432
3433 list_for_each_safe(q, n, &t->if_list) {
3434 cur = list_entry(q, struct pktgen_dev, list);
3435
3436 if (!cur->removal_mark)
3437 continue;
3438
3439 kfree_skb(cur->skb);
3440 cur->skb = NULL;
3441
3442 pktgen_remove_device(t, cur);
3443
3444 break;
3445 }
3446 }
3447
pktgen_rem_all_ifs(struct pktgen_thread * t)3448 static void pktgen_rem_all_ifs(struct pktgen_thread *t)
3449 {
3450 struct list_head *q, *n;
3451 struct pktgen_dev *cur;
3452
3453 func_enter();
3454
3455 /* Remove all devices, free mem */
3456
3457 list_for_each_safe(q, n, &t->if_list) {
3458 cur = list_entry(q, struct pktgen_dev, list);
3459
3460 kfree_skb(cur->skb);
3461 cur->skb = NULL;
3462
3463 pktgen_remove_device(t, cur);
3464 }
3465 }
3466
pktgen_rem_thread(struct pktgen_thread * t)3467 static void pktgen_rem_thread(struct pktgen_thread *t)
3468 {
3469 /* Remove from the thread list */
3470 remove_proc_entry(t->tsk->comm, t->net->proc_dir);
3471 }
3472
pktgen_resched(struct pktgen_dev * pkt_dev)3473 static void pktgen_resched(struct pktgen_dev *pkt_dev)
3474 {
3475 ktime_t idle_start = ktime_get();
3476 schedule();
3477 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start));
3478 }
3479
pktgen_wait_for_skb(struct pktgen_dev * pkt_dev)3480 static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev)
3481 {
3482 ktime_t idle_start = ktime_get();
3483
3484 while (refcount_read(&(pkt_dev->skb->users)) != 1) {
3485 if (signal_pending(current))
3486 break;
3487
3488 if (need_resched())
3489 pktgen_resched(pkt_dev);
3490 else
3491 cpu_relax();
3492 }
3493 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start));
3494 }
3495
pktgen_xmit(struct pktgen_dev * pkt_dev)3496 static void pktgen_xmit(struct pktgen_dev *pkt_dev)
3497 {
3498 bool skb_shared = !!(READ_ONCE(pkt_dev->flags) & F_SHARED);
3499 struct net_device *odev = pkt_dev->odev;
3500 struct netdev_queue *txq;
3501 unsigned int burst = 1;
3502 struct sk_buff *skb;
3503 int clone_skb = 0;
3504 int ret;
3505
3506 /* If 'skb_shared' is false, the read of possible
3507 * new values (if any) for 'burst' and 'clone_skb' will be skipped to
3508 * prevent some concurrent changes from slipping in. And the stabilized
3509 * config will be read in during the next run of pktgen_xmit.
3510 */
3511 if (skb_shared) {
3512 burst = READ_ONCE(pkt_dev->burst);
3513 clone_skb = READ_ONCE(pkt_dev->clone_skb);
3514 }
3515
3516 /* If device is offline, then don't send */
3517 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) {
3518 pktgen_stop_device(pkt_dev);
3519 return;
3520 }
3521
3522 /* This is max DELAY, this has special meaning of
3523 * "never transmit"
3524 */
3525 if (unlikely(pkt_dev->delay == ULLONG_MAX)) {
3526 pkt_dev->next_tx = ktime_add_ns(ktime_get(), ULONG_MAX);
3527 return;
3528 }
3529
3530 /* If no skb or clone count exhausted then get new one */
3531 if (!pkt_dev->skb || (pkt_dev->last_ok &&
3532 ++pkt_dev->clone_count >= clone_skb)) {
3533 /* build a new pkt */
3534 kfree_skb(pkt_dev->skb);
3535
3536 pkt_dev->skb = fill_packet(odev, pkt_dev);
3537 if (pkt_dev->skb == NULL) {
3538 pr_err("ERROR: couldn't allocate skb in fill_packet\n");
3539 schedule();
3540 pkt_dev->clone_count--; /* back out increment, OOM */
3541 return;
3542 }
3543 pkt_dev->last_pkt_size = pkt_dev->skb->len;
3544 pkt_dev->clone_count = 0; /* reset counter */
3545 }
3546
3547 if (pkt_dev->delay && pkt_dev->last_ok)
3548 spin(pkt_dev, pkt_dev->next_tx);
3549
3550 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) {
3551 skb = pkt_dev->skb;
3552 skb->protocol = eth_type_trans(skb, skb->dev);
3553 if (skb_shared)
3554 refcount_add(burst, &skb->users);
3555 local_bh_disable();
3556 do {
3557 ret = netif_receive_skb(skb);
3558 if (ret == NET_RX_DROP)
3559 pkt_dev->errors++;
3560 pkt_dev->sofar++;
3561 pkt_dev->seq_num++;
3562 if (unlikely(!skb_shared)) {
3563 pkt_dev->skb = NULL;
3564 break;
3565 }
3566 if (refcount_read(&skb->users) != burst) {
3567 /* skb was queued by rps/rfs or taps,
3568 * so cannot reuse this skb
3569 */
3570 WARN_ON(refcount_sub_and_test(burst - 1, &skb->users));
3571 /* get out of the loop and wait
3572 * until skb is consumed
3573 */
3574 break;
3575 }
3576 /* skb was 'freed' by stack, so clean few
3577 * bits and reuse it
3578 */
3579 skb_reset_redirect(skb);
3580 } while (--burst > 0);
3581 goto out; /* Skips xmit_mode M_START_XMIT */
3582 } else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) {
3583 local_bh_disable();
3584 if (skb_shared)
3585 refcount_inc(&pkt_dev->skb->users);
3586
3587 ret = dev_queue_xmit(pkt_dev->skb);
3588
3589 if (!skb_shared && dev_xmit_complete(ret))
3590 pkt_dev->skb = NULL;
3591
3592 switch (ret) {
3593 case NET_XMIT_SUCCESS:
3594 pkt_dev->sofar++;
3595 pkt_dev->seq_num++;
3596 pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3597 break;
3598 case NET_XMIT_DROP:
3599 case NET_XMIT_CN:
3600 /* These are all valid return codes for a qdisc but
3601 * indicate packets are being dropped or will likely
3602 * be dropped soon.
3603 */
3604 case NETDEV_TX_BUSY:
3605 /* qdisc may call dev_hard_start_xmit directly in cases
3606 * where no queues exist e.g. loopback device, virtual
3607 * devices, etc. In this case we need to handle
3608 * NETDEV_TX_ codes.
3609 */
3610 default:
3611 pkt_dev->errors++;
3612 net_info_ratelimited("%s xmit error: %d\n",
3613 pkt_dev->odevname, ret);
3614 break;
3615 }
3616 goto out;
3617 }
3618
3619 txq = skb_get_tx_queue(odev, pkt_dev->skb);
3620
3621 local_bh_disable();
3622
3623 HARD_TX_LOCK(odev, txq, smp_processor_id());
3624
3625 if (unlikely(netif_xmit_frozen_or_drv_stopped(txq))) {
3626 pkt_dev->last_ok = 0;
3627 goto unlock;
3628 }
3629 if (skb_shared)
3630 refcount_add(burst, &pkt_dev->skb->users);
3631
3632 xmit_more:
3633 ret = netdev_start_xmit(pkt_dev->skb, odev, txq, --burst > 0);
3634
3635 if (!skb_shared && dev_xmit_complete(ret))
3636 pkt_dev->skb = NULL;
3637
3638 switch (ret) {
3639 case NETDEV_TX_OK:
3640 pkt_dev->last_ok = 1;
3641 pkt_dev->sofar++;
3642 pkt_dev->seq_num++;
3643 pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3644 if (burst > 0 && !netif_xmit_frozen_or_drv_stopped(txq))
3645 goto xmit_more;
3646 break;
3647 case NET_XMIT_DROP:
3648 case NET_XMIT_CN:
3649 /* skb has been consumed */
3650 pkt_dev->errors++;
3651 break;
3652 default: /* Drivers are not supposed to return other values! */
3653 net_info_ratelimited("%s xmit error: %d\n",
3654 pkt_dev->odevname, ret);
3655 pkt_dev->errors++;
3656 fallthrough;
3657 case NETDEV_TX_BUSY:
3658 /* Retry it next time */
3659 if (skb_shared)
3660 refcount_dec(&pkt_dev->skb->users);
3661 pkt_dev->last_ok = 0;
3662 }
3663 if (unlikely(burst))
3664 WARN_ON(refcount_sub_and_test(burst, &pkt_dev->skb->users));
3665 unlock:
3666 HARD_TX_UNLOCK(odev, txq);
3667
3668 out:
3669 local_bh_enable();
3670
3671 /* If pkt_dev->count is zero, then run forever */
3672 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) {
3673 if (pkt_dev->skb)
3674 pktgen_wait_for_skb(pkt_dev);
3675
3676 /* Done with this */
3677 pktgen_stop_device(pkt_dev);
3678 }
3679 }
3680
3681 /*
3682 * Main loop of the thread goes here
3683 */
3684
pktgen_thread_worker(void * arg)3685 static int pktgen_thread_worker(void *arg)
3686 {
3687 struct pktgen_thread *t = arg;
3688 struct pktgen_dev *pkt_dev = NULL;
3689 int cpu = t->cpu;
3690
3691 WARN_ON_ONCE(smp_processor_id() != cpu);
3692
3693 init_waitqueue_head(&t->queue);
3694 complete(&t->start_done);
3695
3696 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current));
3697
3698 set_freezable();
3699
3700 while (!kthread_should_stop()) {
3701 pkt_dev = next_to_run(t);
3702
3703 if (unlikely(!pkt_dev && t->control == 0)) {
3704 if (t->net->pktgen_exiting)
3705 break;
3706 wait_event_freezable_timeout(t->queue,
3707 t->control != 0, HZ / 10);
3708 continue;
3709 }
3710
3711 if (likely(pkt_dev)) {
3712 pktgen_xmit(pkt_dev);
3713
3714 if (need_resched())
3715 pktgen_resched(pkt_dev);
3716 else
3717 cpu_relax();
3718 }
3719
3720 if (t->control & T_STOP) {
3721 pktgen_stop(t);
3722 t->control &= ~(T_STOP);
3723 }
3724
3725 if (t->control & T_RUN) {
3726 pktgen_run(t);
3727 t->control &= ~(T_RUN);
3728 }
3729
3730 if (t->control & T_REMDEVALL) {
3731 pktgen_rem_all_ifs(t);
3732 t->control &= ~(T_REMDEVALL);
3733 }
3734
3735 if (t->control & T_REMDEV) {
3736 pktgen_rem_one_if(t);
3737 t->control &= ~(T_REMDEV);
3738 }
3739
3740 try_to_freeze();
3741 }
3742
3743 pr_debug("%s stopping all device\n", t->tsk->comm);
3744 pktgen_stop(t);
3745
3746 pr_debug("%s removing all device\n", t->tsk->comm);
3747 pktgen_rem_all_ifs(t);
3748
3749 pr_debug("%s removing thread\n", t->tsk->comm);
3750 pktgen_rem_thread(t);
3751
3752 return 0;
3753 }
3754
pktgen_find_dev(struct pktgen_thread * t,const char * ifname,bool exact)3755 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
3756 const char *ifname, bool exact)
3757 {
3758 struct pktgen_dev *p, *pkt_dev = NULL;
3759 size_t len = strlen(ifname);
3760
3761 rcu_read_lock();
3762 list_for_each_entry_rcu(p, &t->if_list, list)
3763 if (strncmp(p->odevname, ifname, len) == 0) {
3764 if (p->odevname[len]) {
3765 if (exact || p->odevname[len] != '@')
3766 continue;
3767 }
3768 pkt_dev = p;
3769 break;
3770 }
3771
3772 rcu_read_unlock();
3773 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev);
3774 return pkt_dev;
3775 }
3776
3777 /*
3778 * Adds a dev at front of if_list.
3779 */
3780
add_dev_to_thread(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3781 static int add_dev_to_thread(struct pktgen_thread *t,
3782 struct pktgen_dev *pkt_dev)
3783 {
3784 int rv = 0;
3785
3786 /* This function cannot be called concurrently, as its called
3787 * under pktgen_thread_lock mutex, but it can run from
3788 * userspace on another CPU than the kthread. The if_lock()
3789 * is used here to sync with concurrent instances of
3790 * _rem_dev_from_if_list() invoked via kthread, which is also
3791 * updating the if_list */
3792 if_lock(t);
3793
3794 if (pkt_dev->pg_thread) {
3795 pr_err("ERROR: already assigned to a thread\n");
3796 rv = -EBUSY;
3797 goto out;
3798 }
3799
3800 pkt_dev->running = 0;
3801 pkt_dev->pg_thread = t;
3802 list_add_rcu(&pkt_dev->list, &t->if_list);
3803
3804 out:
3805 if_unlock(t);
3806 return rv;
3807 }
3808
3809 /* Called under thread lock */
3810
pktgen_add_device(struct pktgen_thread * t,const char * ifname)3811 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname)
3812 {
3813 struct pktgen_dev *pkt_dev;
3814 int err;
3815 int node = cpu_to_node(t->cpu);
3816
3817 /* We don't allow a device to be on several threads */
3818
3819 pkt_dev = __pktgen_NN_threads(t->net, ifname, FIND);
3820 if (pkt_dev) {
3821 pr_err("ERROR: interface already used\n");
3822 return -EBUSY;
3823 }
3824
3825 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node);
3826 if (!pkt_dev)
3827 return -ENOMEM;
3828
3829 strcpy(pkt_dev->odevname, ifname);
3830 pkt_dev->flows = vzalloc_node(array_size(MAX_CFLOWS,
3831 sizeof(struct flow_state)),
3832 node);
3833 if (pkt_dev->flows == NULL) {
3834 kfree(pkt_dev);
3835 return -ENOMEM;
3836 }
3837
3838 pkt_dev->removal_mark = 0;
3839 pkt_dev->nfrags = 0;
3840 pkt_dev->delay = pg_delay_d;
3841 pkt_dev->count = pg_count_d;
3842 pkt_dev->sofar = 0;
3843 pkt_dev->udp_src_min = 9; /* sink port */
3844 pkt_dev->udp_src_max = 9;
3845 pkt_dev->udp_dst_min = 9;
3846 pkt_dev->udp_dst_max = 9;
3847 pkt_dev->vlan_p = 0;
3848 pkt_dev->vlan_cfi = 0;
3849 pkt_dev->vlan_id = 0xffff;
3850 pkt_dev->svlan_p = 0;
3851 pkt_dev->svlan_cfi = 0;
3852 pkt_dev->svlan_id = 0xffff;
3853 pkt_dev->burst = 1;
3854 pkt_dev->node = NUMA_NO_NODE;
3855 pkt_dev->flags = F_SHARED; /* SKB shared by default */
3856
3857 err = pktgen_setup_dev(t->net, pkt_dev, ifname);
3858 if (err)
3859 goto out1;
3860 if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)
3861 pkt_dev->clone_skb = pg_clone_skb_d;
3862
3863 pkt_dev->entry = proc_create_data(ifname, 0600, t->net->proc_dir,
3864 &pktgen_if_proc_ops, pkt_dev);
3865 if (!pkt_dev->entry) {
3866 pr_err("cannot create %s/%s procfs entry\n",
3867 PG_PROC_DIR, ifname);
3868 err = -EINVAL;
3869 goto out2;
3870 }
3871 #ifdef CONFIG_XFRM
3872 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT;
3873 pkt_dev->ipsproto = IPPROTO_ESP;
3874
3875 /* xfrm tunnel mode needs additional dst to extract outer
3876 * ip header protocol/ttl/id field, here create a phony one.
3877 * instead of looking for a valid rt, which definitely hurting
3878 * performance under such circumstance.
3879 */
3880 pkt_dev->dstops.family = AF_INET;
3881 pkt_dev->xdst.u.dst.dev = pkt_dev->odev;
3882 dst_init_metrics(&pkt_dev->xdst.u.dst, pktgen_dst_metrics, false);
3883 pkt_dev->xdst.child = &pkt_dev->xdst.u.dst;
3884 pkt_dev->xdst.u.dst.ops = &pkt_dev->dstops;
3885 #endif
3886
3887 return add_dev_to_thread(t, pkt_dev);
3888 out2:
3889 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
3890 out1:
3891 #ifdef CONFIG_XFRM
3892 free_SAs(pkt_dev);
3893 #endif
3894 vfree(pkt_dev->flows);
3895 kfree(pkt_dev);
3896 return err;
3897 }
3898
pktgen_create_thread(int cpu,struct pktgen_net * pn)3899 static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn)
3900 {
3901 struct pktgen_thread *t;
3902 struct proc_dir_entry *pe;
3903 struct task_struct *p;
3904
3905 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL,
3906 cpu_to_node(cpu));
3907 if (!t) {
3908 pr_err("ERROR: out of memory, can't create new thread\n");
3909 return -ENOMEM;
3910 }
3911
3912 mutex_init(&t->if_lock);
3913 t->cpu = cpu;
3914
3915 INIT_LIST_HEAD(&t->if_list);
3916
3917 list_add_tail(&t->th_list, &pn->pktgen_threads);
3918 init_completion(&t->start_done);
3919
3920 p = kthread_create_on_cpu(pktgen_thread_worker, t, cpu, "kpktgend_%d");
3921 if (IS_ERR(p)) {
3922 pr_err("kthread_create_on_node() failed for cpu %d\n", t->cpu);
3923 list_del(&t->th_list);
3924 kfree(t);
3925 return PTR_ERR(p);
3926 }
3927
3928 t->tsk = p;
3929
3930 pe = proc_create_data(t->tsk->comm, 0600, pn->proc_dir,
3931 &pktgen_thread_proc_ops, t);
3932 if (!pe) {
3933 pr_err("cannot create %s/%s procfs entry\n",
3934 PG_PROC_DIR, t->tsk->comm);
3935 kthread_stop(p);
3936 list_del(&t->th_list);
3937 kfree(t);
3938 return -EINVAL;
3939 }
3940
3941 t->net = pn;
3942 get_task_struct(p);
3943 wake_up_process(p);
3944 wait_for_completion(&t->start_done);
3945
3946 return 0;
3947 }
3948
3949 /*
3950 * Removes a device from the thread if_list.
3951 */
_rem_dev_from_if_list(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3952 static void _rem_dev_from_if_list(struct pktgen_thread *t,
3953 struct pktgen_dev *pkt_dev)
3954 {
3955 struct list_head *q, *n;
3956 struct pktgen_dev *p;
3957
3958 if_lock(t);
3959 list_for_each_safe(q, n, &t->if_list) {
3960 p = list_entry(q, struct pktgen_dev, list);
3961 if (p == pkt_dev)
3962 list_del_rcu(&p->list);
3963 }
3964 if_unlock(t);
3965 }
3966
pktgen_remove_device(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3967 static int pktgen_remove_device(struct pktgen_thread *t,
3968 struct pktgen_dev *pkt_dev)
3969 {
3970 pr_debug("remove_device pkt_dev=%p\n", pkt_dev);
3971
3972 if (pkt_dev->running) {
3973 pr_warn("WARNING: trying to remove a running interface, stopping it now\n");
3974 pktgen_stop_device(pkt_dev);
3975 }
3976
3977 /* Dis-associate from the interface */
3978
3979 if (pkt_dev->odev) {
3980 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
3981 pkt_dev->odev = NULL;
3982 }
3983
3984 /* Remove proc before if_list entry, because add_device uses
3985 * list to determine if interface already exist, avoid race
3986 * with proc_create_data() */
3987 proc_remove(pkt_dev->entry);
3988
3989 /* And update the thread if_list */
3990 _rem_dev_from_if_list(t, pkt_dev);
3991
3992 #ifdef CONFIG_XFRM
3993 free_SAs(pkt_dev);
3994 #endif
3995 vfree(pkt_dev->flows);
3996 if (pkt_dev->page)
3997 put_page(pkt_dev->page);
3998 kfree_rcu(pkt_dev, rcu);
3999 return 0;
4000 }
4001
pg_net_init(struct net * net)4002 static int __net_init pg_net_init(struct net *net)
4003 {
4004 struct pktgen_net *pn = net_generic(net, pg_net_id);
4005 struct proc_dir_entry *pe;
4006 int cpu, ret = 0;
4007
4008 pn->net = net;
4009 INIT_LIST_HEAD(&pn->pktgen_threads);
4010 pn->pktgen_exiting = false;
4011 pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net);
4012 if (!pn->proc_dir) {
4013 pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR);
4014 return -ENODEV;
4015 }
4016 pe = proc_create(PGCTRL, 0600, pn->proc_dir, &pktgen_proc_ops);
4017 if (pe == NULL) {
4018 pr_err("cannot create %s procfs entry\n", PGCTRL);
4019 ret = -EINVAL;
4020 goto remove;
4021 }
4022
4023 cpus_read_lock();
4024 for_each_online_cpu(cpu) {
4025 int err;
4026
4027 err = pktgen_create_thread(cpu, pn);
4028 if (err)
4029 pr_warn("Cannot create thread for cpu %d (%d)\n",
4030 cpu, err);
4031 }
4032 cpus_read_unlock();
4033
4034 if (list_empty(&pn->pktgen_threads)) {
4035 pr_err("Initialization failed for all threads\n");
4036 ret = -ENODEV;
4037 goto remove_entry;
4038 }
4039
4040 return 0;
4041
4042 remove_entry:
4043 remove_proc_entry(PGCTRL, pn->proc_dir);
4044 remove:
4045 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net);
4046 return ret;
4047 }
4048
pg_net_exit(struct net * net)4049 static void __net_exit pg_net_exit(struct net *net)
4050 {
4051 struct pktgen_net *pn = net_generic(net, pg_net_id);
4052 struct pktgen_thread *t;
4053 struct list_head *q, *n;
4054 LIST_HEAD(list);
4055
4056 /* Stop all interfaces & threads */
4057 pn->pktgen_exiting = true;
4058
4059 mutex_lock(&pktgen_thread_lock);
4060 list_splice_init(&pn->pktgen_threads, &list);
4061 mutex_unlock(&pktgen_thread_lock);
4062
4063 list_for_each_safe(q, n, &list) {
4064 t = list_entry(q, struct pktgen_thread, th_list);
4065 list_del(&t->th_list);
4066 kthread_stop_put(t->tsk);
4067 kfree(t);
4068 }
4069
4070 remove_proc_entry(PGCTRL, pn->proc_dir);
4071 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net);
4072 }
4073
4074 static struct pernet_operations pg_net_ops = {
4075 .init = pg_net_init,
4076 .exit = pg_net_exit,
4077 .id = &pg_net_id,
4078 .size = sizeof(struct pktgen_net),
4079 };
4080
pg_init(void)4081 static int __init pg_init(void)
4082 {
4083 int ret = 0;
4084
4085 pr_info("%s", version);
4086 ret = register_pernet_subsys(&pg_net_ops);
4087 if (ret)
4088 return ret;
4089 ret = register_netdevice_notifier(&pktgen_notifier_block);
4090 if (ret)
4091 unregister_pernet_subsys(&pg_net_ops);
4092
4093 return ret;
4094 }
4095
pg_cleanup(void)4096 static void __exit pg_cleanup(void)
4097 {
4098 unregister_netdevice_notifier(&pktgen_notifier_block);
4099 unregister_pernet_subsys(&pg_net_ops);
4100 /* Don't need rcu_barrier() due to use of kfree_rcu() */
4101 }
4102
4103 module_init(pg_init);
4104 module_exit(pg_cleanup);
4105
4106 MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>");
4107 MODULE_DESCRIPTION("Packet Generator tool");
4108 MODULE_LICENSE("GPL");
4109 MODULE_VERSION(VERSION);
4110 module_param(pg_count_d, int, 0);
4111 MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject");
4112 module_param(pg_delay_d, int, 0);
4113 MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)");
4114 module_param(pg_clone_skb_d, int, 0);
4115 MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet");
4116 module_param(debug, int, 0);
4117 MODULE_PARM_DESC(debug, "Enable debugging of pktgen module");
4118