1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Authors: 4 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se> 5 * Uppsala University and 6 * Swedish University of Agricultural Sciences 7 * 8 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 9 * Ben Greear <greearb@candelatech.com> 10 * Jens Låås <jens.laas@data.slu.se> 11 * 12 * A tool for loading the network with preconfigurated packets. 13 * The tool is implemented as a linux module. Parameters are output 14 * device, delay (to hard_xmit), number of packets, and whether 15 * to use multiple SKBs or just the same one. 16 * pktgen uses the installed interface's output routine. 17 * 18 * Additional hacking by: 19 * 20 * Jens.Laas@data.slu.se 21 * Improved by ANK. 010120. 22 * Improved by ANK even more. 010212. 23 * MAC address typo fixed. 010417 --ro 24 * Integrated. 020301 --DaveM 25 * Added multiskb option 020301 --DaveM 26 * Scaling of results. 020417--sigurdur@linpro.no 27 * Significant re-work of the module: 28 * * Convert to threaded model to more efficiently be able to transmit 29 * and receive on multiple interfaces at once. 30 * * Converted many counters to __u64 to allow longer runs. 31 * * Allow configuration of ranges, like min/max IP address, MACs, 32 * and UDP-ports, for both source and destination, and can 33 * set to use a random distribution or sequentially walk the range. 34 * * Can now change most values after starting. 35 * * Place 12-byte packet in UDP payload with magic number, 36 * sequence number, and timestamp. 37 * * Add receiver code that detects dropped pkts, re-ordered pkts, and 38 * latencies (with micro-second) precision. 39 * * Add IOCTL interface to easily get counters & configuration. 40 * --Ben Greear <greearb@candelatech.com> 41 * 42 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct 43 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0 44 * as a "fastpath" with a configurable number of clones after alloc's. 45 * clone_skb=0 means all packets are allocated this also means ranges time 46 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100 47 * clones. 48 * 49 * Also moved to /proc/net/pktgen/ 50 * --ro 51 * 52 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever 53 * mistakes. Also merged in DaveM's patch in the -pre6 patch. 54 * --Ben Greear <greearb@candelatech.com> 55 * 56 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br) 57 * 58 * 021124 Finished major redesign and rewrite for new functionality. 59 * See Documentation/networking/pktgen.rst for how to use this. 60 * 61 * The new operation: 62 * For each CPU one thread/process is created at start. This process checks 63 * for running devices in the if_list and sends packets until count is 0 it 64 * also the thread checks the thread->control which is used for inter-process 65 * communication. controlling process "posts" operations to the threads this 66 * way. 67 * The if_list is RCU protected, and the if_lock remains to protect updating 68 * of if_list, from "add_device" as it invoked from userspace (via proc write). 69 * 70 * By design there should only be *one* "controlling" process. In practice 71 * multiple write accesses gives unpredictable result. Understood by "write" 72 * to /proc gives result code that should be read be the "writer". 73 * For practical use this should be no problem. 74 * 75 * Note when adding devices to a specific CPU there good idea to also assign 76 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU. 77 * --ro 78 * 79 * Fix refcount off by one if first packet fails, potential null deref, 80 * memleak 030710- KJP 81 * 82 * First "ranges" functionality for ipv6 030726 --ro 83 * 84 * Included flow support. 030802 ANK. 85 * 86 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org> 87 * 88 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419 89 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604 90 * 91 * New xmit() return, do_div and misc clean up by Stephen Hemminger 92 * <shemminger@osdl.org> 040923 93 * 94 * Randy Dunlap fixed u64 printk compiler warning 95 * 96 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org> 97 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213 98 * 99 * Corrections from Nikolai Malykh (nmalykh@bilim.com) 100 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230 101 * 102 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com> 103 * 050103 104 * 105 * MPLS support by Steven Whitehouse <steve@chygwyn.com> 106 * 107 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com> 108 * 109 * Fixed src_mac command to set source mac of packet to value specified in 110 * command by Adit Ranadive <adit.262@gmail.com> 111 */ 112 113 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 114 115 #include <linux/sys.h> 116 #include <linux/types.h> 117 #include <linux/minmax.h> 118 #include <linux/module.h> 119 #include <linux/moduleparam.h> 120 #include <linux/kernel.h> 121 #include <linux/mutex.h> 122 #include <linux/sched.h> 123 #include <linux/slab.h> 124 #include <linux/vmalloc.h> 125 #include <linux/unistd.h> 126 #include <linux/string.h> 127 #include <linux/ptrace.h> 128 #include <linux/errno.h> 129 #include <linux/hex.h> 130 #include <linux/ioport.h> 131 #include <linux/interrupt.h> 132 #include <linux/capability.h> 133 #include <linux/hrtimer.h> 134 #include <linux/freezer.h> 135 #include <linux/delay.h> 136 #include <linux/timer.h> 137 #include <linux/list.h> 138 #include <linux/init.h> 139 #include <linux/skbuff.h> 140 #include <linux/netdevice.h> 141 #include <linux/inet.h> 142 #include <linux/inetdevice.h> 143 #include <linux/rtnetlink.h> 144 #include <linux/if_arp.h> 145 #include <linux/if_vlan.h> 146 #include <linux/in.h> 147 #include <linux/ip.h> 148 #include <linux/ipv6.h> 149 #include <linux/udp.h> 150 #include <linux/proc_fs.h> 151 #include <linux/seq_file.h> 152 #include <linux/wait.h> 153 #include <linux/etherdevice.h> 154 #include <linux/kthread.h> 155 #include <linux/prefetch.h> 156 #include <linux/mmzone.h> 157 #include <net/net_namespace.h> 158 #include <net/checksum.h> 159 #include <net/ipv6.h> 160 #include <net/udp.h> 161 #include <net/ip6_checksum.h> 162 #include <net/addrconf.h> 163 #include <net/xfrm.h> 164 #include <net/netns/generic.h> 165 #include <asm/byteorder.h> 166 #include <linux/rcupdate.h> 167 #include <linux/bitops.h> 168 #include <linux/io.h> 169 #include <linux/timex.h> 170 #include <linux/uaccess.h> 171 #include <asm/dma.h> 172 #include <asm/div64.h> /* do_div */ 173 174 #define VERSION "2.75" 175 #define IP_NAME_SZ 32 176 #define MAX_MPLS_LABELS 16 /* This is the max label stack depth */ 177 #define MPLS_STACK_BOTTOM htonl(0x00000100) 178 /* Max number of internet mix entries that can be specified in imix_weights. */ 179 #define MAX_IMIX_ENTRIES 20 180 #define IMIX_PRECISION 100 /* Precision of IMIX distribution */ 181 182 #define func_enter() pr_debug("entering %s\n", __func__) 183 184 #define PKT_FLAGS \ 185 pf(IPV6) /* Interface in IPV6 Mode */ \ 186 pf(IPSRC_RND) /* IP-Src Random */ \ 187 pf(IPDST_RND) /* IP-Dst Random */ \ 188 pf(TXSIZE_RND) /* Transmit size is random */ \ 189 pf(UDPSRC_RND) /* UDP-Src Random */ \ 190 pf(UDPDST_RND) /* UDP-Dst Random */ \ 191 pf(UDPCSUM) /* Include UDP checksum */ \ 192 pf(NO_TIMESTAMP) /* Don't timestamp packets (default TS) */ \ 193 pf(MPLS_RND) /* Random MPLS labels */ \ 194 pf(QUEUE_MAP_RND) /* queue map Random */ \ 195 pf(QUEUE_MAP_CPU) /* queue map mirrors smp_processor_id() */ \ 196 pf(FLOW_SEQ) /* Sequential flows */ \ 197 pf(IPSEC) /* ipsec on for flows */ \ 198 pf(MACSRC_RND) /* MAC-Src Random */ \ 199 pf(MACDST_RND) /* MAC-Dst Random */ \ 200 pf(VID_RND) /* Random VLAN ID */ \ 201 pf(SVID_RND) /* Random SVLAN ID */ \ 202 pf(NODE) /* Node memory alloc*/ \ 203 pf(SHARED) /* Shared SKB */ \ 204 205 #define pf(flag) flag##_SHIFT, 206 enum pkt_flags { 207 PKT_FLAGS 208 }; 209 #undef pf 210 211 /* Device flag bits */ 212 #define pf(flag) static const __u32 F_##flag = (1<<flag##_SHIFT); 213 PKT_FLAGS 214 #undef pf 215 216 #define pf(flag) __stringify(flag), 217 static char *pkt_flag_names[] = { 218 PKT_FLAGS 219 }; 220 #undef pf 221 222 #define NR_PKT_FLAGS ARRAY_SIZE(pkt_flag_names) 223 224 /* Thread control flag bits */ 225 #define T_STOP (1<<0) /* Stop run */ 226 #define T_RUN (1<<1) /* Start run */ 227 #define T_REMDEVALL (1<<2) /* Remove all devs */ 228 #define T_REMDEV (1<<3) /* Remove one dev */ 229 230 /* Xmit modes */ 231 #define M_START_XMIT 0 /* Default normal TX */ 232 #define M_NETIF_RECEIVE 1 /* Inject packets into stack */ 233 #define M_QUEUE_XMIT 2 /* Inject packet into qdisc */ 234 235 /* If lock -- protects updating of if_list */ 236 #define if_lock(t) mutex_lock(&(t->if_lock)) 237 #define if_unlock(t) mutex_unlock(&(t->if_lock)) 238 239 /* Used to help with determining the pkts on receive */ 240 #define PKTGEN_MAGIC 0xbe9be955 241 #define PG_PROC_DIR "pktgen" 242 #define PGCTRL "pgctrl" 243 244 #define MAX_CFLOWS 65536 245 246 #define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4) 247 #define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4) 248 249 struct imix_pkt { 250 u64 size; 251 u64 weight; 252 u64 count_so_far; 253 }; 254 255 struct flow_state { 256 __be32 cur_daddr; 257 int count; 258 #ifdef CONFIG_XFRM 259 struct xfrm_state *x; 260 #endif 261 __u32 flags; 262 }; 263 264 /* flow flag bits */ 265 #define F_INIT (1<<0) /* flow has been initialized */ 266 267 struct pktgen_dev { 268 /* 269 * Try to keep frequent/infrequent used vars. separated. 270 */ 271 struct proc_dir_entry *entry; /* proc file */ 272 struct pktgen_thread *pg_thread;/* the owner */ 273 struct list_head list; /* chaining in the thread's run-queue */ 274 struct rcu_head rcu; /* freed by RCU */ 275 276 int running; /* if false, the test will stop */ 277 278 /* If min != max, then we will either do a linear iteration, or 279 * we will do a random selection from within the range. 280 */ 281 __u32 flags; 282 int xmit_mode; 283 int min_pkt_size; 284 int max_pkt_size; 285 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */ 286 int nfrags; 287 int removal_mark; /* non-zero => the device is marked for 288 * removal by worker thread 289 */ 290 291 struct page *page; 292 u64 delay; /* nano-seconds */ 293 294 __u64 count; /* Default No packets to send */ 295 __u64 sofar; /* How many pkts we've sent so far */ 296 __u64 tx_bytes; /* How many bytes we've transmitted */ 297 __u64 errors; /* Errors when trying to transmit, */ 298 299 /* runtime counters relating to clone_skb */ 300 301 __u32 clone_count; 302 int last_ok; /* Was last skb sent? 303 * Or a failed transmit of some sort? 304 * This will keep sequence numbers in order 305 */ 306 ktime_t next_tx; 307 ktime_t started_at; 308 ktime_t stopped_at; 309 u64 idle_acc; /* nano-seconds */ 310 311 __u32 seq_num; 312 313 int clone_skb; /* 314 * Use multiple SKBs during packet gen. 315 * If this number is greater than 1, then 316 * that many copies of the same packet will be 317 * sent before a new packet is allocated. 318 * If you want to send 1024 identical packets 319 * before creating a new packet, 320 * set clone_skb to 1024. 321 */ 322 323 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 324 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 325 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 326 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 327 328 struct in6_addr in6_saddr; 329 struct in6_addr in6_daddr; 330 struct in6_addr cur_in6_daddr; 331 struct in6_addr cur_in6_saddr; 332 /* For ranges */ 333 struct in6_addr min_in6_daddr; 334 struct in6_addr max_in6_daddr; 335 struct in6_addr min_in6_saddr; 336 struct in6_addr max_in6_saddr; 337 338 /* If we're doing ranges, random or incremental, then this 339 * defines the min/max for those ranges. 340 */ 341 __be32 saddr_min; /* inclusive, source IP address */ 342 __be32 saddr_max; /* exclusive, source IP address */ 343 __be32 daddr_min; /* inclusive, dest IP address */ 344 __be32 daddr_max; /* exclusive, dest IP address */ 345 346 __u16 udp_src_min; /* inclusive, source UDP port */ 347 __u16 udp_src_max; /* exclusive, source UDP port */ 348 __u16 udp_dst_min; /* inclusive, dest UDP port */ 349 __u16 udp_dst_max; /* exclusive, dest UDP port */ 350 351 /* DSCP + ECN */ 352 __u8 tos; /* six MSB of (former) IPv4 TOS 353 * are for dscp codepoint 354 */ 355 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6 356 * (see RFC 3260, sec. 4) 357 */ 358 359 /* IMIX */ 360 unsigned int n_imix_entries; 361 struct imix_pkt imix_entries[MAX_IMIX_ENTRIES]; 362 /* Maps 0-IMIX_PRECISION range to imix_entry based on probability*/ 363 __u8 imix_distribution[IMIX_PRECISION]; 364 365 /* MPLS */ 366 unsigned int nr_labels; /* Depth of stack, 0 = no MPLS */ 367 __be32 labels[MAX_MPLS_LABELS]; 368 369 /* VLAN/SVLAN (802.1Q/Q-in-Q) */ 370 __u8 vlan_p; 371 __u8 vlan_cfi; 372 __u16 vlan_id; /* 0xffff means no vlan tag */ 373 374 __u8 svlan_p; 375 __u8 svlan_cfi; 376 __u16 svlan_id; /* 0xffff means no svlan tag */ 377 378 __u32 src_mac_count; /* How many MACs to iterate through */ 379 __u32 dst_mac_count; /* How many MACs to iterate through */ 380 381 unsigned char dst_mac[ETH_ALEN]; 382 unsigned char src_mac[ETH_ALEN]; 383 384 __u32 cur_dst_mac_offset; 385 __u32 cur_src_mac_offset; 386 __be32 cur_saddr; 387 __be32 cur_daddr; 388 __u16 ip_id; 389 __u16 cur_udp_dst; 390 __u16 cur_udp_src; 391 __u16 cur_queue_map; 392 __u32 cur_pkt_size; 393 __u32 last_pkt_size; 394 395 __u8 hh[14]; 396 /* = { 397 * 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, 398 * 399 * We fill in SRC address later 400 * 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 401 * 0x08, 0x00 402 * }; 403 */ 404 __u16 pad; /* pad out the hh struct to an even 16 bytes */ 405 406 struct sk_buff *skb; /* skb we are to transmit next, used for when we 407 * are transmitting the same one multiple times 408 */ 409 struct net_device *odev; /* The out-going device. 410 * Note that the device should have it's 411 * pg_info pointer pointing back to this 412 * device. 413 * Set when the user specifies the out-going 414 * device name (not when the inject is 415 * started as it used to do.) 416 */ 417 netdevice_tracker dev_tracker; 418 char odevname[32]; 419 struct flow_state *flows; 420 unsigned int cflows; /* Concurrent flows (config) */ 421 unsigned int lflow; /* Flow length (config) */ 422 unsigned int nflows; /* accumulated flows (stats) */ 423 unsigned int curfl; /* current sequenced flow (state)*/ 424 425 u16 queue_map_min; 426 u16 queue_map_max; 427 __u32 skb_priority; /* skb priority field */ 428 unsigned int burst; /* number of duplicated packets to burst */ 429 int node; /* Memory node */ 430 431 #ifdef CONFIG_XFRM 432 __u8 ipsmode; /* IPSEC mode (config) */ 433 __u8 ipsproto; /* IPSEC type (config) */ 434 __u32 spi; 435 struct xfrm_dst xdst; 436 struct dst_ops dstops; 437 #endif 438 char result[512]; 439 }; 440 441 struct pktgen_hdr { 442 __be32 pgh_magic; 443 __be32 seq_num; 444 __be32 tv_sec; 445 __be32 tv_usec; 446 }; 447 448 449 static unsigned int pg_net_id __read_mostly; 450 451 struct pktgen_net { 452 struct net *net; 453 struct proc_dir_entry *proc_dir; 454 struct list_head pktgen_threads; 455 bool pktgen_exiting; 456 }; 457 458 struct pktgen_thread { 459 struct mutex if_lock; /* for list of devices */ 460 struct list_head if_list; /* All device here */ 461 struct list_head th_list; 462 struct task_struct *tsk; 463 char result[512]; 464 465 /* Field for thread to receive "posted" events terminate, 466 * stop ifs etc. 467 */ 468 469 u32 control; 470 int cpu; 471 472 wait_queue_head_t queue; 473 struct completion start_done; 474 struct pktgen_net *net; 475 }; 476 477 #define REMOVE 1 478 #define FIND 0 479 480 static const char version[] = 481 "Packet Generator for packet performance testing. Version: " VERSION "\n"; 482 483 static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i); 484 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname); 485 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 486 const char *ifname, bool exact); 487 static int pktgen_device_event(struct notifier_block *, unsigned long, void *); 488 static void pktgen_run_all_threads(struct pktgen_net *pn); 489 static void pktgen_reset_all_threads(struct pktgen_net *pn); 490 static void pktgen_stop_all_threads(struct pktgen_net *pn); 491 492 static void pktgen_stop(struct pktgen_thread *t); 493 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev); 494 static void fill_imix_distribution(struct pktgen_dev *pkt_dev); 495 496 /* Module parameters, defaults. */ 497 static int pg_count_d __read_mostly = 1000; 498 static int pg_delay_d __read_mostly; 499 static int pg_clone_skb_d __read_mostly; 500 static int debug __read_mostly; 501 502 static DEFINE_MUTEX(pktgen_thread_lock); 503 504 static struct notifier_block pktgen_notifier_block = { 505 .notifier_call = pktgen_device_event, 506 }; 507 508 /* 509 * /proc handling functions 510 * 511 */ 512 513 static int pgctrl_show(struct seq_file *seq, void *v) 514 { 515 seq_puts(seq, version); 516 return 0; 517 } 518 519 static ssize_t pgctrl_write(struct file *file, const char __user *buf, 520 size_t count, loff_t *ppos) 521 { 522 char data[128]; 523 size_t max; 524 struct pktgen_net *pn = net_generic(current->nsproxy->net_ns, pg_net_id); 525 526 if (!capable(CAP_NET_ADMIN)) 527 return -EPERM; 528 529 if (count < 1) 530 return -EINVAL; 531 532 max = min(count, sizeof(data) - 1); 533 if (copy_from_user(data, buf, max)) 534 return -EFAULT; 535 536 if (data[max - 1] == '\n') 537 data[max - 1] = 0; /* strip trailing '\n', terminate string */ 538 else 539 data[max] = 0; /* terminate string */ 540 541 if (!strcmp(data, "stop")) 542 pktgen_stop_all_threads(pn); 543 else if (!strcmp(data, "start")) 544 pktgen_run_all_threads(pn); 545 else if (!strcmp(data, "reset")) 546 pktgen_reset_all_threads(pn); 547 else 548 return -EINVAL; 549 550 return count; 551 } 552 553 static int pgctrl_open(struct inode *inode, struct file *file) 554 { 555 return single_open(file, pgctrl_show, pde_data(inode)); 556 } 557 558 static const struct proc_ops pktgen_proc_ops = { 559 .proc_open = pgctrl_open, 560 .proc_read = seq_read, 561 .proc_lseek = seq_lseek, 562 .proc_write = pgctrl_write, 563 .proc_release = single_release, 564 }; 565 566 static int pktgen_if_show(struct seq_file *seq, void *v) 567 { 568 const struct pktgen_dev *pkt_dev = seq->private; 569 unsigned int cflows = READ_ONCE(pkt_dev->cflows); 570 ktime_t stopped; 571 unsigned int i; 572 u64 idle; 573 574 seq_printf(seq, 575 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n", 576 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size, 577 pkt_dev->max_pkt_size); 578 579 if (pkt_dev->n_imix_entries > 0) { 580 seq_puts(seq, " imix_weights: "); 581 for (i = 0; i < pkt_dev->n_imix_entries; i++) { 582 seq_printf(seq, "%llu,%llu ", 583 pkt_dev->imix_entries[i].size, 584 pkt_dev->imix_entries[i].weight); 585 } 586 seq_puts(seq, "\n"); 587 } 588 589 seq_printf(seq, 590 " frags: %d delay: %llu clone_skb: %d ifname: %s\n", 591 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay, 592 pkt_dev->clone_skb, pkt_dev->odevname); 593 594 seq_printf(seq, " flows: %u flowlen: %u\n", cflows, 595 pkt_dev->lflow); 596 597 seq_printf(seq, 598 " queue_map_min: %u queue_map_max: %u\n", 599 pkt_dev->queue_map_min, 600 pkt_dev->queue_map_max); 601 602 if (pkt_dev->skb_priority) 603 seq_printf(seq, " skb_priority: %u\n", 604 pkt_dev->skb_priority); 605 606 if (pkt_dev->flags & F_IPV6) { 607 seq_printf(seq, 608 " saddr: %pI6c min_saddr: %pI6c max_saddr: %pI6c\n" 609 " daddr: %pI6c min_daddr: %pI6c max_daddr: %pI6c\n", 610 &pkt_dev->in6_saddr, 611 &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr, 612 &pkt_dev->in6_daddr, 613 &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr); 614 } else { 615 seq_printf(seq, 616 " dst_min: %s dst_max: %s\n", 617 pkt_dev->dst_min, pkt_dev->dst_max); 618 seq_printf(seq, 619 " src_min: %s src_max: %s\n", 620 pkt_dev->src_min, pkt_dev->src_max); 621 } 622 623 seq_puts(seq, " src_mac: "); 624 625 seq_printf(seq, "%pM ", 626 is_zero_ether_addr(pkt_dev->src_mac) ? 627 pkt_dev->odev->dev_addr : pkt_dev->src_mac); 628 629 seq_puts(seq, "dst_mac: "); 630 seq_printf(seq, "%pM\n", pkt_dev->dst_mac); 631 632 seq_printf(seq, 633 " udp_src_min: %d udp_src_max: %d udp_dst_min: %d udp_dst_max: %d\n", 634 pkt_dev->udp_src_min, pkt_dev->udp_src_max, 635 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max); 636 637 seq_printf(seq, 638 " src_mac_count: %d dst_mac_count: %d\n", 639 pkt_dev->src_mac_count, pkt_dev->dst_mac_count); 640 641 if (pkt_dev->nr_labels) { 642 seq_puts(seq, " mpls: "); 643 for (i = 0; i < pkt_dev->nr_labels; i++) 644 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]), 645 i == pkt_dev->nr_labels-1 ? "\n" : ", "); 646 } 647 648 if (pkt_dev->vlan_id != 0xffff) 649 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n", 650 pkt_dev->vlan_id, pkt_dev->vlan_p, 651 pkt_dev->vlan_cfi); 652 653 if (pkt_dev->svlan_id != 0xffff) 654 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n", 655 pkt_dev->svlan_id, pkt_dev->svlan_p, 656 pkt_dev->svlan_cfi); 657 658 if (pkt_dev->tos) 659 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos); 660 661 if (pkt_dev->traffic_class) 662 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class); 663 664 if (pkt_dev->burst > 1) 665 seq_printf(seq, " burst: %d\n", pkt_dev->burst); 666 667 if (pkt_dev->node >= 0) 668 seq_printf(seq, " node: %d\n", pkt_dev->node); 669 670 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) 671 seq_puts(seq, " xmit_mode: netif_receive\n"); 672 else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) 673 seq_puts(seq, " xmit_mode: xmit_queue\n"); 674 675 seq_puts(seq, " Flags: "); 676 677 for (i = 0; i < NR_PKT_FLAGS; i++) { 678 if (i == FLOW_SEQ_SHIFT) 679 if (!cflows) 680 continue; 681 682 if (pkt_dev->flags & (1 << i)) { 683 seq_printf(seq, "%s ", pkt_flag_names[i]); 684 #ifdef CONFIG_XFRM 685 if (i == IPSEC_SHIFT && pkt_dev->spi) 686 seq_printf(seq, "spi:%u ", pkt_dev->spi); 687 #endif 688 } else if (i == FLOW_SEQ_SHIFT) { 689 seq_puts(seq, "FLOW_RND "); 690 } 691 } 692 693 seq_puts(seq, "\n"); 694 695 /* not really stopped, more like last-running-at */ 696 stopped = pkt_dev->running ? ktime_get() : pkt_dev->stopped_at; 697 idle = pkt_dev->idle_acc; 698 do_div(idle, NSEC_PER_USEC); 699 700 seq_printf(seq, 701 "Current:\n pkts-sofar: %llu errors: %llu\n", 702 (unsigned long long)pkt_dev->sofar, 703 (unsigned long long)pkt_dev->errors); 704 705 if (pkt_dev->n_imix_entries > 0) { 706 int i; 707 708 seq_puts(seq, " imix_size_counts: "); 709 for (i = 0; i < pkt_dev->n_imix_entries; i++) { 710 seq_printf(seq, "%llu,%llu ", 711 pkt_dev->imix_entries[i].size, 712 pkt_dev->imix_entries[i].count_so_far); 713 } 714 seq_puts(seq, "\n"); 715 } 716 717 seq_printf(seq, 718 " started: %lluus stopped: %lluus idle: %lluus\n", 719 (unsigned long long) ktime_to_us(pkt_dev->started_at), 720 (unsigned long long) ktime_to_us(stopped), 721 (unsigned long long) idle); 722 723 seq_printf(seq, 724 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n", 725 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset, 726 pkt_dev->cur_src_mac_offset); 727 728 if (pkt_dev->flags & F_IPV6) { 729 seq_printf(seq, " cur_saddr: %pI6c cur_daddr: %pI6c\n", 730 &pkt_dev->cur_in6_saddr, 731 &pkt_dev->cur_in6_daddr); 732 } else 733 seq_printf(seq, " cur_saddr: %pI4 cur_daddr: %pI4\n", 734 &pkt_dev->cur_saddr, &pkt_dev->cur_daddr); 735 736 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n", 737 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src); 738 739 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map); 740 741 seq_printf(seq, " flows: %u\n", pkt_dev->nflows); 742 743 if (pkt_dev->result[0]) 744 seq_printf(seq, "Result: %s\n", pkt_dev->result); 745 else 746 seq_puts(seq, "Result: Idle\n"); 747 748 return 0; 749 } 750 751 752 static ssize_t hex32_arg(const char __user *user_buffer, size_t maxlen, 753 __u32 *num) 754 { 755 size_t i = 0; 756 757 *num = 0; 758 759 for (; i < maxlen; i++) { 760 int value; 761 char c; 762 763 if (get_user(c, &user_buffer[i])) 764 return -EFAULT; 765 value = hex_to_bin(c); 766 if (value >= 0) { 767 *num <<= 4; 768 *num |= value; 769 } else { 770 break; 771 } 772 } 773 return i; 774 } 775 776 static ssize_t count_trail_chars(const char __user *user_buffer, size_t maxlen) 777 { 778 size_t i; 779 780 for (i = 0; i < maxlen; i++) { 781 char c; 782 783 if (get_user(c, &user_buffer[i])) 784 return -EFAULT; 785 switch (c) { 786 case '\"': 787 case '\n': 788 case '\r': 789 case '\t': 790 case ' ': 791 case '=': 792 break; 793 default: 794 goto done; 795 } 796 } 797 done: 798 return i; 799 } 800 801 static ssize_t num_arg(const char __user *user_buffer, size_t maxlen, 802 unsigned long *num) 803 { 804 size_t i; 805 *num = 0; 806 807 for (i = 0; i < maxlen; i++) { 808 char c; 809 810 if (get_user(c, &user_buffer[i])) 811 return -EFAULT; 812 if ((c >= '0') && (c <= '9')) { 813 *num *= 10; 814 *num += c - '0'; 815 } else 816 break; 817 } 818 return i; 819 } 820 821 static ssize_t strn_len(const char __user *user_buffer, size_t maxlen) 822 { 823 size_t i; 824 825 for (i = 0; i < maxlen; i++) { 826 char c; 827 828 if (get_user(c, &user_buffer[i])) 829 return -EFAULT; 830 switch (c) { 831 case '\"': 832 case '\n': 833 case '\r': 834 case '\t': 835 case ' ': 836 case '=': 837 goto done_str; 838 default: 839 break; 840 } 841 } 842 done_str: 843 return i; 844 } 845 846 /* Parses imix entries from user buffer. 847 * The user buffer should consist of imix entries separated by spaces 848 * where each entry consists of size and weight delimited by commas. 849 * "size1,weight_1 size2,weight_2 ... size_n,weight_n" for example. 850 */ 851 static ssize_t get_imix_entries(const char __user *buffer, 852 size_t maxlen, 853 struct pktgen_dev *pkt_dev) 854 { 855 size_t i = 0, max; 856 ssize_t len; 857 char c; 858 859 pkt_dev->n_imix_entries = 0; 860 861 do { 862 unsigned long weight; 863 unsigned long size; 864 865 if (pkt_dev->n_imix_entries >= MAX_IMIX_ENTRIES) 866 return -E2BIG; 867 868 if (i >= maxlen) 869 return -EINVAL; 870 871 max = min(10, maxlen - i); 872 len = num_arg(&buffer[i], max, &size); 873 if (len < 0) 874 return len; 875 i += len; 876 if (i >= maxlen) 877 return -EINVAL; 878 if (get_user(c, &buffer[i])) 879 return -EFAULT; 880 /* Check for comma between size_i and weight_i */ 881 if (c != ',') 882 return -EINVAL; 883 i++; 884 if (i >= maxlen) 885 return -EINVAL; 886 887 if (size < 14 + 20 + 8) 888 size = 14 + 20 + 8; 889 890 max = min(10, maxlen - i); 891 len = num_arg(&buffer[i], max, &weight); 892 if (len < 0) 893 return len; 894 if (weight <= 0) 895 return -EINVAL; 896 897 pkt_dev->imix_entries[pkt_dev->n_imix_entries].size = size; 898 pkt_dev->imix_entries[pkt_dev->n_imix_entries].weight = weight; 899 900 i += len; 901 pkt_dev->n_imix_entries++; 902 903 if (i >= maxlen) 904 break; 905 if (get_user(c, &buffer[i])) 906 return -EFAULT; 907 i++; 908 } while (c == ' '); 909 910 return i; 911 } 912 913 static ssize_t get_labels(const char __user *buffer, 914 size_t maxlen, struct pktgen_dev *pkt_dev) 915 { 916 unsigned int n = 0; 917 size_t i = 0, max; 918 ssize_t len; 919 char c; 920 921 pkt_dev->nr_labels = 0; 922 do { 923 __u32 tmp; 924 925 if (n >= MAX_MPLS_LABELS) 926 return -E2BIG; 927 928 if (i >= maxlen) 929 return -EINVAL; 930 931 max = min(8, maxlen - i); 932 len = hex32_arg(&buffer[i], max, &tmp); 933 if (len < 0) 934 return len; 935 936 /* return empty list in case of invalid input or zero value */ 937 if (len == 0 || tmp == 0) 938 return maxlen; 939 940 pkt_dev->labels[n] = htonl(tmp); 941 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM) 942 pkt_dev->flags |= F_MPLS_RND; 943 i += len; 944 n++; 945 if (i >= maxlen) 946 break; 947 if (get_user(c, &buffer[i])) 948 return -EFAULT; 949 i++; 950 } while (c == ','); 951 952 pkt_dev->nr_labels = n; 953 return i; 954 } 955 956 static __u32 pktgen_read_flag(const char *f, bool *disable) 957 { 958 __u32 i; 959 960 if (f[0] == '!') { 961 *disable = true; 962 f++; 963 } 964 965 for (i = 0; i < NR_PKT_FLAGS; i++) { 966 if (!IS_ENABLED(CONFIG_XFRM) && i == IPSEC_SHIFT) 967 continue; 968 969 /* allow only disabling ipv6 flag */ 970 if (!*disable && i == IPV6_SHIFT) 971 continue; 972 973 if (strcmp(f, pkt_flag_names[i]) == 0) 974 return 1 << i; 975 } 976 977 if (strcmp(f, "FLOW_RND") == 0) { 978 *disable = !*disable; 979 return F_FLOW_SEQ; 980 } 981 982 return 0; 983 } 984 985 static ssize_t pktgen_if_write(struct file *file, 986 const char __user *user_buffer, size_t count, 987 loff_t *offset) 988 { 989 struct seq_file *seq = file->private_data; 990 struct pktgen_dev *pkt_dev = seq->private; 991 size_t i, max; 992 ssize_t len; 993 char name[16], valstr[32]; 994 unsigned long value = 0; 995 char *pg_result = NULL; 996 char buf[128]; 997 998 pg_result = &(pkt_dev->result[0]); 999 1000 if (count < 1) { 1001 pr_warn("wrong command format\n"); 1002 return -EINVAL; 1003 } 1004 1005 max = count; 1006 len = count_trail_chars(user_buffer, max); 1007 if (len < 0) { 1008 pr_warn("illegal format\n"); 1009 return len; 1010 } 1011 i = len; 1012 1013 /* Read variable name */ 1014 max = min(sizeof(name) - 1, count - i); 1015 len = strn_len(&user_buffer[i], max); 1016 if (len < 0) 1017 return len; 1018 1019 memset(name, 0, sizeof(name)); 1020 if (copy_from_user(name, &user_buffer[i], len)) 1021 return -EFAULT; 1022 i += len; 1023 1024 max = count - i; 1025 len = count_trail_chars(&user_buffer[i], max); 1026 if (len < 0) 1027 return len; 1028 1029 i += len; 1030 1031 if (debug) { 1032 size_t copy = min_t(size_t, count + 1, 1024); 1033 char *tp = strndup_user(user_buffer, copy); 1034 1035 if (IS_ERR(tp)) 1036 return PTR_ERR(tp); 1037 1038 pr_debug("%s,%zu buffer -:%s:-\n", name, count, tp); 1039 kfree(tp); 1040 } 1041 1042 if (!strcmp(name, "min_pkt_size")) { 1043 max = min(10, count - i); 1044 len = num_arg(&user_buffer[i], max, &value); 1045 if (len < 0) 1046 return len; 1047 1048 if (value < 14 + 20 + 8) 1049 value = 14 + 20 + 8; 1050 if (value != pkt_dev->min_pkt_size) { 1051 pkt_dev->min_pkt_size = value; 1052 pkt_dev->cur_pkt_size = value; 1053 } 1054 sprintf(pg_result, "OK: min_pkt_size=%d", 1055 pkt_dev->min_pkt_size); 1056 return count; 1057 } 1058 1059 if (!strcmp(name, "max_pkt_size")) { 1060 max = min(10, count - i); 1061 len = num_arg(&user_buffer[i], max, &value); 1062 if (len < 0) 1063 return len; 1064 1065 if (value < 14 + 20 + 8) 1066 value = 14 + 20 + 8; 1067 if (value != pkt_dev->max_pkt_size) { 1068 pkt_dev->max_pkt_size = value; 1069 pkt_dev->cur_pkt_size = value; 1070 } 1071 sprintf(pg_result, "OK: max_pkt_size=%d", 1072 pkt_dev->max_pkt_size); 1073 return count; 1074 } 1075 1076 /* Shortcut for min = max */ 1077 1078 if (!strcmp(name, "pkt_size")) { 1079 max = min(10, count - i); 1080 len = num_arg(&user_buffer[i], max, &value); 1081 if (len < 0) 1082 return len; 1083 1084 if (value < 14 + 20 + 8) 1085 value = 14 + 20 + 8; 1086 if (value != pkt_dev->min_pkt_size) { 1087 pkt_dev->min_pkt_size = value; 1088 pkt_dev->max_pkt_size = value; 1089 pkt_dev->cur_pkt_size = value; 1090 } 1091 sprintf(pg_result, "OK: pkt_size=%d", pkt_dev->min_pkt_size); 1092 return count; 1093 } 1094 1095 if (!strcmp(name, "imix_weights")) { 1096 if (pkt_dev->clone_skb > 0) 1097 return -EINVAL; 1098 1099 max = count - i; 1100 len = get_imix_entries(&user_buffer[i], max, pkt_dev); 1101 if (len < 0) 1102 return len; 1103 1104 fill_imix_distribution(pkt_dev); 1105 1106 return count; 1107 } 1108 1109 if (!strcmp(name, "debug")) { 1110 max = min(10, count - i); 1111 len = num_arg(&user_buffer[i], max, &value); 1112 if (len < 0) 1113 return len; 1114 1115 debug = value; 1116 sprintf(pg_result, "OK: debug=%u", debug); 1117 return count; 1118 } 1119 1120 if (!strcmp(name, "frags")) { 1121 max = min(10, count - i); 1122 len = num_arg(&user_buffer[i], max, &value); 1123 if (len < 0) 1124 return len; 1125 1126 pkt_dev->nfrags = value; 1127 sprintf(pg_result, "OK: frags=%d", pkt_dev->nfrags); 1128 return count; 1129 } 1130 if (!strcmp(name, "delay")) { 1131 max = min(10, count - i); 1132 len = num_arg(&user_buffer[i], max, &value); 1133 if (len < 0) 1134 return len; 1135 1136 if (value == 0x7FFFFFFF) 1137 pkt_dev->delay = ULLONG_MAX; 1138 else 1139 pkt_dev->delay = (u64)value; 1140 1141 sprintf(pg_result, "OK: delay=%llu", 1142 (unsigned long long) pkt_dev->delay); 1143 return count; 1144 } 1145 if (!strcmp(name, "rate")) { 1146 max = min(10, count - i); 1147 len = num_arg(&user_buffer[i], max, &value); 1148 if (len < 0) 1149 return len; 1150 1151 if (!value) 1152 return -EINVAL; 1153 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value; 1154 if (debug) 1155 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1156 1157 sprintf(pg_result, "OK: rate=%lu", value); 1158 return count; 1159 } 1160 if (!strcmp(name, "ratep")) { 1161 max = min(10, count - i); 1162 len = num_arg(&user_buffer[i], max, &value); 1163 if (len < 0) 1164 return len; 1165 1166 if (!value) 1167 return -EINVAL; 1168 pkt_dev->delay = NSEC_PER_SEC/value; 1169 if (debug) 1170 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1171 1172 sprintf(pg_result, "OK: rate=%lu", value); 1173 return count; 1174 } 1175 if (!strcmp(name, "udp_src_min")) { 1176 max = min(10, count - i); 1177 len = num_arg(&user_buffer[i], max, &value); 1178 if (len < 0) 1179 return len; 1180 1181 if (value != pkt_dev->udp_src_min) { 1182 pkt_dev->udp_src_min = value; 1183 pkt_dev->cur_udp_src = value; 1184 } 1185 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min); 1186 return count; 1187 } 1188 if (!strcmp(name, "udp_dst_min")) { 1189 max = min(10, count - i); 1190 len = num_arg(&user_buffer[i], max, &value); 1191 if (len < 0) 1192 return len; 1193 1194 if (value != pkt_dev->udp_dst_min) { 1195 pkt_dev->udp_dst_min = value; 1196 pkt_dev->cur_udp_dst = value; 1197 } 1198 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min); 1199 return count; 1200 } 1201 if (!strcmp(name, "udp_src_max")) { 1202 max = min(10, count - i); 1203 len = num_arg(&user_buffer[i], max, &value); 1204 if (len < 0) 1205 return len; 1206 1207 if (value != pkt_dev->udp_src_max) { 1208 pkt_dev->udp_src_max = value; 1209 pkt_dev->cur_udp_src = value; 1210 } 1211 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max); 1212 return count; 1213 } 1214 if (!strcmp(name, "udp_dst_max")) { 1215 max = min(10, count - i); 1216 len = num_arg(&user_buffer[i], max, &value); 1217 if (len < 0) 1218 return len; 1219 1220 if (value != pkt_dev->udp_dst_max) { 1221 pkt_dev->udp_dst_max = value; 1222 pkt_dev->cur_udp_dst = value; 1223 } 1224 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max); 1225 return count; 1226 } 1227 if (!strcmp(name, "clone_skb")) { 1228 max = min(10, count - i); 1229 len = num_arg(&user_buffer[i], max, &value); 1230 if (len < 0) 1231 return len; 1232 /* clone_skb is not supported for netif_receive xmit_mode and 1233 * IMIX mode. 1234 */ 1235 if ((value > 0) && 1236 ((pkt_dev->xmit_mode == M_NETIF_RECEIVE) || 1237 !(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))) 1238 return -EOPNOTSUPP; 1239 if (value > 0 && (pkt_dev->n_imix_entries > 0 || 1240 !(pkt_dev->flags & F_SHARED))) 1241 return -EINVAL; 1242 1243 pkt_dev->clone_skb = value; 1244 1245 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb); 1246 return count; 1247 } 1248 if (!strcmp(name, "count")) { 1249 max = min(10, count - i); 1250 len = num_arg(&user_buffer[i], max, &value); 1251 if (len < 0) 1252 return len; 1253 1254 pkt_dev->count = value; 1255 sprintf(pg_result, "OK: count=%llu", 1256 (unsigned long long)pkt_dev->count); 1257 return count; 1258 } 1259 if (!strcmp(name, "src_mac_count")) { 1260 max = min(10, count - i); 1261 len = num_arg(&user_buffer[i], max, &value); 1262 if (len < 0) 1263 return len; 1264 1265 if (pkt_dev->src_mac_count != value) { 1266 pkt_dev->src_mac_count = value; 1267 pkt_dev->cur_src_mac_offset = 0; 1268 } 1269 sprintf(pg_result, "OK: src_mac_count=%d", 1270 pkt_dev->src_mac_count); 1271 return count; 1272 } 1273 if (!strcmp(name, "dst_mac_count")) { 1274 max = min(10, count - i); 1275 len = num_arg(&user_buffer[i], max, &value); 1276 if (len < 0) 1277 return len; 1278 1279 if (pkt_dev->dst_mac_count != value) { 1280 pkt_dev->dst_mac_count = value; 1281 pkt_dev->cur_dst_mac_offset = 0; 1282 } 1283 sprintf(pg_result, "OK: dst_mac_count=%d", 1284 pkt_dev->dst_mac_count); 1285 return count; 1286 } 1287 if (!strcmp(name, "burst")) { 1288 max = min(10, count - i); 1289 len = num_arg(&user_buffer[i], max, &value); 1290 if (len < 0) 1291 return len; 1292 1293 if ((value > 1) && 1294 ((pkt_dev->xmit_mode == M_QUEUE_XMIT) || 1295 ((pkt_dev->xmit_mode == M_START_XMIT) && 1296 (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))))) 1297 return -EOPNOTSUPP; 1298 1299 if (value > 1 && !(pkt_dev->flags & F_SHARED)) 1300 return -EINVAL; 1301 1302 pkt_dev->burst = value < 1 ? 1 : value; 1303 sprintf(pg_result, "OK: burst=%u", pkt_dev->burst); 1304 return count; 1305 } 1306 if (!strcmp(name, "node")) { 1307 max = min(10, count - i); 1308 len = num_arg(&user_buffer[i], max, &value); 1309 if (len < 0) 1310 return len; 1311 1312 if (node_possible(value)) { 1313 pkt_dev->node = value; 1314 sprintf(pg_result, "OK: node=%d", pkt_dev->node); 1315 if (pkt_dev->page) { 1316 put_page(pkt_dev->page); 1317 pkt_dev->page = NULL; 1318 } 1319 } else { 1320 sprintf(pg_result, "ERROR: node not possible"); 1321 } 1322 return count; 1323 } 1324 if (!strcmp(name, "xmit_mode")) { 1325 char f[32]; 1326 1327 max = min(sizeof(f) - 1, count - i); 1328 len = strn_len(&user_buffer[i], max); 1329 if (len < 0) 1330 return len; 1331 1332 memset(f, 0, sizeof(f)); 1333 if (copy_from_user(f, &user_buffer[i], len)) 1334 return -EFAULT; 1335 1336 if (strcmp(f, "start_xmit") == 0) { 1337 pkt_dev->xmit_mode = M_START_XMIT; 1338 } else if (strcmp(f, "netif_receive") == 0) { 1339 /* clone_skb set earlier, not supported in this mode */ 1340 if (pkt_dev->clone_skb > 0) 1341 return -EOPNOTSUPP; 1342 1343 pkt_dev->xmit_mode = M_NETIF_RECEIVE; 1344 1345 /* make sure new packet is allocated every time 1346 * pktgen_xmit() is called 1347 */ 1348 pkt_dev->last_ok = 1; 1349 } else if (strcmp(f, "queue_xmit") == 0) { 1350 pkt_dev->xmit_mode = M_QUEUE_XMIT; 1351 pkt_dev->last_ok = 1; 1352 } else { 1353 sprintf(pg_result, 1354 "xmit_mode -:%s:- unknown\nAvailable modes: %s", 1355 f, "start_xmit, netif_receive\n"); 1356 return count; 1357 } 1358 sprintf(pg_result, "OK: xmit_mode=%s", f); 1359 return count; 1360 } 1361 if (!strcmp(name, "flag")) { 1362 bool disable = false; 1363 __u32 flag; 1364 char f[32]; 1365 char *end; 1366 1367 max = min(sizeof(f) - 1, count - i); 1368 len = strn_len(&user_buffer[i], max); 1369 if (len < 0) 1370 return len; 1371 1372 memset(f, 0, 32); 1373 if (copy_from_user(f, &user_buffer[i], len)) 1374 return -EFAULT; 1375 1376 flag = pktgen_read_flag(f, &disable); 1377 if (flag) { 1378 if (disable) { 1379 /* If "clone_skb", or "burst" parameters are 1380 * configured, it means that the skb still 1381 * needs to be referenced by the pktgen, so 1382 * the skb must be shared. 1383 */ 1384 if (flag == F_SHARED && (pkt_dev->clone_skb || 1385 pkt_dev->burst > 1)) 1386 return -EINVAL; 1387 pkt_dev->flags &= ~flag; 1388 } else { 1389 pkt_dev->flags |= flag; 1390 } 1391 1392 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags); 1393 return count; 1394 } 1395 1396 /* Unknown flag */ 1397 end = pkt_dev->result + sizeof(pkt_dev->result); 1398 pg_result += sprintf(pg_result, 1399 "Flag -:%s:- unknown\n" 1400 "Available flags, (prepend ! to un-set flag):\n", f); 1401 1402 for (int n = 0; n < NR_PKT_FLAGS && pg_result < end; n++) { 1403 if (!IS_ENABLED(CONFIG_XFRM) && n == IPSEC_SHIFT) 1404 continue; 1405 pg_result += snprintf(pg_result, end - pg_result, 1406 "%s, ", pkt_flag_names[n]); 1407 } 1408 if (!WARN_ON_ONCE(pg_result >= end)) { 1409 /* Remove the comma and whitespace at the end */ 1410 *(pg_result - 2) = '\0'; 1411 } 1412 1413 return count; 1414 } 1415 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) { 1416 max = min(sizeof(pkt_dev->dst_min) - 1, count - i); 1417 len = strn_len(&user_buffer[i], max); 1418 if (len < 0) 1419 return len; 1420 1421 if (copy_from_user(buf, &user_buffer[i], len)) 1422 return -EFAULT; 1423 buf[len] = 0; 1424 if (strcmp(buf, pkt_dev->dst_min) != 0) { 1425 strscpy_pad(pkt_dev->dst_min, buf); 1426 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 1427 pkt_dev->cur_daddr = pkt_dev->daddr_min; 1428 } 1429 if (debug) 1430 pr_debug("dst_min set to: %s\n", pkt_dev->dst_min); 1431 1432 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min); 1433 return count; 1434 } 1435 if (!strcmp(name, "dst_max")) { 1436 max = min(sizeof(pkt_dev->dst_max) - 1, count - i); 1437 len = strn_len(&user_buffer[i], max); 1438 if (len < 0) 1439 return len; 1440 1441 if (copy_from_user(buf, &user_buffer[i], len)) 1442 return -EFAULT; 1443 buf[len] = 0; 1444 if (strcmp(buf, pkt_dev->dst_max) != 0) { 1445 strscpy_pad(pkt_dev->dst_max, buf); 1446 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 1447 pkt_dev->cur_daddr = pkt_dev->daddr_max; 1448 } 1449 if (debug) 1450 pr_debug("dst_max set to: %s\n", pkt_dev->dst_max); 1451 1452 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max); 1453 return count; 1454 } 1455 if (!strcmp(name, "dst6")) { 1456 max = min(sizeof(buf) - 1, count - i); 1457 len = strn_len(&user_buffer[i], max); 1458 if (len < 0) 1459 return len; 1460 1461 pkt_dev->flags |= F_IPV6; 1462 1463 if (copy_from_user(buf, &user_buffer[i], len)) 1464 return -EFAULT; 1465 buf[len] = 0; 1466 1467 in6_pton(buf, -1, pkt_dev->in6_daddr.s6_addr, -1, NULL); 1468 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr); 1469 1470 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr; 1471 1472 if (debug) 1473 pr_debug("dst6 set to: %s\n", buf); 1474 1475 sprintf(pg_result, "OK: dst6=%s", buf); 1476 return count; 1477 } 1478 if (!strcmp(name, "dst6_min")) { 1479 max = min(sizeof(buf) - 1, count - i); 1480 len = strn_len(&user_buffer[i], max); 1481 if (len < 0) 1482 return len; 1483 1484 pkt_dev->flags |= F_IPV6; 1485 1486 if (copy_from_user(buf, &user_buffer[i], len)) 1487 return -EFAULT; 1488 buf[len] = 0; 1489 1490 in6_pton(buf, -1, pkt_dev->min_in6_daddr.s6_addr, -1, NULL); 1491 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr); 1492 1493 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr; 1494 if (debug) 1495 pr_debug("dst6_min set to: %s\n", buf); 1496 1497 sprintf(pg_result, "OK: dst6_min=%s", buf); 1498 return count; 1499 } 1500 if (!strcmp(name, "dst6_max")) { 1501 max = min(sizeof(buf) - 1, count - i); 1502 len = strn_len(&user_buffer[i], max); 1503 if (len < 0) 1504 return len; 1505 1506 pkt_dev->flags |= F_IPV6; 1507 1508 if (copy_from_user(buf, &user_buffer[i], len)) 1509 return -EFAULT; 1510 buf[len] = 0; 1511 1512 in6_pton(buf, -1, pkt_dev->max_in6_daddr.s6_addr, -1, NULL); 1513 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr); 1514 1515 if (debug) 1516 pr_debug("dst6_max set to: %s\n", buf); 1517 1518 sprintf(pg_result, "OK: dst6_max=%s", buf); 1519 return count; 1520 } 1521 if (!strcmp(name, "src6")) { 1522 max = min(sizeof(buf) - 1, count - i); 1523 len = strn_len(&user_buffer[i], max); 1524 if (len < 0) 1525 return len; 1526 1527 pkt_dev->flags |= F_IPV6; 1528 1529 if (copy_from_user(buf, &user_buffer[i], len)) 1530 return -EFAULT; 1531 buf[len] = 0; 1532 1533 in6_pton(buf, -1, pkt_dev->in6_saddr.s6_addr, -1, NULL); 1534 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr); 1535 1536 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr; 1537 1538 if (debug) 1539 pr_debug("src6 set to: %s\n", buf); 1540 1541 sprintf(pg_result, "OK: src6=%s", buf); 1542 return count; 1543 } 1544 if (!strcmp(name, "src_min")) { 1545 max = min(sizeof(pkt_dev->src_min) - 1, count - i); 1546 len = strn_len(&user_buffer[i], max); 1547 if (len < 0) 1548 return len; 1549 1550 if (copy_from_user(buf, &user_buffer[i], len)) 1551 return -EFAULT; 1552 buf[len] = 0; 1553 if (strcmp(buf, pkt_dev->src_min) != 0) { 1554 strscpy_pad(pkt_dev->src_min, buf); 1555 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 1556 pkt_dev->cur_saddr = pkt_dev->saddr_min; 1557 } 1558 if (debug) 1559 pr_debug("src_min set to: %s\n", pkt_dev->src_min); 1560 1561 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min); 1562 return count; 1563 } 1564 if (!strcmp(name, "src_max")) { 1565 max = min(sizeof(pkt_dev->src_max) - 1, count - i); 1566 len = strn_len(&user_buffer[i], max); 1567 if (len < 0) 1568 return len; 1569 1570 if (copy_from_user(buf, &user_buffer[i], len)) 1571 return -EFAULT; 1572 buf[len] = 0; 1573 if (strcmp(buf, pkt_dev->src_max) != 0) { 1574 strscpy_pad(pkt_dev->src_max, buf); 1575 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 1576 pkt_dev->cur_saddr = pkt_dev->saddr_max; 1577 } 1578 if (debug) 1579 pr_debug("src_max set to: %s\n", pkt_dev->src_max); 1580 1581 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max); 1582 return count; 1583 } 1584 if (!strcmp(name, "dst_mac")) { 1585 max = min(sizeof(valstr) - 1, count - i); 1586 len = strn_len(&user_buffer[i], max); 1587 if (len < 0) 1588 return len; 1589 1590 memset(valstr, 0, sizeof(valstr)); 1591 if (copy_from_user(valstr, &user_buffer[i], len)) 1592 return -EFAULT; 1593 1594 if (!mac_pton(valstr, pkt_dev->dst_mac)) 1595 return -EINVAL; 1596 /* Set up Dest MAC */ 1597 ether_addr_copy(&pkt_dev->hh[0], pkt_dev->dst_mac); 1598 1599 sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac); 1600 return count; 1601 } 1602 if (!strcmp(name, "src_mac")) { 1603 max = min(sizeof(valstr) - 1, count - i); 1604 len = strn_len(&user_buffer[i], max); 1605 if (len < 0) 1606 return len; 1607 1608 memset(valstr, 0, sizeof(valstr)); 1609 if (copy_from_user(valstr, &user_buffer[i], len)) 1610 return -EFAULT; 1611 1612 if (!mac_pton(valstr, pkt_dev->src_mac)) 1613 return -EINVAL; 1614 /* Set up Src MAC */ 1615 ether_addr_copy(&pkt_dev->hh[6], pkt_dev->src_mac); 1616 1617 sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac); 1618 return count; 1619 } 1620 1621 if (!strcmp(name, "clear_counters")) { 1622 pktgen_clear_counters(pkt_dev); 1623 sprintf(pg_result, "OK: Clearing counters.\n"); 1624 return count; 1625 } 1626 1627 if (!strcmp(name, "flows")) { 1628 max = min(10, count - i); 1629 len = num_arg(&user_buffer[i], max, &value); 1630 if (len < 0) 1631 return len; 1632 1633 if (value > MAX_CFLOWS) 1634 value = MAX_CFLOWS; 1635 1636 WRITE_ONCE(pkt_dev->cflows, value); 1637 sprintf(pg_result, "OK: flows=%u", (unsigned int)value); 1638 return count; 1639 } 1640 #ifdef CONFIG_XFRM 1641 if (!strcmp(name, "spi")) { 1642 max = min(10, count - i); 1643 len = num_arg(&user_buffer[i], max, &value); 1644 if (len < 0) 1645 return len; 1646 1647 pkt_dev->spi = value; 1648 sprintf(pg_result, "OK: spi=%u", pkt_dev->spi); 1649 return count; 1650 } 1651 #endif 1652 if (!strcmp(name, "flowlen")) { 1653 max = min(10, count - i); 1654 len = num_arg(&user_buffer[i], max, &value); 1655 if (len < 0) 1656 return len; 1657 1658 pkt_dev->lflow = value; 1659 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow); 1660 return count; 1661 } 1662 1663 if (!strcmp(name, "queue_map_min")) { 1664 max = min(5, count - i); 1665 len = num_arg(&user_buffer[i], max, &value); 1666 if (len < 0) 1667 return len; 1668 1669 pkt_dev->queue_map_min = value; 1670 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min); 1671 return count; 1672 } 1673 1674 if (!strcmp(name, "queue_map_max")) { 1675 max = min(5, count - i); 1676 len = num_arg(&user_buffer[i], max, &value); 1677 if (len < 0) 1678 return len; 1679 1680 pkt_dev->queue_map_max = value; 1681 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max); 1682 return count; 1683 } 1684 1685 if (!strcmp(name, "mpls")) { 1686 unsigned int n, cnt; 1687 1688 max = count - i; 1689 len = get_labels(&user_buffer[i], max, pkt_dev); 1690 if (len < 0) 1691 return len; 1692 1693 cnt = sprintf(pg_result, "OK: mpls="); 1694 for (n = 0; n < pkt_dev->nr_labels; n++) 1695 cnt += sprintf(pg_result + cnt, 1696 "%08x%s", ntohl(pkt_dev->labels[n]), 1697 n == pkt_dev->nr_labels-1 ? "" : ","); 1698 1699 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) { 1700 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1701 pkt_dev->svlan_id = 0xffff; 1702 1703 if (debug) 1704 pr_debug("VLAN/SVLAN auto turned off\n"); 1705 } 1706 return count; 1707 } 1708 1709 if (!strcmp(name, "vlan_id")) { 1710 max = min(4, count - i); 1711 len = num_arg(&user_buffer[i], max, &value); 1712 if (len < 0) 1713 return len; 1714 1715 if (value <= 4095) { 1716 pkt_dev->vlan_id = value; /* turn on VLAN */ 1717 1718 if (debug) 1719 pr_debug("VLAN turned on\n"); 1720 1721 if (debug && pkt_dev->nr_labels) 1722 pr_debug("MPLS auto turned off\n"); 1723 1724 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1725 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id); 1726 } else { 1727 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1728 pkt_dev->svlan_id = 0xffff; 1729 1730 if (debug) 1731 pr_debug("VLAN/SVLAN turned off\n"); 1732 } 1733 return count; 1734 } 1735 1736 if (!strcmp(name, "vlan_p")) { 1737 max = min(1, count - i); 1738 len = num_arg(&user_buffer[i], max, &value); 1739 if (len < 0) 1740 return len; 1741 1742 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) { 1743 pkt_dev->vlan_p = value; 1744 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p); 1745 } else { 1746 sprintf(pg_result, "ERROR: vlan_p must be 0-7"); 1747 } 1748 return count; 1749 } 1750 1751 if (!strcmp(name, "vlan_cfi")) { 1752 max = min(1, count - i); 1753 len = num_arg(&user_buffer[i], max, &value); 1754 if (len < 0) 1755 return len; 1756 1757 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) { 1758 pkt_dev->vlan_cfi = value; 1759 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi); 1760 } else { 1761 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1"); 1762 } 1763 return count; 1764 } 1765 1766 if (!strcmp(name, "svlan_id")) { 1767 max = min(4, count - i); 1768 len = num_arg(&user_buffer[i], max, &value); 1769 if (len < 0) 1770 return len; 1771 1772 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) { 1773 pkt_dev->svlan_id = value; /* turn on SVLAN */ 1774 1775 if (debug) 1776 pr_debug("SVLAN turned on\n"); 1777 1778 if (debug && pkt_dev->nr_labels) 1779 pr_debug("MPLS auto turned off\n"); 1780 1781 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1782 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id); 1783 } else { 1784 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1785 pkt_dev->svlan_id = 0xffff; 1786 1787 if (debug) 1788 pr_debug("VLAN/SVLAN turned off\n"); 1789 } 1790 return count; 1791 } 1792 1793 if (!strcmp(name, "svlan_p")) { 1794 max = min(1, count - i); 1795 len = num_arg(&user_buffer[i], max, &value); 1796 if (len < 0) 1797 return len; 1798 1799 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) { 1800 pkt_dev->svlan_p = value; 1801 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p); 1802 } else { 1803 sprintf(pg_result, "ERROR: svlan_p must be 0-7"); 1804 } 1805 return count; 1806 } 1807 1808 if (!strcmp(name, "svlan_cfi")) { 1809 max = min(1, count - i); 1810 len = num_arg(&user_buffer[i], max, &value); 1811 if (len < 0) 1812 return len; 1813 1814 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) { 1815 pkt_dev->svlan_cfi = value; 1816 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi); 1817 } else { 1818 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1"); 1819 } 1820 return count; 1821 } 1822 1823 if (!strcmp(name, "tos")) { 1824 __u32 tmp_value; 1825 1826 max = min(2, count - i); 1827 len = hex32_arg(&user_buffer[i], max, &tmp_value); 1828 if (len < 0) 1829 return len; 1830 1831 if (len == 2) { 1832 pkt_dev->tos = tmp_value; 1833 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos); 1834 } else { 1835 sprintf(pg_result, "ERROR: tos must be 00-ff"); 1836 } 1837 return count; 1838 } 1839 1840 if (!strcmp(name, "traffic_class")) { 1841 __u32 tmp_value; 1842 1843 max = min(2, count - i); 1844 len = hex32_arg(&user_buffer[i], max, &tmp_value); 1845 if (len < 0) 1846 return len; 1847 1848 if (len == 2) { 1849 pkt_dev->traffic_class = tmp_value; 1850 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class); 1851 } else { 1852 sprintf(pg_result, "ERROR: traffic_class must be 00-ff"); 1853 } 1854 return count; 1855 } 1856 1857 if (!strcmp(name, "skb_priority")) { 1858 max = min(9, count - i); 1859 len = num_arg(&user_buffer[i], max, &value); 1860 if (len < 0) 1861 return len; 1862 1863 pkt_dev->skb_priority = value; 1864 sprintf(pg_result, "OK: skb_priority=%i", 1865 pkt_dev->skb_priority); 1866 return count; 1867 } 1868 1869 sprintf(pkt_dev->result, "No such parameter \"%s\"", name); 1870 return -EINVAL; 1871 } 1872 1873 static int pktgen_if_open(struct inode *inode, struct file *file) 1874 { 1875 return single_open(file, pktgen_if_show, pde_data(inode)); 1876 } 1877 1878 static const struct proc_ops pktgen_if_proc_ops = { 1879 .proc_open = pktgen_if_open, 1880 .proc_read = seq_read, 1881 .proc_lseek = seq_lseek, 1882 .proc_write = pktgen_if_write, 1883 .proc_release = single_release, 1884 }; 1885 1886 static int pktgen_thread_show(struct seq_file *seq, void *v) 1887 { 1888 struct pktgen_thread *t = seq->private; 1889 const struct pktgen_dev *pkt_dev; 1890 1891 BUG_ON(!t); 1892 1893 seq_puts(seq, "Running: "); 1894 1895 rcu_read_lock(); 1896 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 1897 if (pkt_dev->running) 1898 seq_printf(seq, "%s ", pkt_dev->odevname); 1899 1900 seq_puts(seq, "\nStopped: "); 1901 1902 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 1903 if (!pkt_dev->running) 1904 seq_printf(seq, "%s ", pkt_dev->odevname); 1905 1906 if (t->result[0]) 1907 seq_printf(seq, "\nResult: %s\n", t->result); 1908 else 1909 seq_puts(seq, "\nResult: NA\n"); 1910 1911 rcu_read_unlock(); 1912 1913 return 0; 1914 } 1915 1916 static ssize_t pktgen_thread_write(struct file *file, 1917 const char __user *user_buffer, 1918 size_t count, loff_t *offset) 1919 { 1920 struct seq_file *seq = file->private_data; 1921 struct pktgen_thread *t = seq->private; 1922 size_t i, max; 1923 ssize_t len, ret; 1924 char name[40]; 1925 char *pg_result; 1926 1927 if (count < 1) { 1928 // sprintf(pg_result, "Wrong command format"); 1929 return -EINVAL; 1930 } 1931 1932 max = count; 1933 len = count_trail_chars(user_buffer, max); 1934 if (len < 0) 1935 return len; 1936 1937 i = len; 1938 1939 /* Read variable name */ 1940 max = min(sizeof(name) - 1, count - i); 1941 len = strn_len(&user_buffer[i], max); 1942 if (len < 0) 1943 return len; 1944 1945 memset(name, 0, sizeof(name)); 1946 if (copy_from_user(name, &user_buffer[i], len)) 1947 return -EFAULT; 1948 i += len; 1949 1950 max = count - i; 1951 len = count_trail_chars(&user_buffer[i], max); 1952 if (len < 0) 1953 return len; 1954 1955 i += len; 1956 1957 if (debug) 1958 pr_debug("t=%s, count=%lu\n", name, (unsigned long)count); 1959 1960 if (!t) { 1961 pr_err("ERROR: No thread\n"); 1962 ret = -EINVAL; 1963 goto out; 1964 } 1965 1966 pg_result = &(t->result[0]); 1967 1968 if (!strcmp(name, "add_device")) { 1969 char f[32]; 1970 1971 memset(f, 0, 32); 1972 max = min(sizeof(f) - 1, count - i); 1973 len = strn_len(&user_buffer[i], max); 1974 if (len < 0) { 1975 ret = len; 1976 goto out; 1977 } 1978 if (copy_from_user(f, &user_buffer[i], len)) 1979 return -EFAULT; 1980 1981 mutex_lock(&pktgen_thread_lock); 1982 ret = pktgen_add_device(t, f); 1983 mutex_unlock(&pktgen_thread_lock); 1984 if (!ret) { 1985 ret = count; 1986 sprintf(pg_result, "OK: add_device=%s", f); 1987 } else 1988 sprintf(pg_result, "ERROR: can not add device %s", f); 1989 goto out; 1990 } 1991 1992 if (!strcmp(name, "rem_device_all")) { 1993 mutex_lock(&pktgen_thread_lock); 1994 t->control |= T_REMDEVALL; 1995 mutex_unlock(&pktgen_thread_lock); 1996 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */ 1997 ret = count; 1998 sprintf(pg_result, "OK: rem_device_all"); 1999 goto out; 2000 } 2001 2002 if (!strcmp(name, "max_before_softirq")) { 2003 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use"); 2004 ret = count; 2005 goto out; 2006 } 2007 2008 ret = -EINVAL; 2009 out: 2010 return ret; 2011 } 2012 2013 static int pktgen_thread_open(struct inode *inode, struct file *file) 2014 { 2015 return single_open(file, pktgen_thread_show, pde_data(inode)); 2016 } 2017 2018 static const struct proc_ops pktgen_thread_proc_ops = { 2019 .proc_open = pktgen_thread_open, 2020 .proc_read = seq_read, 2021 .proc_lseek = seq_lseek, 2022 .proc_write = pktgen_thread_write, 2023 .proc_release = single_release, 2024 }; 2025 2026 /* Think find or remove for NN */ 2027 static bool __pktgen_NN_threads(const struct pktgen_net *pn, 2028 const char *ifname, int remove) 2029 { 2030 struct pktgen_thread *t; 2031 struct pktgen_dev *pkt_dev; 2032 bool exact = (remove == FIND); 2033 2034 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 2035 pkt_dev = pktgen_find_dev(t, ifname, exact); 2036 if (pkt_dev) { 2037 if (remove) { 2038 pkt_dev->removal_mark = 1; 2039 t->control |= T_REMDEV; 2040 } 2041 return true; 2042 } 2043 } 2044 return false; 2045 } 2046 2047 /* 2048 * mark a device for removal 2049 */ 2050 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname) 2051 { 2052 const int max_tries = 10, msec_per_try = 125; 2053 int i = 0; 2054 2055 mutex_lock(&pktgen_thread_lock); 2056 pr_debug("%s: marking %s for removal\n", __func__, ifname); 2057 2058 while (1) { 2059 2060 if (!__pktgen_NN_threads(pn, ifname, REMOVE)) 2061 break; /* success */ 2062 2063 mutex_unlock(&pktgen_thread_lock); 2064 pr_debug("%s: waiting for %s to disappear....\n", 2065 __func__, ifname); 2066 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try)); 2067 mutex_lock(&pktgen_thread_lock); 2068 2069 if (++i >= max_tries) { 2070 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n", 2071 __func__, msec_per_try * i, ifname); 2072 break; 2073 } 2074 2075 } 2076 2077 mutex_unlock(&pktgen_thread_lock); 2078 } 2079 2080 static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *dev) 2081 { 2082 struct pktgen_thread *t; 2083 2084 mutex_lock(&pktgen_thread_lock); 2085 2086 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 2087 struct pktgen_dev *pkt_dev; 2088 2089 if_lock(t); 2090 list_for_each_entry(pkt_dev, &t->if_list, list) { 2091 if (pkt_dev->odev != dev) 2092 continue; 2093 2094 proc_remove(pkt_dev->entry); 2095 2096 pkt_dev->entry = proc_create_data(dev->name, 0600, 2097 pn->proc_dir, 2098 &pktgen_if_proc_ops, 2099 pkt_dev); 2100 if (!pkt_dev->entry) 2101 pr_err("can't move proc entry for '%s'\n", 2102 dev->name); 2103 break; 2104 } 2105 if_unlock(t); 2106 } 2107 mutex_unlock(&pktgen_thread_lock); 2108 } 2109 2110 static int pktgen_device_event(struct notifier_block *unused, 2111 unsigned long event, void *ptr) 2112 { 2113 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 2114 struct pktgen_net *pn = net_generic(dev_net(dev), pg_net_id); 2115 2116 if (pn->pktgen_exiting) 2117 return NOTIFY_DONE; 2118 2119 /* It is OK that we do not hold the group lock right now, 2120 * as we run under the RTNL lock. 2121 */ 2122 2123 switch (event) { 2124 case NETDEV_CHANGENAME: 2125 pktgen_change_name(pn, dev); 2126 break; 2127 2128 case NETDEV_UNREGISTER: 2129 pktgen_mark_device(pn, dev->name); 2130 break; 2131 } 2132 2133 return NOTIFY_DONE; 2134 } 2135 2136 static struct net_device *pktgen_dev_get_by_name(const struct pktgen_net *pn, 2137 struct pktgen_dev *pkt_dev, 2138 const char *ifname) 2139 { 2140 char b[IFNAMSIZ+5]; 2141 int i; 2142 2143 for (i = 0; ifname[i] != '@'; i++) { 2144 if (i == IFNAMSIZ) 2145 break; 2146 2147 b[i] = ifname[i]; 2148 } 2149 b[i] = 0; 2150 2151 return dev_get_by_name(pn->net, b); 2152 } 2153 2154 2155 /* Associate pktgen_dev with a device. */ 2156 2157 static int pktgen_setup_dev(const struct pktgen_net *pn, 2158 struct pktgen_dev *pkt_dev, const char *ifname) 2159 { 2160 struct net_device *odev; 2161 int err; 2162 2163 /* Clean old setups */ 2164 if (pkt_dev->odev) { 2165 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); 2166 pkt_dev->odev = NULL; 2167 } 2168 2169 odev = pktgen_dev_get_by_name(pn, pkt_dev, ifname); 2170 if (!odev) { 2171 pr_err("no such netdevice: \"%s\"\n", ifname); 2172 return -ENODEV; 2173 } 2174 2175 if (odev->type != ARPHRD_ETHER && odev->type != ARPHRD_LOOPBACK) { 2176 pr_err("not an ethernet or loopback device: \"%s\"\n", ifname); 2177 err = -EINVAL; 2178 } else if (!netif_running(odev)) { 2179 pr_err("device is down: \"%s\"\n", ifname); 2180 err = -ENETDOWN; 2181 } else { 2182 pkt_dev->odev = odev; 2183 netdev_tracker_alloc(odev, &pkt_dev->dev_tracker, GFP_KERNEL); 2184 return 0; 2185 } 2186 2187 dev_put(odev); 2188 return err; 2189 } 2190 2191 /* Read pkt_dev from the interface and set up internal pktgen_dev 2192 * structure to have the right information to create/send packets 2193 */ 2194 static void pktgen_setup_inject(struct pktgen_dev *pkt_dev) 2195 { 2196 int ntxq; 2197 2198 if (!pkt_dev->odev) { 2199 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n"); 2200 sprintf(pkt_dev->result, 2201 "ERROR: pkt_dev->odev == NULL in setup_inject.\n"); 2202 return; 2203 } 2204 2205 /* make sure that we don't pick a non-existing transmit queue */ 2206 ntxq = pkt_dev->odev->real_num_tx_queues; 2207 2208 if (ntxq <= pkt_dev->queue_map_min) { 2209 pr_warn("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2210 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq, 2211 pkt_dev->odevname); 2212 pkt_dev->queue_map_min = (ntxq ?: 1) - 1; 2213 } 2214 if (pkt_dev->queue_map_max >= ntxq) { 2215 pr_warn("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2216 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq, 2217 pkt_dev->odevname); 2218 pkt_dev->queue_map_max = (ntxq ?: 1) - 1; 2219 } 2220 2221 /* Default to the interface's mac if not explicitly set. */ 2222 2223 if (is_zero_ether_addr(pkt_dev->src_mac)) 2224 ether_addr_copy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr); 2225 2226 /* Set up Dest MAC */ 2227 ether_addr_copy(&(pkt_dev->hh[0]), pkt_dev->dst_mac); 2228 2229 if (pkt_dev->flags & F_IPV6) { 2230 int i, set = 0, err = 1; 2231 struct inet6_dev *idev; 2232 2233 if (pkt_dev->min_pkt_size == 0) { 2234 pkt_dev->min_pkt_size = 14 + sizeof(struct ipv6hdr) 2235 + sizeof(struct udphdr) 2236 + sizeof(struct pktgen_hdr) 2237 + pkt_dev->pkt_overhead; 2238 } 2239 2240 for (i = 0; i < sizeof(struct in6_addr); i++) 2241 if (pkt_dev->cur_in6_saddr.s6_addr[i]) { 2242 set = 1; 2243 break; 2244 } 2245 2246 if (!set) { 2247 2248 /* 2249 * Use linklevel address if unconfigured. 2250 * 2251 * use ipv6_get_lladdr if/when it's get exported 2252 */ 2253 2254 rcu_read_lock(); 2255 idev = __in6_dev_get(pkt_dev->odev); 2256 if (idev) { 2257 struct inet6_ifaddr *ifp; 2258 2259 read_lock_bh(&idev->lock); 2260 list_for_each_entry(ifp, &idev->addr_list, if_list) { 2261 if ((ifp->scope & IFA_LINK) && 2262 !(ifp->flags & IFA_F_TENTATIVE)) { 2263 pkt_dev->cur_in6_saddr = ifp->addr; 2264 err = 0; 2265 break; 2266 } 2267 } 2268 read_unlock_bh(&idev->lock); 2269 } 2270 rcu_read_unlock(); 2271 if (err) 2272 pr_err("ERROR: IPv6 link address not available\n"); 2273 } 2274 } else { 2275 if (pkt_dev->min_pkt_size == 0) { 2276 pkt_dev->min_pkt_size = 14 + sizeof(struct iphdr) 2277 + sizeof(struct udphdr) 2278 + sizeof(struct pktgen_hdr) 2279 + pkt_dev->pkt_overhead; 2280 } 2281 2282 pkt_dev->saddr_min = 0; 2283 pkt_dev->saddr_max = 0; 2284 if (strlen(pkt_dev->src_min) == 0) { 2285 2286 struct in_device *in_dev; 2287 2288 rcu_read_lock(); 2289 in_dev = __in_dev_get_rcu(pkt_dev->odev); 2290 if (in_dev) { 2291 const struct in_ifaddr *ifa; 2292 2293 ifa = rcu_dereference(in_dev->ifa_list); 2294 if (ifa) { 2295 pkt_dev->saddr_min = ifa->ifa_address; 2296 pkt_dev->saddr_max = pkt_dev->saddr_min; 2297 } 2298 } 2299 rcu_read_unlock(); 2300 } else { 2301 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 2302 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 2303 } 2304 2305 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 2306 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 2307 } 2308 /* Initialize current values. */ 2309 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size; 2310 if (pkt_dev->min_pkt_size > pkt_dev->max_pkt_size) 2311 pkt_dev->max_pkt_size = pkt_dev->min_pkt_size; 2312 2313 pkt_dev->cur_dst_mac_offset = 0; 2314 pkt_dev->cur_src_mac_offset = 0; 2315 pkt_dev->cur_saddr = pkt_dev->saddr_min; 2316 pkt_dev->cur_daddr = pkt_dev->daddr_min; 2317 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2318 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2319 pkt_dev->nflows = 0; 2320 } 2321 2322 2323 static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) 2324 { 2325 ktime_t start_time, end_time; 2326 s64 remaining; 2327 struct hrtimer_sleeper t; 2328 2329 hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 2330 hrtimer_set_expires(&t.timer, spin_until); 2331 2332 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer)); 2333 if (remaining <= 0) 2334 goto out; 2335 2336 start_time = ktime_get(); 2337 if (remaining < 100000) { 2338 /* for small delays (<100us), just loop until limit is reached */ 2339 do { 2340 end_time = ktime_get(); 2341 } while (ktime_compare(end_time, spin_until) < 0); 2342 } else { 2343 do { 2344 set_current_state(TASK_INTERRUPTIBLE); 2345 hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS); 2346 2347 if (likely(hrtimer_sleeper_task_get(&t))) 2348 schedule(); 2349 2350 hrtimer_cancel(&t.timer); 2351 } while (hrtimer_sleeper_task_get(&t) && pkt_dev->running && !signal_pending(current)); 2352 __set_current_state(TASK_RUNNING); 2353 end_time = ktime_get(); 2354 } 2355 2356 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time)); 2357 out: 2358 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay); 2359 destroy_hrtimer_on_stack(&t.timer); 2360 } 2361 2362 static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev) 2363 { 2364 pkt_dev->pkt_overhead = 0; 2365 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32); 2366 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev); 2367 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev); 2368 } 2369 2370 static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow) 2371 { 2372 return !!(pkt_dev->flows[flow].flags & F_INIT); 2373 } 2374 2375 static inline int f_pick(struct pktgen_dev *pkt_dev, unsigned int cflows) 2376 { 2377 int flow = pkt_dev->curfl; 2378 2379 if (pkt_dev->flags & F_FLOW_SEQ) { 2380 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) { 2381 /* reset time */ 2382 pkt_dev->flows[flow].count = 0; 2383 pkt_dev->flows[flow].flags = 0; 2384 pkt_dev->curfl += 1; 2385 if (pkt_dev->curfl >= cflows) 2386 pkt_dev->curfl = 0; /*reset */ 2387 } 2388 } else { 2389 flow = get_random_u32_below(cflows); 2390 pkt_dev->curfl = flow; 2391 2392 if (pkt_dev->flows[flow].count > pkt_dev->lflow) { 2393 pkt_dev->flows[flow].count = 0; 2394 pkt_dev->flows[flow].flags = 0; 2395 } 2396 } 2397 2398 return pkt_dev->curfl; 2399 } 2400 2401 2402 /* If there was already an IPSEC SA, we keep it as is, else 2403 * we go look for it ... 2404 */ 2405 #define DUMMY_MARK 0 2406 static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow) 2407 { 2408 #ifdef CONFIG_XFRM 2409 struct xfrm_state *x = pkt_dev->flows[flow].x; 2410 struct pktgen_net *pn = net_generic(dev_net(pkt_dev->odev), pg_net_id); 2411 2412 if (!x) { 2413 2414 if (pkt_dev->spi) { 2415 /* We need as quick as possible to find the right SA 2416 * Searching with minimum criteria to achieve, this. 2417 */ 2418 x = xfrm_state_lookup_byspi(pn->net, htonl(pkt_dev->spi), AF_INET); 2419 } else { 2420 /* slow path: we don't already have xfrm_state */ 2421 x = xfrm_stateonly_find(pn->net, DUMMY_MARK, 0, 2422 (xfrm_address_t *)&pkt_dev->cur_daddr, 2423 (xfrm_address_t *)&pkt_dev->cur_saddr, 2424 AF_INET, 2425 pkt_dev->ipsmode, 2426 pkt_dev->ipsproto, 0); 2427 } 2428 if (x) { 2429 pkt_dev->flows[flow].x = x; 2430 set_pkt_overhead(pkt_dev); 2431 pkt_dev->pkt_overhead += x->props.header_len; 2432 } 2433 2434 } 2435 #endif 2436 } 2437 static void set_cur_queue_map(struct pktgen_dev *pkt_dev) 2438 { 2439 if (pkt_dev->flags & F_QUEUE_MAP_CPU) 2440 pkt_dev->cur_queue_map = smp_processor_id(); 2441 2442 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) { 2443 __u16 t; 2444 2445 if (pkt_dev->flags & F_QUEUE_MAP_RND) { 2446 t = get_random_u32_inclusive(pkt_dev->queue_map_min, 2447 pkt_dev->queue_map_max); 2448 } else { 2449 t = pkt_dev->cur_queue_map + 1; 2450 if (t > pkt_dev->queue_map_max) 2451 t = pkt_dev->queue_map_min; 2452 } 2453 pkt_dev->cur_queue_map = t; 2454 } 2455 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues; 2456 } 2457 2458 /* Increment/randomize headers according to flags and current values 2459 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst 2460 */ 2461 static void mod_cur_headers(struct pktgen_dev *pkt_dev) 2462 { 2463 unsigned int cflows; 2464 __u32 imn; 2465 __u32 imx; 2466 int flow = 0; 2467 2468 cflows = READ_ONCE(pkt_dev->cflows); 2469 if (cflows) 2470 flow = f_pick(pkt_dev, cflows); 2471 2472 /* Deal with source MAC */ 2473 if (pkt_dev->src_mac_count > 1) { 2474 __u32 mc; 2475 __u32 tmp; 2476 2477 if (pkt_dev->flags & F_MACSRC_RND) 2478 mc = get_random_u32_below(pkt_dev->src_mac_count); 2479 else { 2480 mc = pkt_dev->cur_src_mac_offset++; 2481 if (pkt_dev->cur_src_mac_offset >= 2482 pkt_dev->src_mac_count) 2483 pkt_dev->cur_src_mac_offset = 0; 2484 } 2485 2486 tmp = pkt_dev->src_mac[5] + (mc & 0xFF); 2487 pkt_dev->hh[11] = tmp; 2488 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2489 pkt_dev->hh[10] = tmp; 2490 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2491 pkt_dev->hh[9] = tmp; 2492 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2493 pkt_dev->hh[8] = tmp; 2494 tmp = (pkt_dev->src_mac[1] + (tmp >> 8)); 2495 pkt_dev->hh[7] = tmp; 2496 } 2497 2498 /* Deal with Destination MAC */ 2499 if (pkt_dev->dst_mac_count > 1) { 2500 __u32 mc; 2501 __u32 tmp; 2502 2503 if (pkt_dev->flags & F_MACDST_RND) 2504 mc = get_random_u32_below(pkt_dev->dst_mac_count); 2505 2506 else { 2507 mc = pkt_dev->cur_dst_mac_offset++; 2508 if (pkt_dev->cur_dst_mac_offset >= 2509 pkt_dev->dst_mac_count) { 2510 pkt_dev->cur_dst_mac_offset = 0; 2511 } 2512 } 2513 2514 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF); 2515 pkt_dev->hh[5] = tmp; 2516 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2517 pkt_dev->hh[4] = tmp; 2518 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2519 pkt_dev->hh[3] = tmp; 2520 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2521 pkt_dev->hh[2] = tmp; 2522 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8)); 2523 pkt_dev->hh[1] = tmp; 2524 } 2525 2526 if (pkt_dev->flags & F_MPLS_RND) { 2527 unsigned int i; 2528 2529 for (i = 0; i < pkt_dev->nr_labels; i++) 2530 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM) 2531 pkt_dev->labels[i] = MPLS_STACK_BOTTOM | 2532 ((__force __be32)get_random_u32() & 2533 htonl(0x000fffff)); 2534 } 2535 2536 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) { 2537 pkt_dev->vlan_id = get_random_u32_below(4096); 2538 } 2539 2540 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) { 2541 pkt_dev->svlan_id = get_random_u32_below(4096); 2542 } 2543 2544 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) { 2545 if (pkt_dev->flags & F_UDPSRC_RND) 2546 pkt_dev->cur_udp_src = get_random_u32_inclusive(pkt_dev->udp_src_min, 2547 pkt_dev->udp_src_max - 1); 2548 2549 else { 2550 pkt_dev->cur_udp_src++; 2551 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max) 2552 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2553 } 2554 } 2555 2556 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) { 2557 if (pkt_dev->flags & F_UDPDST_RND) { 2558 pkt_dev->cur_udp_dst = get_random_u32_inclusive(pkt_dev->udp_dst_min, 2559 pkt_dev->udp_dst_max - 1); 2560 } else { 2561 pkt_dev->cur_udp_dst++; 2562 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max) 2563 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2564 } 2565 } 2566 2567 if (!(pkt_dev->flags & F_IPV6)) { 2568 2569 imn = ntohl(pkt_dev->saddr_min); 2570 imx = ntohl(pkt_dev->saddr_max); 2571 if (imn < imx) { 2572 __u32 t; 2573 2574 if (pkt_dev->flags & F_IPSRC_RND) 2575 t = get_random_u32_inclusive(imn, imx - 1); 2576 else { 2577 t = ntohl(pkt_dev->cur_saddr); 2578 t++; 2579 if (t > imx) 2580 t = imn; 2581 2582 } 2583 pkt_dev->cur_saddr = htonl(t); 2584 } 2585 2586 if (cflows && f_seen(pkt_dev, flow)) { 2587 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr; 2588 } else { 2589 imn = ntohl(pkt_dev->daddr_min); 2590 imx = ntohl(pkt_dev->daddr_max); 2591 if (imn < imx) { 2592 __u32 t; 2593 __be32 s; 2594 2595 if (pkt_dev->flags & F_IPDST_RND) { 2596 2597 do { 2598 t = get_random_u32_inclusive(imn, imx - 1); 2599 s = htonl(t); 2600 } while (ipv4_is_loopback(s) || 2601 ipv4_is_multicast(s) || 2602 ipv4_is_lbcast(s) || 2603 ipv4_is_zeronet(s) || 2604 ipv4_is_local_multicast(s)); 2605 pkt_dev->cur_daddr = s; 2606 } else { 2607 t = ntohl(pkt_dev->cur_daddr); 2608 t++; 2609 if (t > imx) { 2610 t = imn; 2611 } 2612 pkt_dev->cur_daddr = htonl(t); 2613 } 2614 } 2615 if (cflows) { 2616 pkt_dev->flows[flow].flags |= F_INIT; 2617 pkt_dev->flows[flow].cur_daddr = 2618 pkt_dev->cur_daddr; 2619 if (pkt_dev->flags & F_IPSEC) 2620 get_ipsec_sa(pkt_dev, flow); 2621 pkt_dev->nflows++; 2622 } 2623 } 2624 } else { /* IPV6 * */ 2625 2626 if (!ipv6_addr_any(&pkt_dev->min_in6_daddr)) { 2627 int i; 2628 2629 /* Only random destinations yet */ 2630 2631 for (i = 0; i < 4; i++) { 2632 pkt_dev->cur_in6_daddr.s6_addr32[i] = 2633 (((__force __be32)get_random_u32() | 2634 pkt_dev->min_in6_daddr.s6_addr32[i]) & 2635 pkt_dev->max_in6_daddr.s6_addr32[i]); 2636 } 2637 } 2638 } 2639 2640 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) { 2641 __u32 t; 2642 2643 if (pkt_dev->flags & F_TXSIZE_RND) { 2644 t = get_random_u32_inclusive(pkt_dev->min_pkt_size, 2645 pkt_dev->max_pkt_size - 1); 2646 } else { 2647 t = pkt_dev->cur_pkt_size + 1; 2648 if (t > pkt_dev->max_pkt_size) 2649 t = pkt_dev->min_pkt_size; 2650 } 2651 pkt_dev->cur_pkt_size = t; 2652 } else if (pkt_dev->n_imix_entries > 0) { 2653 struct imix_pkt *entry; 2654 __u32 t = get_random_u32_below(IMIX_PRECISION); 2655 __u8 entry_index = pkt_dev->imix_distribution[t]; 2656 2657 entry = &pkt_dev->imix_entries[entry_index]; 2658 entry->count_so_far++; 2659 pkt_dev->cur_pkt_size = entry->size; 2660 } 2661 2662 set_cur_queue_map(pkt_dev); 2663 2664 pkt_dev->flows[flow].count++; 2665 } 2666 2667 static void fill_imix_distribution(struct pktgen_dev *pkt_dev) 2668 { 2669 int cumulative_probabilites[MAX_IMIX_ENTRIES]; 2670 int j = 0; 2671 __u64 cumulative_prob = 0; 2672 __u64 total_weight = 0; 2673 int i = 0; 2674 2675 for (i = 0; i < pkt_dev->n_imix_entries; i++) 2676 total_weight += pkt_dev->imix_entries[i].weight; 2677 2678 /* Fill cumulative_probabilites with sum of normalized probabilities */ 2679 for (i = 0; i < pkt_dev->n_imix_entries - 1; i++) { 2680 cumulative_prob += div64_u64(pkt_dev->imix_entries[i].weight * 2681 IMIX_PRECISION, 2682 total_weight); 2683 cumulative_probabilites[i] = cumulative_prob; 2684 } 2685 cumulative_probabilites[pkt_dev->n_imix_entries - 1] = 100; 2686 2687 for (i = 0; i < IMIX_PRECISION; i++) { 2688 if (i == cumulative_probabilites[j]) 2689 j++; 2690 pkt_dev->imix_distribution[i] = j; 2691 } 2692 } 2693 2694 #ifdef CONFIG_XFRM 2695 static u32 pktgen_dst_metrics[RTAX_MAX + 1] = { 2696 2697 [RTAX_HOPLIMIT] = 0x5, /* Set a static hoplimit */ 2698 }; 2699 2700 static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev) 2701 { 2702 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2703 int err = 0; 2704 struct net *net = dev_net(pkt_dev->odev); 2705 2706 if (!x) 2707 return 0; 2708 /* XXX: we dont support tunnel mode for now until 2709 * we resolve the dst issue 2710 */ 2711 if ((x->props.mode != XFRM_MODE_TRANSPORT) && (pkt_dev->spi == 0)) 2712 return 0; 2713 2714 /* But when user specify an valid SPI, transformation 2715 * supports both transport/tunnel mode + ESP/AH type. 2716 */ 2717 if ((x->props.mode == XFRM_MODE_TUNNEL) && (pkt_dev->spi != 0)) 2718 skb->_skb_refdst = (unsigned long)&pkt_dev->xdst.u.dst | SKB_DST_NOREF; 2719 2720 rcu_read_lock_bh(); 2721 err = pktgen_xfrm_outer_mode_output(x, skb); 2722 rcu_read_unlock_bh(); 2723 if (err) { 2724 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR); 2725 goto error; 2726 } 2727 err = x->type->output(x, skb); 2728 if (err) { 2729 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR); 2730 goto error; 2731 } 2732 spin_lock_bh(&x->lock); 2733 x->curlft.bytes += skb->len; 2734 x->curlft.packets++; 2735 spin_unlock_bh(&x->lock); 2736 error: 2737 return err; 2738 } 2739 2740 static void free_SAs(struct pktgen_dev *pkt_dev) 2741 { 2742 if (pkt_dev->cflows) { 2743 /* let go of the SAs if we have them */ 2744 int i; 2745 2746 for (i = 0; i < pkt_dev->cflows; i++) { 2747 struct xfrm_state *x = pkt_dev->flows[i].x; 2748 2749 if (x) { 2750 xfrm_state_put(x); 2751 pkt_dev->flows[i].x = NULL; 2752 } 2753 } 2754 } 2755 } 2756 2757 static int process_ipsec(struct pktgen_dev *pkt_dev, 2758 struct sk_buff *skb, __be16 protocol) 2759 { 2760 if (pkt_dev->flags & F_IPSEC) { 2761 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2762 int nhead = 0; 2763 2764 if (x) { 2765 struct ethhdr *eth; 2766 struct iphdr *iph; 2767 int ret; 2768 2769 nhead = x->props.header_len - skb_headroom(skb); 2770 if (nhead > 0) { 2771 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC); 2772 if (ret < 0) { 2773 pr_err("Error expanding ipsec packet %d\n", 2774 ret); 2775 goto err; 2776 } 2777 } 2778 2779 /* ipsec is not expecting ll header */ 2780 skb_pull(skb, ETH_HLEN); 2781 ret = pktgen_output_ipsec(skb, pkt_dev); 2782 if (ret) { 2783 pr_err("Error creating ipsec packet %d\n", ret); 2784 goto err; 2785 } 2786 /* restore ll */ 2787 eth = skb_push(skb, ETH_HLEN); 2788 memcpy(eth, pkt_dev->hh, 2 * ETH_ALEN); 2789 eth->h_proto = protocol; 2790 2791 /* Update IPv4 header len as well as checksum value */ 2792 iph = ip_hdr(skb); 2793 iph->tot_len = htons(skb->len - ETH_HLEN); 2794 ip_send_check(iph); 2795 } 2796 } 2797 return 1; 2798 err: 2799 kfree_skb(skb); 2800 return 0; 2801 } 2802 #endif 2803 2804 static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev) 2805 { 2806 unsigned int i; 2807 2808 for (i = 0; i < pkt_dev->nr_labels; i++) 2809 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM; 2810 2811 mpls--; 2812 *mpls |= MPLS_STACK_BOTTOM; 2813 } 2814 2815 static inline __be16 build_tci(unsigned int id, unsigned int cfi, 2816 unsigned int prio) 2817 { 2818 return htons(id | (cfi << 12) | (prio << 13)); 2819 } 2820 2821 static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb, 2822 int datalen) 2823 { 2824 struct timespec64 timestamp; 2825 struct pktgen_hdr *pgh; 2826 2827 pgh = skb_put(skb, sizeof(*pgh)); 2828 datalen -= sizeof(*pgh); 2829 2830 if (pkt_dev->nfrags <= 0) { 2831 skb_put_zero(skb, datalen); 2832 } else { 2833 int frags = pkt_dev->nfrags; 2834 int i, len; 2835 int frag_len; 2836 2837 2838 if (frags > MAX_SKB_FRAGS) 2839 frags = MAX_SKB_FRAGS; 2840 len = datalen - frags * PAGE_SIZE; 2841 if (len > 0) { 2842 skb_put_zero(skb, len); 2843 datalen = frags * PAGE_SIZE; 2844 } 2845 2846 i = 0; 2847 frag_len = min_t(int, datalen / frags, PAGE_SIZE); 2848 while (datalen > 0) { 2849 if (unlikely(!pkt_dev->page)) { 2850 int node = numa_node_id(); 2851 2852 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE)) 2853 node = pkt_dev->node; 2854 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); 2855 if (!pkt_dev->page) 2856 break; 2857 } 2858 get_page(pkt_dev->page); 2859 2860 /*last fragment, fill rest of data*/ 2861 if (i == (frags - 1)) 2862 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i], 2863 pkt_dev->page, 0, 2864 min(datalen, PAGE_SIZE)); 2865 else 2866 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i], 2867 pkt_dev->page, 0, frag_len); 2868 2869 datalen -= skb_frag_size(&skb_shinfo(skb)->frags[i]); 2870 skb->len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2871 skb->data_len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2872 i++; 2873 skb_shinfo(skb)->nr_frags = i; 2874 } 2875 } 2876 2877 /* Stamp the time, and sequence number, 2878 * convert them to network byte order 2879 */ 2880 pgh->pgh_magic = htonl(PKTGEN_MAGIC); 2881 pgh->seq_num = htonl(pkt_dev->seq_num); 2882 2883 if (pkt_dev->flags & F_NO_TIMESTAMP) { 2884 pgh->tv_sec = 0; 2885 pgh->tv_usec = 0; 2886 } else { 2887 /* 2888 * pgh->tv_sec wraps in y2106 when interpreted as unsigned 2889 * as done by wireshark, or y2038 when interpreted as signed. 2890 * This is probably harmless, but if anyone wants to improve 2891 * it, we could introduce a variant that puts 64-bit nanoseconds 2892 * into the respective header bytes. 2893 * This would also be slightly faster to read. 2894 */ 2895 ktime_get_real_ts64(×tamp); 2896 pgh->tv_sec = htonl(timestamp.tv_sec); 2897 pgh->tv_usec = htonl(timestamp.tv_nsec / NSEC_PER_USEC); 2898 } 2899 } 2900 2901 static struct sk_buff *pktgen_alloc_skb(struct net_device *dev, 2902 struct pktgen_dev *pkt_dev) 2903 { 2904 unsigned int extralen = LL_RESERVED_SPACE(dev); 2905 struct sk_buff *skb = NULL; 2906 unsigned int size; 2907 2908 size = pkt_dev->cur_pkt_size + 64 + extralen + pkt_dev->pkt_overhead; 2909 if (pkt_dev->flags & F_NODE) { 2910 int node = pkt_dev->node >= 0 ? pkt_dev->node : numa_node_id(); 2911 2912 skb = __alloc_skb(NET_SKB_PAD + size, GFP_NOWAIT, 0, node); 2913 if (likely(skb)) { 2914 skb_reserve(skb, NET_SKB_PAD); 2915 skb->dev = dev; 2916 } 2917 } else { 2918 skb = __netdev_alloc_skb(dev, size, GFP_NOWAIT); 2919 } 2920 2921 /* the caller pre-fetches from skb->data and reserves for the mac hdr */ 2922 if (likely(skb)) 2923 skb_reserve(skb, extralen - 16); 2924 2925 return skb; 2926 } 2927 2928 static struct sk_buff *fill_packet_ipv4(struct net_device *odev, 2929 struct pktgen_dev *pkt_dev) 2930 { 2931 struct sk_buff *skb = NULL; 2932 __u8 *eth; 2933 struct udphdr *udph; 2934 int datalen, iplen; 2935 struct iphdr *iph; 2936 __be16 protocol = htons(ETH_P_IP); 2937 __be32 *mpls; 2938 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2939 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 2940 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 2941 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 2942 u16 queue_map; 2943 2944 if (pkt_dev->nr_labels) 2945 protocol = htons(ETH_P_MPLS_UC); 2946 2947 if (pkt_dev->vlan_id != 0xffff) 2948 protocol = htons(ETH_P_8021Q); 2949 2950 /* Update any of the values, used when we're incrementing various 2951 * fields. 2952 */ 2953 mod_cur_headers(pkt_dev); 2954 queue_map = pkt_dev->cur_queue_map; 2955 2956 skb = pktgen_alloc_skb(odev, pkt_dev); 2957 if (!skb) { 2958 sprintf(pkt_dev->result, "No memory"); 2959 return NULL; 2960 } 2961 2962 prefetchw(skb->data); 2963 skb_reserve(skb, 16); 2964 2965 /* Reserve for ethernet and IP header */ 2966 eth = skb_push(skb, 14); 2967 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32)); 2968 if (pkt_dev->nr_labels) 2969 mpls_push(mpls, pkt_dev); 2970 2971 if (pkt_dev->vlan_id != 0xffff) { 2972 if (pkt_dev->svlan_id != 0xffff) { 2973 svlan_tci = skb_put(skb, sizeof(__be16)); 2974 *svlan_tci = build_tci(pkt_dev->svlan_id, 2975 pkt_dev->svlan_cfi, 2976 pkt_dev->svlan_p); 2977 svlan_encapsulated_proto = skb_put(skb, 2978 sizeof(__be16)); 2979 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 2980 } 2981 vlan_tci = skb_put(skb, sizeof(__be16)); 2982 *vlan_tci = build_tci(pkt_dev->vlan_id, 2983 pkt_dev->vlan_cfi, 2984 pkt_dev->vlan_p); 2985 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16)); 2986 *vlan_encapsulated_proto = htons(ETH_P_IP); 2987 } 2988 2989 skb_reset_mac_header(skb); 2990 skb_set_network_header(skb, skb->len); 2991 iph = skb_put(skb, sizeof(struct iphdr)); 2992 2993 skb_set_transport_header(skb, skb->len); 2994 udph = skb_put(skb, sizeof(struct udphdr)); 2995 skb_set_queue_mapping(skb, queue_map); 2996 skb->priority = pkt_dev->skb_priority; 2997 2998 memcpy(eth, pkt_dev->hh, 12); 2999 *(__be16 *)ð[12] = protocol; 3000 3001 /* Eth + IPh + UDPh + mpls */ 3002 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 - 3003 pkt_dev->pkt_overhead; 3004 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) 3005 datalen = sizeof(struct pktgen_hdr); 3006 3007 udph->source = htons(pkt_dev->cur_udp_src); 3008 udph->dest = htons(pkt_dev->cur_udp_dst); 3009 udp_set_len_short(udph, datalen + 8); /* DATA + udphdr */ 3010 udph->check = 0; 3011 3012 iph->ihl = 5; 3013 iph->version = 4; 3014 iph->ttl = 32; 3015 iph->tos = pkt_dev->tos; 3016 iph->protocol = IPPROTO_UDP; /* UDP */ 3017 iph->saddr = pkt_dev->cur_saddr; 3018 iph->daddr = pkt_dev->cur_daddr; 3019 iph->id = htons(pkt_dev->ip_id); 3020 pkt_dev->ip_id++; 3021 iph->frag_off = 0; 3022 iplen = 20 + 8 + datalen; 3023 iph->tot_len = htons(iplen); 3024 ip_send_check(iph); 3025 skb->protocol = protocol; 3026 skb->dev = odev; 3027 skb->pkt_type = PACKET_HOST; 3028 3029 pktgen_finalize_skb(pkt_dev, skb, datalen); 3030 3031 if (!(pkt_dev->flags & F_UDPCSUM)) { 3032 skb->ip_summed = CHECKSUM_NONE; 3033 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)) { 3034 skb->ip_summed = CHECKSUM_PARTIAL; 3035 skb->csum = 0; 3036 udp4_hwcsum(skb, iph->saddr, iph->daddr); 3037 } else { 3038 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), datalen + 8, 0); 3039 3040 /* add protocol-dependent pseudo-header */ 3041 udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 3042 datalen + 8, IPPROTO_UDP, csum); 3043 3044 if (udph->check == 0) 3045 udph->check = CSUM_MANGLED_0; 3046 } 3047 3048 #ifdef CONFIG_XFRM 3049 if (!process_ipsec(pkt_dev, skb, protocol)) 3050 return NULL; 3051 #endif 3052 3053 return skb; 3054 } 3055 3056 static struct sk_buff *fill_packet_ipv6(struct net_device *odev, 3057 struct pktgen_dev *pkt_dev) 3058 { 3059 struct sk_buff *skb = NULL; 3060 __u8 *eth; 3061 struct udphdr *udph; 3062 int datalen, udplen; 3063 struct ipv6hdr *iph; 3064 __be16 protocol = htons(ETH_P_IPV6); 3065 __be32 *mpls; 3066 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 3067 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 3068 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 3069 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 3070 u16 queue_map; 3071 3072 if (pkt_dev->nr_labels) 3073 protocol = htons(ETH_P_MPLS_UC); 3074 3075 if (pkt_dev->vlan_id != 0xffff) 3076 protocol = htons(ETH_P_8021Q); 3077 3078 /* Update any of the values, used when we're incrementing various 3079 * fields. 3080 */ 3081 mod_cur_headers(pkt_dev); 3082 queue_map = pkt_dev->cur_queue_map; 3083 3084 skb = pktgen_alloc_skb(odev, pkt_dev); 3085 if (!skb) { 3086 sprintf(pkt_dev->result, "No memory"); 3087 return NULL; 3088 } 3089 3090 prefetchw(skb->data); 3091 skb_reserve(skb, 16); 3092 3093 /* Reserve for ethernet and IP header */ 3094 eth = skb_push(skb, 14); 3095 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32)); 3096 if (pkt_dev->nr_labels) 3097 mpls_push(mpls, pkt_dev); 3098 3099 if (pkt_dev->vlan_id != 0xffff) { 3100 if (pkt_dev->svlan_id != 0xffff) { 3101 svlan_tci = skb_put(skb, sizeof(__be16)); 3102 *svlan_tci = build_tci(pkt_dev->svlan_id, 3103 pkt_dev->svlan_cfi, 3104 pkt_dev->svlan_p); 3105 svlan_encapsulated_proto = skb_put(skb, 3106 sizeof(__be16)); 3107 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 3108 } 3109 vlan_tci = skb_put(skb, sizeof(__be16)); 3110 *vlan_tci = build_tci(pkt_dev->vlan_id, 3111 pkt_dev->vlan_cfi, 3112 pkt_dev->vlan_p); 3113 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16)); 3114 *vlan_encapsulated_proto = htons(ETH_P_IPV6); 3115 } 3116 3117 skb_reset_mac_header(skb); 3118 skb_set_network_header(skb, skb->len); 3119 iph = skb_put(skb, sizeof(struct ipv6hdr)); 3120 3121 skb_set_transport_header(skb, skb->len); 3122 udph = skb_put(skb, sizeof(struct udphdr)); 3123 skb_set_queue_mapping(skb, queue_map); 3124 skb->priority = pkt_dev->skb_priority; 3125 3126 memcpy(eth, pkt_dev->hh, 12); 3127 *(__be16 *) ð[12] = protocol; 3128 3129 /* Eth + IPh + UDPh + mpls */ 3130 datalen = pkt_dev->cur_pkt_size - 14 - 3131 sizeof(struct ipv6hdr) - sizeof(struct udphdr) - 3132 pkt_dev->pkt_overhead; 3133 3134 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) { 3135 datalen = sizeof(struct pktgen_hdr); 3136 net_info_ratelimited("increased datalen to %d\n", datalen); 3137 } 3138 3139 udplen = datalen + sizeof(struct udphdr); 3140 udph->source = htons(pkt_dev->cur_udp_src); 3141 udph->dest = htons(pkt_dev->cur_udp_dst); 3142 udp_set_len_short(udph, udplen); 3143 udph->check = 0; 3144 3145 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */ 3146 3147 if (pkt_dev->traffic_class) { 3148 /* Version + traffic class + flow (0) */ 3149 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20)); 3150 } 3151 3152 iph->hop_limit = 32; 3153 3154 iph->payload_len = htons(udplen); 3155 iph->nexthdr = IPPROTO_UDP; 3156 3157 iph->daddr = pkt_dev->cur_in6_daddr; 3158 iph->saddr = pkt_dev->cur_in6_saddr; 3159 3160 skb->protocol = protocol; 3161 skb->dev = odev; 3162 skb->pkt_type = PACKET_HOST; 3163 3164 pktgen_finalize_skb(pkt_dev, skb, datalen); 3165 3166 if (!(pkt_dev->flags & F_UDPCSUM)) { 3167 skb->ip_summed = CHECKSUM_NONE; 3168 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM)) { 3169 skb->ip_summed = CHECKSUM_PARTIAL; 3170 skb->csum_start = skb_transport_header(skb) - skb->head; 3171 skb->csum_offset = offsetof(struct udphdr, check); 3172 udph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, 0); 3173 } else { 3174 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), udplen, 0); 3175 3176 /* add protocol-dependent pseudo-header */ 3177 udph->check = csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, csum); 3178 3179 if (udph->check == 0) 3180 udph->check = CSUM_MANGLED_0; 3181 } 3182 3183 return skb; 3184 } 3185 3186 static struct sk_buff *fill_packet(struct net_device *odev, 3187 struct pktgen_dev *pkt_dev) 3188 { 3189 if (pkt_dev->flags & F_IPV6) 3190 return fill_packet_ipv6(odev, pkt_dev); 3191 else 3192 return fill_packet_ipv4(odev, pkt_dev); 3193 } 3194 3195 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev) 3196 { 3197 pkt_dev->seq_num = 1; 3198 pkt_dev->idle_acc = 0; 3199 pkt_dev->sofar = 0; 3200 pkt_dev->tx_bytes = 0; 3201 pkt_dev->errors = 0; 3202 } 3203 3204 /* Set up structure for sending pkts, clear counters */ 3205 3206 static void pktgen_run(struct pktgen_thread *t) 3207 { 3208 struct pktgen_dev *pkt_dev; 3209 int started = 0; 3210 3211 func_enter(); 3212 3213 rcu_read_lock(); 3214 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3215 3216 /* 3217 * setup odev and create initial packet. 3218 */ 3219 pktgen_setup_inject(pkt_dev); 3220 3221 if (pkt_dev->odev) { 3222 pktgen_clear_counters(pkt_dev); 3223 pkt_dev->skb = NULL; 3224 pkt_dev->started_at = pkt_dev->next_tx = ktime_get(); 3225 3226 set_pkt_overhead(pkt_dev); 3227 3228 strscpy(pkt_dev->result, "Starting"); 3229 pkt_dev->running = 1; /* Cranke yeself! */ 3230 started++; 3231 } else 3232 strscpy(pkt_dev->result, "Error starting"); 3233 } 3234 rcu_read_unlock(); 3235 if (started) 3236 t->control &= ~(T_STOP); 3237 } 3238 3239 static void pktgen_handle_all_threads(struct pktgen_net *pn, u32 flags) 3240 { 3241 struct pktgen_thread *t; 3242 3243 mutex_lock(&pktgen_thread_lock); 3244 3245 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3246 t->control |= (flags); 3247 3248 mutex_unlock(&pktgen_thread_lock); 3249 } 3250 3251 static void pktgen_stop_all_threads(struct pktgen_net *pn) 3252 { 3253 func_enter(); 3254 3255 pktgen_handle_all_threads(pn, T_STOP); 3256 } 3257 3258 static int thread_is_running(const struct pktgen_thread *t) 3259 { 3260 const struct pktgen_dev *pkt_dev; 3261 3262 rcu_read_lock(); 3263 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 3264 if (pkt_dev->running) { 3265 rcu_read_unlock(); 3266 return 1; 3267 } 3268 rcu_read_unlock(); 3269 return 0; 3270 } 3271 3272 static int pktgen_wait_thread_run(struct pktgen_thread *t) 3273 { 3274 while (thread_is_running(t)) { 3275 3276 /* note: 't' will still be around even after the unlock/lock 3277 * cycle because pktgen_thread threads are only cleared at 3278 * net exit 3279 */ 3280 mutex_unlock(&pktgen_thread_lock); 3281 msleep_interruptible(100); 3282 mutex_lock(&pktgen_thread_lock); 3283 3284 if (signal_pending(current)) 3285 goto signal; 3286 } 3287 return 1; 3288 signal: 3289 return 0; 3290 } 3291 3292 static int pktgen_wait_all_threads_run(struct pktgen_net *pn) 3293 { 3294 struct pktgen_thread *t; 3295 int sig = 1; 3296 3297 /* prevent from racing with rmmod */ 3298 if (!try_module_get(THIS_MODULE)) 3299 return sig; 3300 3301 mutex_lock(&pktgen_thread_lock); 3302 3303 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 3304 sig = pktgen_wait_thread_run(t); 3305 if (sig == 0) 3306 break; 3307 } 3308 3309 if (sig == 0) 3310 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3311 t->control |= (T_STOP); 3312 3313 mutex_unlock(&pktgen_thread_lock); 3314 module_put(THIS_MODULE); 3315 return sig; 3316 } 3317 3318 static void pktgen_run_all_threads(struct pktgen_net *pn) 3319 { 3320 func_enter(); 3321 3322 pktgen_handle_all_threads(pn, T_RUN); 3323 3324 /* Propagate thread->control */ 3325 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3326 3327 pktgen_wait_all_threads_run(pn); 3328 } 3329 3330 static void pktgen_reset_all_threads(struct pktgen_net *pn) 3331 { 3332 func_enter(); 3333 3334 pktgen_handle_all_threads(pn, T_REMDEVALL); 3335 3336 /* Propagate thread->control */ 3337 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3338 3339 pktgen_wait_all_threads_run(pn); 3340 } 3341 3342 static void show_results(struct pktgen_dev *pkt_dev, int nr_frags) 3343 { 3344 __u64 bps, mbps, pps; 3345 char *p = pkt_dev->result; 3346 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at, 3347 pkt_dev->started_at); 3348 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc); 3349 3350 p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n", 3351 (unsigned long long)ktime_to_us(elapsed), 3352 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)), 3353 (unsigned long long)ktime_to_us(idle), 3354 (unsigned long long)pkt_dev->sofar, 3355 pkt_dev->cur_pkt_size, nr_frags); 3356 3357 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC, 3358 ktime_to_ns(elapsed)); 3359 3360 if (pkt_dev->n_imix_entries > 0) { 3361 int i; 3362 struct imix_pkt *entry; 3363 3364 bps = 0; 3365 for (i = 0; i < pkt_dev->n_imix_entries; i++) { 3366 entry = &pkt_dev->imix_entries[i]; 3367 bps += entry->size * entry->count_so_far; 3368 } 3369 bps = div64_u64(bps * 8 * NSEC_PER_SEC, ktime_to_ns(elapsed)); 3370 } else { 3371 bps = pps * 8 * pkt_dev->cur_pkt_size; 3372 } 3373 3374 mbps = bps; 3375 do_div(mbps, 1000000); 3376 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu", 3377 (unsigned long long)pps, 3378 (unsigned long long)mbps, 3379 (unsigned long long)bps, 3380 (unsigned long long)pkt_dev->errors); 3381 } 3382 3383 /* Set stopped-at timer, remove from running list, do counters & statistics */ 3384 static int pktgen_stop_device(struct pktgen_dev *pkt_dev) 3385 { 3386 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1; 3387 3388 if (!pkt_dev->running) { 3389 pr_warn("interface: %s is already stopped\n", 3390 pkt_dev->odevname); 3391 return -EINVAL; 3392 } 3393 3394 pkt_dev->running = 0; 3395 kfree_skb(pkt_dev->skb); 3396 pkt_dev->skb = NULL; 3397 pkt_dev->stopped_at = ktime_get(); 3398 3399 show_results(pkt_dev, nr_frags); 3400 3401 return 0; 3402 } 3403 3404 static struct pktgen_dev *next_to_run(struct pktgen_thread *t) 3405 { 3406 struct pktgen_dev *pkt_dev, *best = NULL; 3407 3408 rcu_read_lock(); 3409 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3410 if (!pkt_dev->running) 3411 continue; 3412 if (best == NULL) 3413 best = pkt_dev; 3414 else if (ktime_compare(pkt_dev->next_tx, best->next_tx) < 0) 3415 best = pkt_dev; 3416 } 3417 rcu_read_unlock(); 3418 3419 return best; 3420 } 3421 3422 static void pktgen_stop(struct pktgen_thread *t) 3423 { 3424 struct pktgen_dev *pkt_dev; 3425 3426 func_enter(); 3427 3428 rcu_read_lock(); 3429 3430 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3431 pktgen_stop_device(pkt_dev); 3432 } 3433 3434 rcu_read_unlock(); 3435 } 3436 3437 /* 3438 * one of our devices needs to be removed - find it 3439 * and remove it 3440 */ 3441 static void pktgen_rem_one_if(struct pktgen_thread *t) 3442 { 3443 struct list_head *q, *n; 3444 struct pktgen_dev *cur; 3445 3446 func_enter(); 3447 3448 list_for_each_safe(q, n, &t->if_list) { 3449 cur = list_entry(q, struct pktgen_dev, list); 3450 3451 if (!cur->removal_mark) 3452 continue; 3453 3454 kfree_skb(cur->skb); 3455 cur->skb = NULL; 3456 3457 pktgen_remove_device(t, cur); 3458 3459 break; 3460 } 3461 } 3462 3463 static void pktgen_rem_all_ifs(struct pktgen_thread *t) 3464 { 3465 struct list_head *q, *n; 3466 struct pktgen_dev *cur; 3467 3468 func_enter(); 3469 3470 /* Remove all devices, free mem */ 3471 3472 list_for_each_safe(q, n, &t->if_list) { 3473 cur = list_entry(q, struct pktgen_dev, list); 3474 3475 kfree_skb(cur->skb); 3476 cur->skb = NULL; 3477 3478 pktgen_remove_device(t, cur); 3479 } 3480 } 3481 3482 static void pktgen_rem_thread(struct pktgen_thread *t) 3483 { 3484 /* Remove from the thread list */ 3485 remove_proc_entry(t->tsk->comm, t->net->proc_dir); 3486 } 3487 3488 static void pktgen_resched(struct pktgen_dev *pkt_dev) 3489 { 3490 ktime_t idle_start = ktime_get(); 3491 3492 schedule(); 3493 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start)); 3494 } 3495 3496 static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev) 3497 { 3498 ktime_t idle_start = ktime_get(); 3499 3500 while (refcount_read(&(pkt_dev->skb->users)) != 1) { 3501 if (signal_pending(current)) 3502 break; 3503 3504 if (need_resched()) 3505 pktgen_resched(pkt_dev); 3506 else 3507 cpu_relax(); 3508 } 3509 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start)); 3510 } 3511 3512 static void pktgen_xmit(struct pktgen_dev *pkt_dev) 3513 { 3514 bool skb_shared = !!(READ_ONCE(pkt_dev->flags) & F_SHARED); 3515 struct net_device *odev = pkt_dev->odev; 3516 struct netdev_queue *txq; 3517 unsigned int burst = 1; 3518 struct sk_buff *skb; 3519 int clone_skb = 0; 3520 int ret; 3521 3522 /* If 'skb_shared' is false, the read of possible 3523 * new values (if any) for 'burst' and 'clone_skb' will be skipped to 3524 * prevent some concurrent changes from slipping in. And the stabilized 3525 * config will be read in during the next run of pktgen_xmit. 3526 */ 3527 if (skb_shared) { 3528 burst = READ_ONCE(pkt_dev->burst); 3529 clone_skb = READ_ONCE(pkt_dev->clone_skb); 3530 } 3531 3532 /* If device is offline, then don't send */ 3533 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) { 3534 pktgen_stop_device(pkt_dev); 3535 return; 3536 } 3537 3538 /* This is max DELAY, this has special meaning of 3539 * "never transmit" 3540 */ 3541 if (unlikely(pkt_dev->delay == ULLONG_MAX)) { 3542 pkt_dev->next_tx = ktime_add_ns(ktime_get(), ULONG_MAX); 3543 return; 3544 } 3545 3546 /* If no skb or clone count exhausted then get new one */ 3547 if (!pkt_dev->skb || (pkt_dev->last_ok && 3548 ++pkt_dev->clone_count >= clone_skb)) { 3549 /* build a new pkt */ 3550 kfree_skb(pkt_dev->skb); 3551 3552 pkt_dev->skb = fill_packet(odev, pkt_dev); 3553 if (pkt_dev->skb == NULL) { 3554 pr_err("ERROR: couldn't allocate skb in fill_packet\n"); 3555 schedule(); 3556 pkt_dev->clone_count--; /* back out increment, OOM */ 3557 return; 3558 } 3559 pkt_dev->last_pkt_size = pkt_dev->skb->len; 3560 pkt_dev->clone_count = 0; /* reset counter */ 3561 } 3562 3563 if (pkt_dev->delay && pkt_dev->last_ok) 3564 spin(pkt_dev, pkt_dev->next_tx); 3565 3566 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) { 3567 skb = pkt_dev->skb; 3568 skb->protocol = eth_type_trans(skb, skb->dev); 3569 if (skb_shared) 3570 refcount_add(burst, &skb->users); 3571 local_bh_disable(); 3572 do { 3573 ret = netif_receive_skb(skb); 3574 if (ret == NET_RX_DROP) 3575 pkt_dev->errors++; 3576 pkt_dev->sofar++; 3577 pkt_dev->seq_num++; 3578 if (unlikely(!skb_shared)) { 3579 pkt_dev->skb = NULL; 3580 break; 3581 } 3582 if (refcount_read(&skb->users) != burst) { 3583 /* skb was queued by rps/rfs or taps, 3584 * so cannot reuse this skb 3585 */ 3586 WARN_ON(refcount_sub_and_test(burst - 1, &skb->users)); 3587 /* get out of the loop and wait 3588 * until skb is consumed 3589 */ 3590 break; 3591 } 3592 /* skb was 'freed' by stack, so clean few 3593 * bits and reuse it 3594 */ 3595 skb_reset_redirect(skb); 3596 } while (--burst > 0); 3597 goto out; /* Skips xmit_mode M_START_XMIT */ 3598 } else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) { 3599 local_bh_disable(); 3600 if (skb_shared) 3601 refcount_inc(&pkt_dev->skb->users); 3602 3603 ret = dev_queue_xmit(pkt_dev->skb); 3604 3605 if (!skb_shared && dev_xmit_complete(ret)) 3606 pkt_dev->skb = NULL; 3607 3608 switch (ret) { 3609 case NET_XMIT_SUCCESS: 3610 pkt_dev->sofar++; 3611 pkt_dev->seq_num++; 3612 pkt_dev->tx_bytes += pkt_dev->last_pkt_size; 3613 break; 3614 case NET_XMIT_DROP: 3615 case NET_XMIT_CN: 3616 /* These are all valid return codes for a qdisc but 3617 * indicate packets are being dropped or will likely 3618 * be dropped soon. 3619 */ 3620 case NETDEV_TX_BUSY: 3621 /* qdisc may call dev_hard_start_xmit directly in cases 3622 * where no queues exist e.g. loopback device, virtual 3623 * devices, etc. In this case we need to handle 3624 * NETDEV_TX_ codes. 3625 */ 3626 default: 3627 pkt_dev->errors++; 3628 net_info_ratelimited("%s xmit error: %d\n", 3629 pkt_dev->odevname, ret); 3630 break; 3631 } 3632 goto out; 3633 } 3634 3635 txq = skb_get_tx_queue(odev, pkt_dev->skb); 3636 3637 local_bh_disable(); 3638 3639 HARD_TX_LOCK(odev, txq, smp_processor_id()); 3640 3641 if (unlikely(netif_xmit_frozen_or_drv_stopped(txq))) { 3642 pkt_dev->last_ok = 0; 3643 goto unlock; 3644 } 3645 if (skb_shared) 3646 refcount_add(burst, &pkt_dev->skb->users); 3647 3648 xmit_more: 3649 ret = netdev_start_xmit(pkt_dev->skb, odev, txq, --burst > 0); 3650 3651 if (!skb_shared && dev_xmit_complete(ret)) 3652 pkt_dev->skb = NULL; 3653 3654 switch (ret) { 3655 case NETDEV_TX_OK: 3656 pkt_dev->last_ok = 1; 3657 pkt_dev->sofar++; 3658 pkt_dev->seq_num++; 3659 pkt_dev->tx_bytes += pkt_dev->last_pkt_size; 3660 if (burst > 0 && !netif_xmit_frozen_or_drv_stopped(txq)) 3661 goto xmit_more; 3662 break; 3663 case NET_XMIT_DROP: 3664 case NET_XMIT_CN: 3665 /* skb has been consumed */ 3666 pkt_dev->errors++; 3667 break; 3668 default: /* Drivers are not supposed to return other values! */ 3669 net_info_ratelimited("%s xmit error: %d\n", 3670 pkt_dev->odevname, ret); 3671 pkt_dev->errors++; 3672 fallthrough; 3673 case NETDEV_TX_BUSY: 3674 /* Retry it next time */ 3675 if (skb_shared) 3676 refcount_dec(&pkt_dev->skb->users); 3677 pkt_dev->last_ok = 0; 3678 } 3679 if (unlikely(burst)) 3680 WARN_ON(refcount_sub_and_test(burst, &pkt_dev->skb->users)); 3681 unlock: 3682 HARD_TX_UNLOCK(odev, txq); 3683 3684 out: 3685 local_bh_enable(); 3686 3687 /* If pkt_dev->count is zero, then run forever */ 3688 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) { 3689 if (pkt_dev->skb) 3690 pktgen_wait_for_skb(pkt_dev); 3691 3692 /* Done with this */ 3693 pktgen_stop_device(pkt_dev); 3694 } 3695 } 3696 3697 /* 3698 * Main loop of the thread goes here 3699 */ 3700 3701 static int pktgen_thread_worker(void *arg) 3702 { 3703 struct pktgen_thread *t = arg; 3704 struct pktgen_dev *pkt_dev = NULL; 3705 int cpu = t->cpu; 3706 3707 WARN_ON_ONCE(smp_processor_id() != cpu); 3708 3709 init_waitqueue_head(&t->queue); 3710 complete(&t->start_done); 3711 3712 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current)); 3713 3714 set_freezable(); 3715 3716 while (!kthread_should_stop()) { 3717 pkt_dev = next_to_run(t); 3718 3719 if (unlikely(!pkt_dev && t->control == 0)) { 3720 if (t->net->pktgen_exiting) 3721 break; 3722 wait_event_freezable_timeout(t->queue, 3723 t->control != 0, HZ / 10); 3724 continue; 3725 } 3726 3727 if (likely(pkt_dev)) { 3728 pktgen_xmit(pkt_dev); 3729 3730 if (need_resched()) 3731 pktgen_resched(pkt_dev); 3732 else 3733 cpu_relax(); 3734 } 3735 3736 if (t->control & T_STOP) { 3737 pktgen_stop(t); 3738 t->control &= ~(T_STOP); 3739 } 3740 3741 if (t->control & T_RUN) { 3742 pktgen_run(t); 3743 t->control &= ~(T_RUN); 3744 } 3745 3746 if (t->control & T_REMDEVALL) { 3747 pktgen_rem_all_ifs(t); 3748 t->control &= ~(T_REMDEVALL); 3749 } 3750 3751 if (t->control & T_REMDEV) { 3752 pktgen_rem_one_if(t); 3753 t->control &= ~(T_REMDEV); 3754 } 3755 3756 try_to_freeze(); 3757 } 3758 3759 pr_debug("%s stopping all device\n", t->tsk->comm); 3760 pktgen_stop(t); 3761 3762 pr_debug("%s removing all device\n", t->tsk->comm); 3763 pktgen_rem_all_ifs(t); 3764 3765 pr_debug("%s removing thread\n", t->tsk->comm); 3766 pktgen_rem_thread(t); 3767 3768 return 0; 3769 } 3770 3771 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 3772 const char *ifname, bool exact) 3773 { 3774 struct pktgen_dev *p, *pkt_dev = NULL; 3775 size_t len = strlen(ifname); 3776 3777 rcu_read_lock(); 3778 list_for_each_entry_rcu(p, &t->if_list, list) 3779 if (strncmp(p->odevname, ifname, len) == 0) { 3780 if (p->odevname[len]) { 3781 if (exact || p->odevname[len] != '@') 3782 continue; 3783 } 3784 pkt_dev = p; 3785 break; 3786 } 3787 3788 rcu_read_unlock(); 3789 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev); 3790 return pkt_dev; 3791 } 3792 3793 /* 3794 * Adds a dev at front of if_list. 3795 */ 3796 3797 static int add_dev_to_thread(struct pktgen_thread *t, 3798 struct pktgen_dev *pkt_dev) 3799 { 3800 int rv = 0; 3801 3802 /* This function cannot be called concurrently, as its called 3803 * under pktgen_thread_lock mutex, but it can run from 3804 * userspace on another CPU than the kthread. The if_lock() 3805 * is used here to sync with concurrent instances of 3806 * _rem_dev_from_if_list() invoked via kthread, which is also 3807 * updating the if_list 3808 */ 3809 if_lock(t); 3810 3811 if (pkt_dev->pg_thread) { 3812 pr_err("ERROR: already assigned to a thread\n"); 3813 rv = -EBUSY; 3814 goto out; 3815 } 3816 3817 pkt_dev->running = 0; 3818 pkt_dev->pg_thread = t; 3819 list_add_rcu(&pkt_dev->list, &t->if_list); 3820 3821 out: 3822 if_unlock(t); 3823 return rv; 3824 } 3825 3826 /* Called under thread lock */ 3827 3828 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) 3829 { 3830 struct pktgen_dev *pkt_dev; 3831 int err; 3832 int node = cpu_to_node(t->cpu); 3833 3834 /* We don't allow a device to be on several threads */ 3835 3836 if (__pktgen_NN_threads(t->net, ifname, FIND)) { 3837 pr_err("ERROR: interface already used\n"); 3838 return -EBUSY; 3839 } 3840 3841 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node); 3842 if (!pkt_dev) 3843 return -ENOMEM; 3844 3845 strscpy(pkt_dev->odevname, ifname); 3846 pkt_dev->flows = vzalloc_node(array_size(MAX_CFLOWS, 3847 sizeof(struct flow_state)), 3848 node); 3849 if (pkt_dev->flows == NULL) { 3850 kfree(pkt_dev); 3851 return -ENOMEM; 3852 } 3853 3854 pkt_dev->removal_mark = 0; 3855 pkt_dev->nfrags = 0; 3856 pkt_dev->delay = pg_delay_d; 3857 pkt_dev->count = pg_count_d; 3858 pkt_dev->sofar = 0; 3859 pkt_dev->udp_src_min = 9; /* sink port */ 3860 pkt_dev->udp_src_max = 9; 3861 pkt_dev->udp_dst_min = 9; 3862 pkt_dev->udp_dst_max = 9; 3863 pkt_dev->vlan_p = 0; 3864 pkt_dev->vlan_cfi = 0; 3865 pkt_dev->vlan_id = 0xffff; 3866 pkt_dev->svlan_p = 0; 3867 pkt_dev->svlan_cfi = 0; 3868 pkt_dev->svlan_id = 0xffff; 3869 pkt_dev->burst = 1; 3870 pkt_dev->node = NUMA_NO_NODE; 3871 pkt_dev->flags = F_SHARED; /* SKB shared by default */ 3872 3873 err = pktgen_setup_dev(t->net, pkt_dev, ifname); 3874 if (err) 3875 goto out1; 3876 if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING) 3877 pkt_dev->clone_skb = pg_clone_skb_d; 3878 3879 pkt_dev->entry = proc_create_data(ifname, 0600, t->net->proc_dir, 3880 &pktgen_if_proc_ops, pkt_dev); 3881 if (!pkt_dev->entry) { 3882 pr_err("cannot create %s/%s procfs entry\n", 3883 PG_PROC_DIR, ifname); 3884 err = -EINVAL; 3885 goto out2; 3886 } 3887 #ifdef CONFIG_XFRM 3888 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT; 3889 pkt_dev->ipsproto = IPPROTO_ESP; 3890 3891 /* xfrm tunnel mode needs additional dst to extract outer 3892 * ip header protocol/ttl/id field, here create a phony one. 3893 * instead of looking for a valid rt, which definitely hurting 3894 * performance under such circumstance. 3895 */ 3896 pkt_dev->dstops.family = AF_INET; 3897 pkt_dev->xdst.u.dst.dev = pkt_dev->odev; 3898 dst_init_metrics(&pkt_dev->xdst.u.dst, pktgen_dst_metrics, false); 3899 pkt_dev->xdst.child = &pkt_dev->xdst.u.dst; 3900 pkt_dev->xdst.u.dst.ops = &pkt_dev->dstops; 3901 #endif 3902 3903 return add_dev_to_thread(t, pkt_dev); 3904 out2: 3905 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); 3906 out1: 3907 #ifdef CONFIG_XFRM 3908 free_SAs(pkt_dev); 3909 #endif 3910 vfree(pkt_dev->flows); 3911 kfree(pkt_dev); 3912 return err; 3913 } 3914 3915 static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn) 3916 { 3917 struct pktgen_thread *t; 3918 struct proc_dir_entry *pe; 3919 struct task_struct *p; 3920 3921 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL, 3922 cpu_to_node(cpu)); 3923 if (!t) { 3924 pr_err("ERROR: out of memory, can't create new thread\n"); 3925 return -ENOMEM; 3926 } 3927 3928 mutex_init(&t->if_lock); 3929 t->cpu = cpu; 3930 3931 INIT_LIST_HEAD(&t->if_list); 3932 3933 list_add_tail(&t->th_list, &pn->pktgen_threads); 3934 init_completion(&t->start_done); 3935 3936 p = kthread_create_on_cpu(pktgen_thread_worker, t, cpu, "kpktgend_%d"); 3937 if (IS_ERR(p)) { 3938 pr_err("kthread_create_on_node() failed for cpu %d\n", t->cpu); 3939 list_del(&t->th_list); 3940 kfree(t); 3941 return PTR_ERR(p); 3942 } 3943 3944 t->tsk = p; 3945 3946 pe = proc_create_data(t->tsk->comm, 0600, pn->proc_dir, 3947 &pktgen_thread_proc_ops, t); 3948 if (!pe) { 3949 pr_err("cannot create %s/%s procfs entry\n", 3950 PG_PROC_DIR, t->tsk->comm); 3951 kthread_stop(p); 3952 list_del(&t->th_list); 3953 kfree(t); 3954 return -EINVAL; 3955 } 3956 3957 t->net = pn; 3958 get_task_struct(p); 3959 wake_up_process(p); 3960 wait_for_completion(&t->start_done); 3961 3962 return 0; 3963 } 3964 3965 /* 3966 * Removes a device from the thread if_list. 3967 */ 3968 static void _rem_dev_from_if_list(struct pktgen_thread *t, 3969 struct pktgen_dev *pkt_dev) 3970 { 3971 struct list_head *q, *n; 3972 struct pktgen_dev *p; 3973 3974 if_lock(t); 3975 proc_remove(pkt_dev->entry); 3976 list_for_each_safe(q, n, &t->if_list) { 3977 p = list_entry(q, struct pktgen_dev, list); 3978 if (p == pkt_dev) 3979 list_del_rcu(&p->list); 3980 } 3981 if_unlock(t); 3982 } 3983 3984 static int pktgen_remove_device(struct pktgen_thread *t, 3985 struct pktgen_dev *pkt_dev) 3986 { 3987 pr_debug("remove_device pkt_dev=%p\n", pkt_dev); 3988 3989 if (pkt_dev->running) { 3990 pr_warn("WARNING: trying to remove a running interface, stopping it now\n"); 3991 pktgen_stop_device(pkt_dev); 3992 } 3993 3994 /* Dis-associate from the interface */ 3995 3996 if (pkt_dev->odev) { 3997 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); 3998 pkt_dev->odev = NULL; 3999 } 4000 4001 /* Remove proc before if_list entry, because add_device uses 4002 * list to determine if interface already exist, avoid race 4003 * with proc_create_data() 4004 */ 4005 _rem_dev_from_if_list(t, pkt_dev); 4006 4007 #ifdef CONFIG_XFRM 4008 free_SAs(pkt_dev); 4009 #endif 4010 vfree(pkt_dev->flows); 4011 if (pkt_dev->page) 4012 put_page(pkt_dev->page); 4013 kfree_rcu(pkt_dev, rcu); 4014 return 0; 4015 } 4016 4017 static int __net_init pg_net_init(struct net *net) 4018 { 4019 struct pktgen_net *pn = net_generic(net, pg_net_id); 4020 struct proc_dir_entry *pe; 4021 int cpu, ret = 0; 4022 4023 pn->net = net; 4024 INIT_LIST_HEAD(&pn->pktgen_threads); 4025 pn->pktgen_exiting = false; 4026 pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net); 4027 if (!pn->proc_dir) { 4028 pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR); 4029 return -ENODEV; 4030 } 4031 pe = proc_create(PGCTRL, 0600, pn->proc_dir, &pktgen_proc_ops); 4032 if (pe == NULL) { 4033 pr_err("cannot create %s procfs entry\n", PGCTRL); 4034 ret = -EINVAL; 4035 goto remove; 4036 } 4037 4038 cpus_read_lock(); 4039 for_each_online_cpu(cpu) { 4040 int err; 4041 4042 err = pktgen_create_thread(cpu, pn); 4043 if (err) 4044 pr_warn("Cannot create thread for cpu %d (%d)\n", 4045 cpu, err); 4046 } 4047 cpus_read_unlock(); 4048 4049 if (list_empty(&pn->pktgen_threads)) { 4050 pr_err("Initialization failed for all threads\n"); 4051 ret = -ENODEV; 4052 goto remove_entry; 4053 } 4054 4055 return 0; 4056 4057 remove_entry: 4058 remove_proc_entry(PGCTRL, pn->proc_dir); 4059 remove: 4060 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net); 4061 return ret; 4062 } 4063 4064 static void __net_exit pg_net_exit(struct net *net) 4065 { 4066 struct pktgen_net *pn = net_generic(net, pg_net_id); 4067 struct pktgen_thread *t; 4068 struct list_head *q, *n; 4069 LIST_HEAD(list); 4070 4071 /* Stop all interfaces & threads */ 4072 pn->pktgen_exiting = true; 4073 4074 mutex_lock(&pktgen_thread_lock); 4075 list_splice_init(&pn->pktgen_threads, &list); 4076 mutex_unlock(&pktgen_thread_lock); 4077 4078 list_for_each_safe(q, n, &list) { 4079 t = list_entry(q, struct pktgen_thread, th_list); 4080 list_del(&t->th_list); 4081 kthread_stop_put(t->tsk); 4082 kfree(t); 4083 } 4084 4085 remove_proc_entry(PGCTRL, pn->proc_dir); 4086 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net); 4087 } 4088 4089 static struct pernet_operations pg_net_ops = { 4090 .init = pg_net_init, 4091 .exit = pg_net_exit, 4092 .id = &pg_net_id, 4093 .size = sizeof(struct pktgen_net), 4094 }; 4095 4096 static int __init pg_init(void) 4097 { 4098 int ret = 0; 4099 4100 pr_info("%s", version); 4101 ret = register_pernet_subsys(&pg_net_ops); 4102 if (ret) 4103 return ret; 4104 ret = register_netdevice_notifier(&pktgen_notifier_block); 4105 if (ret) 4106 unregister_pernet_subsys(&pg_net_ops); 4107 4108 return ret; 4109 } 4110 4111 static void __exit pg_cleanup(void) 4112 { 4113 unregister_netdevice_notifier(&pktgen_notifier_block); 4114 unregister_pernet_subsys(&pg_net_ops); 4115 /* Don't need rcu_barrier() due to use of kfree_rcu() */ 4116 } 4117 4118 module_init(pg_init); 4119 module_exit(pg_cleanup); 4120 4121 MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>"); 4122 MODULE_DESCRIPTION("Packet Generator tool"); 4123 MODULE_LICENSE("GPL"); 4124 MODULE_VERSION(VERSION); 4125 module_param(pg_count_d, int, 0); 4126 MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject"); 4127 module_param(pg_delay_d, int, 0); 4128 MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)"); 4129 module_param(pg_clone_skb_d, int, 0); 4130 MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet"); 4131 module_param(debug, int, 0); 4132 MODULE_PARM_DESC(debug, "Enable debugging of pktgen module"); 4133