1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Authors: 4 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se> 5 * Uppsala University and 6 * Swedish University of Agricultural Sciences 7 * 8 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 9 * Ben Greear <greearb@candelatech.com> 10 * Jens Låås <jens.laas@data.slu.se> 11 * 12 * A tool for loading the network with preconfigurated packets. 13 * The tool is implemented as a linux module. Parameters are output 14 * device, delay (to hard_xmit), number of packets, and whether 15 * to use multiple SKBs or just the same one. 16 * pktgen uses the installed interface's output routine. 17 * 18 * Additional hacking by: 19 * 20 * Jens.Laas@data.slu.se 21 * Improved by ANK. 010120. 22 * Improved by ANK even more. 010212. 23 * MAC address typo fixed. 010417 --ro 24 * Integrated. 020301 --DaveM 25 * Added multiskb option 020301 --DaveM 26 * Scaling of results. 020417--sigurdur@linpro.no 27 * Significant re-work of the module: 28 * * Convert to threaded model to more efficiently be able to transmit 29 * and receive on multiple interfaces at once. 30 * * Converted many counters to __u64 to allow longer runs. 31 * * Allow configuration of ranges, like min/max IP address, MACs, 32 * and UDP-ports, for both source and destination, and can 33 * set to use a random distribution or sequentially walk the range. 34 * * Can now change most values after starting. 35 * * Place 12-byte packet in UDP payload with magic number, 36 * sequence number, and timestamp. 37 * * Add receiver code that detects dropped pkts, re-ordered pkts, and 38 * latencies (with micro-second) precision. 39 * * Add IOCTL interface to easily get counters & configuration. 40 * --Ben Greear <greearb@candelatech.com> 41 * 42 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct 43 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0 44 * as a "fastpath" with a configurable number of clones after alloc's. 45 * clone_skb=0 means all packets are allocated this also means ranges time 46 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100 47 * clones. 48 * 49 * Also moved to /proc/net/pktgen/ 50 * --ro 51 * 52 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever 53 * mistakes. Also merged in DaveM's patch in the -pre6 patch. 54 * --Ben Greear <greearb@candelatech.com> 55 * 56 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br) 57 * 58 * 021124 Finished major redesign and rewrite for new functionality. 59 * See Documentation/networking/pktgen.rst for how to use this. 60 * 61 * The new operation: 62 * For each CPU one thread/process is created at start. This process checks 63 * for running devices in the if_list and sends packets until count is 0 it 64 * also the thread checks the thread->control which is used for inter-process 65 * communication. controlling process "posts" operations to the threads this 66 * way. 67 * The if_list is RCU protected, and the if_lock remains to protect updating 68 * of if_list, from "add_device" as it invoked from userspace (via proc write). 69 * 70 * By design there should only be *one* "controlling" process. In practice 71 * multiple write accesses gives unpredictable result. Understood by "write" 72 * to /proc gives result code that should be read be the "writer". 73 * For practical use this should be no problem. 74 * 75 * Note when adding devices to a specific CPU there good idea to also assign 76 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU. 77 * --ro 78 * 79 * Fix refcount off by one if first packet fails, potential null deref, 80 * memleak 030710- KJP 81 * 82 * First "ranges" functionality for ipv6 030726 --ro 83 * 84 * Included flow support. 030802 ANK. 85 * 86 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org> 87 * 88 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419 89 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604 90 * 91 * New xmit() return, do_div and misc clean up by Stephen Hemminger 92 * <shemminger@osdl.org> 040923 93 * 94 * Randy Dunlap fixed u64 printk compiler warning 95 * 96 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org> 97 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213 98 * 99 * Corrections from Nikolai Malykh (nmalykh@bilim.com) 100 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230 101 * 102 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com> 103 * 050103 104 * 105 * MPLS support by Steven Whitehouse <steve@chygwyn.com> 106 * 107 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com> 108 * 109 * Fixed src_mac command to set source mac of packet to value specified in 110 * command by Adit Ranadive <adit.262@gmail.com> 111 */ 112 113 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 114 115 #include <linux/sys.h> 116 #include <linux/types.h> 117 #include <linux/minmax.h> 118 #include <linux/module.h> 119 #include <linux/moduleparam.h> 120 #include <linux/kernel.h> 121 #include <linux/mutex.h> 122 #include <linux/sched.h> 123 #include <linux/slab.h> 124 #include <linux/vmalloc.h> 125 #include <linux/unistd.h> 126 #include <linux/string.h> 127 #include <linux/ptrace.h> 128 #include <linux/errno.h> 129 #include <linux/hex.h> 130 #include <linux/ioport.h> 131 #include <linux/interrupt.h> 132 #include <linux/capability.h> 133 #include <linux/hrtimer.h> 134 #include <linux/freezer.h> 135 #include <linux/delay.h> 136 #include <linux/timer.h> 137 #include <linux/list.h> 138 #include <linux/init.h> 139 #include <linux/skbuff.h> 140 #include <linux/netdevice.h> 141 #include <linux/inet.h> 142 #include <linux/inetdevice.h> 143 #include <linux/rtnetlink.h> 144 #include <linux/if_arp.h> 145 #include <linux/if_vlan.h> 146 #include <linux/in.h> 147 #include <linux/ip.h> 148 #include <linux/ipv6.h> 149 #include <linux/udp.h> 150 #include <linux/proc_fs.h> 151 #include <linux/seq_file.h> 152 #include <linux/wait.h> 153 #include <linux/etherdevice.h> 154 #include <linux/kthread.h> 155 #include <linux/prefetch.h> 156 #include <linux/mmzone.h> 157 #include <net/net_namespace.h> 158 #include <net/checksum.h> 159 #include <net/ipv6.h> 160 #include <net/udp.h> 161 #include <net/ip6_checksum.h> 162 #include <net/addrconf.h> 163 #include <net/xfrm.h> 164 #include <net/netns/generic.h> 165 #include <asm/byteorder.h> 166 #include <linux/rcupdate.h> 167 #include <linux/bitops.h> 168 #include <linux/io.h> 169 #include <linux/timex.h> 170 #include <linux/uaccess.h> 171 #include <asm/dma.h> 172 #include <asm/div64.h> /* do_div */ 173 174 #define VERSION "2.75" 175 #define IP_NAME_SZ 32 176 #define MAX_MPLS_LABELS 16 /* This is the max label stack depth */ 177 #define MPLS_STACK_BOTTOM htonl(0x00000100) 178 /* Max number of internet mix entries that can be specified in imix_weights. */ 179 #define MAX_IMIX_ENTRIES 20 180 #define IMIX_PRECISION 100 /* Precision of IMIX distribution */ 181 182 #define func_enter() pr_debug("entering %s\n", __func__) 183 184 #define PKT_FLAGS \ 185 pf(IPV6) /* Interface in IPV6 Mode */ \ 186 pf(IPSRC_RND) /* IP-Src Random */ \ 187 pf(IPDST_RND) /* IP-Dst Random */ \ 188 pf(TXSIZE_RND) /* Transmit size is random */ \ 189 pf(UDPSRC_RND) /* UDP-Src Random */ \ 190 pf(UDPDST_RND) /* UDP-Dst Random */ \ 191 pf(UDPCSUM) /* Include UDP checksum */ \ 192 pf(NO_TIMESTAMP) /* Don't timestamp packets (default TS) */ \ 193 pf(MPLS_RND) /* Random MPLS labels */ \ 194 pf(QUEUE_MAP_RND) /* queue map Random */ \ 195 pf(QUEUE_MAP_CPU) /* queue map mirrors smp_processor_id() */ \ 196 pf(FLOW_SEQ) /* Sequential flows */ \ 197 pf(IPSEC) /* ipsec on for flows */ \ 198 pf(MACSRC_RND) /* MAC-Src Random */ \ 199 pf(MACDST_RND) /* MAC-Dst Random */ \ 200 pf(VID_RND) /* Random VLAN ID */ \ 201 pf(SVID_RND) /* Random SVLAN ID */ \ 202 pf(NODE) /* Node memory alloc*/ \ 203 pf(SHARED) /* Shared SKB */ \ 204 205 #define pf(flag) flag##_SHIFT, 206 enum pkt_flags { 207 PKT_FLAGS 208 }; 209 #undef pf 210 211 /* Device flag bits */ 212 #define pf(flag) static const __u32 F_##flag = (1<<flag##_SHIFT); 213 PKT_FLAGS 214 #undef pf 215 216 #define pf(flag) __stringify(flag), 217 static char *pkt_flag_names[] = { 218 PKT_FLAGS 219 }; 220 #undef pf 221 222 #define NR_PKT_FLAGS ARRAY_SIZE(pkt_flag_names) 223 224 /* Thread control flag bits */ 225 #define T_STOP (1<<0) /* Stop run */ 226 #define T_RUN (1<<1) /* Start run */ 227 #define T_REMDEVALL (1<<2) /* Remove all devs */ 228 #define T_REMDEV (1<<3) /* Remove one dev */ 229 230 /* Xmit modes */ 231 #define M_START_XMIT 0 /* Default normal TX */ 232 #define M_NETIF_RECEIVE 1 /* Inject packets into stack */ 233 #define M_QUEUE_XMIT 2 /* Inject packet into qdisc */ 234 235 /* If lock -- protects updating of if_list */ 236 #define if_lock(t) mutex_lock(&(t->if_lock)) 237 #define if_unlock(t) mutex_unlock(&(t->if_lock)) 238 239 /* Used to help with determining the pkts on receive */ 240 #define PKTGEN_MAGIC 0xbe9be955 241 #define PG_PROC_DIR "pktgen" 242 #define PGCTRL "pgctrl" 243 244 #define MAX_CFLOWS 65536 245 246 #define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4) 247 #define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4) 248 249 struct imix_pkt { 250 u64 size; 251 u64 weight; 252 u64 count_so_far; 253 }; 254 255 struct flow_state { 256 __be32 cur_daddr; 257 int count; 258 #ifdef CONFIG_XFRM 259 struct xfrm_state *x; 260 #endif 261 __u32 flags; 262 }; 263 264 /* flow flag bits */ 265 #define F_INIT (1<<0) /* flow has been initialized */ 266 267 struct pktgen_dev { 268 /* 269 * Try to keep frequent/infrequent used vars. separated. 270 */ 271 struct proc_dir_entry *entry; /* proc file */ 272 struct pktgen_thread *pg_thread;/* the owner */ 273 struct list_head list; /* chaining in the thread's run-queue */ 274 struct rcu_head rcu; /* freed by RCU */ 275 276 int running; /* if false, the test will stop */ 277 278 /* If min != max, then we will either do a linear iteration, or 279 * we will do a random selection from within the range. 280 */ 281 __u32 flags; 282 int xmit_mode; 283 int min_pkt_size; 284 int max_pkt_size; 285 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */ 286 int nfrags; 287 int removal_mark; /* non-zero => the device is marked for 288 * removal by worker thread 289 */ 290 291 struct page *page; 292 u64 delay; /* nano-seconds */ 293 294 __u64 count; /* Default No packets to send */ 295 __u64 sofar; /* How many pkts we've sent so far */ 296 __u64 tx_bytes; /* How many bytes we've transmitted */ 297 __u64 errors; /* Errors when trying to transmit, */ 298 299 /* runtime counters relating to clone_skb */ 300 301 __u32 clone_count; 302 int last_ok; /* Was last skb sent? 303 * Or a failed transmit of some sort? 304 * This will keep sequence numbers in order 305 */ 306 ktime_t next_tx; 307 ktime_t started_at; 308 ktime_t stopped_at; 309 u64 idle_acc; /* nano-seconds */ 310 311 __u32 seq_num; 312 313 int clone_skb; /* 314 * Use multiple SKBs during packet gen. 315 * If this number is greater than 1, then 316 * that many copies of the same packet will be 317 * sent before a new packet is allocated. 318 * If you want to send 1024 identical packets 319 * before creating a new packet, 320 * set clone_skb to 1024. 321 */ 322 323 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 324 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 325 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 326 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 327 328 struct in6_addr in6_saddr; 329 struct in6_addr in6_daddr; 330 struct in6_addr cur_in6_daddr; 331 struct in6_addr cur_in6_saddr; 332 /* For ranges */ 333 struct in6_addr min_in6_daddr; 334 struct in6_addr max_in6_daddr; 335 struct in6_addr min_in6_saddr; 336 struct in6_addr max_in6_saddr; 337 338 /* If we're doing ranges, random or incremental, then this 339 * defines the min/max for those ranges. 340 */ 341 __be32 saddr_min; /* inclusive, source IP address */ 342 __be32 saddr_max; /* exclusive, source IP address */ 343 __be32 daddr_min; /* inclusive, dest IP address */ 344 __be32 daddr_max; /* exclusive, dest IP address */ 345 346 __u16 udp_src_min; /* inclusive, source UDP port */ 347 __u16 udp_src_max; /* exclusive, source UDP port */ 348 __u16 udp_dst_min; /* inclusive, dest UDP port */ 349 __u16 udp_dst_max; /* exclusive, dest UDP port */ 350 351 /* DSCP + ECN */ 352 __u8 tos; /* six MSB of (former) IPv4 TOS 353 * are for dscp codepoint 354 */ 355 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6 356 * (see RFC 3260, sec. 4) 357 */ 358 359 /* IMIX */ 360 unsigned int n_imix_entries; 361 struct imix_pkt imix_entries[MAX_IMIX_ENTRIES]; 362 /* Maps 0-IMIX_PRECISION range to imix_entry based on probability*/ 363 __u8 imix_distribution[IMIX_PRECISION]; 364 365 /* MPLS */ 366 unsigned int nr_labels; /* Depth of stack, 0 = no MPLS */ 367 __be32 labels[MAX_MPLS_LABELS]; 368 369 /* VLAN/SVLAN (802.1Q/Q-in-Q) */ 370 __u8 vlan_p; 371 __u8 vlan_cfi; 372 __u16 vlan_id; /* 0xffff means no vlan tag */ 373 374 __u8 svlan_p; 375 __u8 svlan_cfi; 376 __u16 svlan_id; /* 0xffff means no svlan tag */ 377 378 __u32 src_mac_count; /* How many MACs to iterate through */ 379 __u32 dst_mac_count; /* How many MACs to iterate through */ 380 381 unsigned char dst_mac[ETH_ALEN]; 382 unsigned char src_mac[ETH_ALEN]; 383 384 __u32 cur_dst_mac_offset; 385 __u32 cur_src_mac_offset; 386 __be32 cur_saddr; 387 __be32 cur_daddr; 388 __u16 ip_id; 389 __u16 cur_udp_dst; 390 __u16 cur_udp_src; 391 __u16 cur_queue_map; 392 __u32 cur_pkt_size; 393 __u32 last_pkt_size; 394 395 __u8 hh[14]; 396 /* = { 397 * 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, 398 * 399 * We fill in SRC address later 400 * 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 401 * 0x08, 0x00 402 * }; 403 */ 404 __u16 pad; /* pad out the hh struct to an even 16 bytes */ 405 406 struct sk_buff *skb; /* skb we are to transmit next, used for when we 407 * are transmitting the same one multiple times 408 */ 409 struct net_device *odev; /* The out-going device. 410 * Note that the device should have it's 411 * pg_info pointer pointing back to this 412 * device. 413 * Set when the user specifies the out-going 414 * device name (not when the inject is 415 * started as it used to do.) 416 */ 417 netdevice_tracker dev_tracker; 418 char odevname[32]; 419 struct flow_state *flows; 420 unsigned int cflows; /* Concurrent flows (config) */ 421 unsigned int lflow; /* Flow length (config) */ 422 unsigned int nflows; /* accumulated flows (stats) */ 423 unsigned int curfl; /* current sequenced flow (state)*/ 424 425 u16 queue_map_min; 426 u16 queue_map_max; 427 __u32 skb_priority; /* skb priority field */ 428 unsigned int burst; /* number of duplicated packets to burst */ 429 int node; /* Memory node */ 430 431 #ifdef CONFIG_XFRM 432 __u8 ipsmode; /* IPSEC mode (config) */ 433 __u8 ipsproto; /* IPSEC type (config) */ 434 __u32 spi; 435 struct xfrm_dst xdst; 436 struct dst_ops dstops; 437 #endif 438 char result[512]; 439 }; 440 441 struct pktgen_hdr { 442 __be32 pgh_magic; 443 __be32 seq_num; 444 __be32 tv_sec; 445 __be32 tv_usec; 446 }; 447 448 449 static unsigned int pg_net_id __read_mostly; 450 451 struct pktgen_net { 452 struct net *net; 453 struct proc_dir_entry *proc_dir; 454 struct list_head pktgen_threads; 455 bool pktgen_exiting; 456 }; 457 458 struct pktgen_thread { 459 struct mutex if_lock; /* for list of devices */ 460 struct list_head if_list; /* All device here */ 461 struct list_head th_list; 462 struct task_struct *tsk; 463 char result[512]; 464 465 /* Field for thread to receive "posted" events terminate, 466 * stop ifs etc. 467 */ 468 469 u32 control; 470 int cpu; 471 472 wait_queue_head_t queue; 473 struct completion start_done; 474 struct pktgen_net *net; 475 }; 476 477 #define REMOVE 1 478 #define FIND 0 479 480 static const char version[] = 481 "Packet Generator for packet performance testing. Version: " VERSION "\n"; 482 483 static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i); 484 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname); 485 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 486 const char *ifname, bool exact); 487 static int pktgen_device_event(struct notifier_block *, unsigned long, void *); 488 static void pktgen_run_all_threads(struct pktgen_net *pn); 489 static void pktgen_reset_all_threads(struct pktgen_net *pn); 490 static void pktgen_stop_all_threads(struct pktgen_net *pn); 491 492 static void pktgen_stop(struct pktgen_thread *t); 493 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev); 494 static void fill_imix_distribution(struct pktgen_dev *pkt_dev); 495 496 /* Module parameters, defaults. */ 497 static int pg_count_d __read_mostly = 1000; 498 static int pg_delay_d __read_mostly; 499 static int pg_clone_skb_d __read_mostly; 500 static int debug __read_mostly; 501 502 static DEFINE_MUTEX(pktgen_thread_lock); 503 504 static struct notifier_block pktgen_notifier_block = { 505 .notifier_call = pktgen_device_event, 506 }; 507 508 /* 509 * /proc handling functions 510 * 511 */ 512 513 static int pgctrl_show(struct seq_file *seq, void *v) 514 { 515 seq_puts(seq, version); 516 return 0; 517 } 518 519 static ssize_t pgctrl_write(struct file *file, const char __user *buf, 520 size_t count, loff_t *ppos) 521 { 522 char data[128]; 523 size_t max; 524 struct pktgen_net *pn = net_generic(current->nsproxy->net_ns, pg_net_id); 525 526 if (!capable(CAP_NET_ADMIN)) 527 return -EPERM; 528 529 if (count < 1) 530 return -EINVAL; 531 532 max = min(count, sizeof(data) - 1); 533 if (copy_from_user(data, buf, max)) 534 return -EFAULT; 535 536 if (data[max - 1] == '\n') 537 data[max - 1] = 0; /* strip trailing '\n', terminate string */ 538 else 539 data[max] = 0; /* terminate string */ 540 541 if (!strcmp(data, "stop")) 542 pktgen_stop_all_threads(pn); 543 else if (!strcmp(data, "start")) 544 pktgen_run_all_threads(pn); 545 else if (!strcmp(data, "reset")) 546 pktgen_reset_all_threads(pn); 547 else 548 return -EINVAL; 549 550 return count; 551 } 552 553 static int pgctrl_open(struct inode *inode, struct file *file) 554 { 555 return single_open(file, pgctrl_show, pde_data(inode)); 556 } 557 558 static const struct proc_ops pktgen_proc_ops = { 559 .proc_open = pgctrl_open, 560 .proc_read = seq_read, 561 .proc_lseek = seq_lseek, 562 .proc_write = pgctrl_write, 563 .proc_release = single_release, 564 }; 565 566 static int pktgen_if_show(struct seq_file *seq, void *v) 567 { 568 const struct pktgen_dev *pkt_dev = seq->private; 569 unsigned int cflows = READ_ONCE(pkt_dev->cflows); 570 ktime_t stopped; 571 unsigned int i; 572 u64 idle; 573 574 seq_printf(seq, 575 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n", 576 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size, 577 pkt_dev->max_pkt_size); 578 579 if (pkt_dev->n_imix_entries > 0) { 580 seq_puts(seq, " imix_weights: "); 581 for (i = 0; i < pkt_dev->n_imix_entries; i++) { 582 seq_printf(seq, "%llu,%llu ", 583 pkt_dev->imix_entries[i].size, 584 pkt_dev->imix_entries[i].weight); 585 } 586 seq_puts(seq, "\n"); 587 } 588 589 seq_printf(seq, 590 " frags: %d delay: %llu clone_skb: %d ifname: %s\n", 591 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay, 592 pkt_dev->clone_skb, pkt_dev->odevname); 593 594 seq_printf(seq, " flows: %u flowlen: %u\n", cflows, 595 pkt_dev->lflow); 596 597 seq_printf(seq, 598 " queue_map_min: %u queue_map_max: %u\n", 599 pkt_dev->queue_map_min, 600 pkt_dev->queue_map_max); 601 602 if (pkt_dev->skb_priority) 603 seq_printf(seq, " skb_priority: %u\n", 604 pkt_dev->skb_priority); 605 606 if (pkt_dev->flags & F_IPV6) { 607 seq_printf(seq, 608 " saddr: %pI6c min_saddr: %pI6c max_saddr: %pI6c\n" 609 " daddr: %pI6c min_daddr: %pI6c max_daddr: %pI6c\n", 610 &pkt_dev->in6_saddr, 611 &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr, 612 &pkt_dev->in6_daddr, 613 &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr); 614 } else { 615 seq_printf(seq, 616 " dst_min: %s dst_max: %s\n", 617 pkt_dev->dst_min, pkt_dev->dst_max); 618 seq_printf(seq, 619 " src_min: %s src_max: %s\n", 620 pkt_dev->src_min, pkt_dev->src_max); 621 } 622 623 seq_puts(seq, " src_mac: "); 624 625 seq_printf(seq, "%pM ", 626 is_zero_ether_addr(pkt_dev->src_mac) ? 627 pkt_dev->odev->dev_addr : pkt_dev->src_mac); 628 629 seq_puts(seq, "dst_mac: "); 630 seq_printf(seq, "%pM\n", pkt_dev->dst_mac); 631 632 seq_printf(seq, 633 " udp_src_min: %d udp_src_max: %d udp_dst_min: %d udp_dst_max: %d\n", 634 pkt_dev->udp_src_min, pkt_dev->udp_src_max, 635 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max); 636 637 seq_printf(seq, 638 " src_mac_count: %d dst_mac_count: %d\n", 639 pkt_dev->src_mac_count, pkt_dev->dst_mac_count); 640 641 if (pkt_dev->nr_labels) { 642 seq_puts(seq, " mpls: "); 643 for (i = 0; i < pkt_dev->nr_labels; i++) 644 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]), 645 i == pkt_dev->nr_labels-1 ? "\n" : ", "); 646 } 647 648 if (pkt_dev->vlan_id != 0xffff) 649 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n", 650 pkt_dev->vlan_id, pkt_dev->vlan_p, 651 pkt_dev->vlan_cfi); 652 653 if (pkt_dev->svlan_id != 0xffff) 654 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n", 655 pkt_dev->svlan_id, pkt_dev->svlan_p, 656 pkt_dev->svlan_cfi); 657 658 if (pkt_dev->tos) 659 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos); 660 661 if (pkt_dev->traffic_class) 662 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class); 663 664 if (pkt_dev->burst > 1) 665 seq_printf(seq, " burst: %d\n", pkt_dev->burst); 666 667 if (pkt_dev->node >= 0) 668 seq_printf(seq, " node: %d\n", pkt_dev->node); 669 670 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) 671 seq_puts(seq, " xmit_mode: netif_receive\n"); 672 else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) 673 seq_puts(seq, " xmit_mode: xmit_queue\n"); 674 675 seq_puts(seq, " Flags: "); 676 677 for (i = 0; i < NR_PKT_FLAGS; i++) { 678 if (i == FLOW_SEQ_SHIFT) 679 if (!cflows) 680 continue; 681 682 if (pkt_dev->flags & (1 << i)) { 683 seq_printf(seq, "%s ", pkt_flag_names[i]); 684 #ifdef CONFIG_XFRM 685 if (i == IPSEC_SHIFT && pkt_dev->spi) 686 seq_printf(seq, "spi:%u ", pkt_dev->spi); 687 #endif 688 } else if (i == FLOW_SEQ_SHIFT) { 689 seq_puts(seq, "FLOW_RND "); 690 } 691 } 692 693 seq_puts(seq, "\n"); 694 695 /* not really stopped, more like last-running-at */ 696 stopped = pkt_dev->running ? ktime_get() : pkt_dev->stopped_at; 697 idle = pkt_dev->idle_acc; 698 do_div(idle, NSEC_PER_USEC); 699 700 seq_printf(seq, 701 "Current:\n pkts-sofar: %llu errors: %llu\n", 702 (unsigned long long)pkt_dev->sofar, 703 (unsigned long long)pkt_dev->errors); 704 705 if (pkt_dev->n_imix_entries > 0) { 706 int i; 707 708 seq_puts(seq, " imix_size_counts: "); 709 for (i = 0; i < pkt_dev->n_imix_entries; i++) { 710 seq_printf(seq, "%llu,%llu ", 711 pkt_dev->imix_entries[i].size, 712 pkt_dev->imix_entries[i].count_so_far); 713 } 714 seq_puts(seq, "\n"); 715 } 716 717 seq_printf(seq, 718 " started: %lluus stopped: %lluus idle: %lluus\n", 719 (unsigned long long) ktime_to_us(pkt_dev->started_at), 720 (unsigned long long) ktime_to_us(stopped), 721 (unsigned long long) idle); 722 723 seq_printf(seq, 724 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n", 725 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset, 726 pkt_dev->cur_src_mac_offset); 727 728 if (pkt_dev->flags & F_IPV6) { 729 seq_printf(seq, " cur_saddr: %pI6c cur_daddr: %pI6c\n", 730 &pkt_dev->cur_in6_saddr, 731 &pkt_dev->cur_in6_daddr); 732 } else 733 seq_printf(seq, " cur_saddr: %pI4 cur_daddr: %pI4\n", 734 &pkt_dev->cur_saddr, &pkt_dev->cur_daddr); 735 736 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n", 737 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src); 738 739 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map); 740 741 seq_printf(seq, " flows: %u\n", pkt_dev->nflows); 742 743 if (pkt_dev->result[0]) 744 seq_printf(seq, "Result: %s\n", pkt_dev->result); 745 else 746 seq_puts(seq, "Result: Idle\n"); 747 748 return 0; 749 } 750 751 752 static ssize_t hex32_arg(const char __user *user_buffer, size_t maxlen, 753 __u32 *num) 754 { 755 size_t i = 0; 756 757 *num = 0; 758 759 for (; i < maxlen; i++) { 760 int value; 761 char c; 762 763 if (get_user(c, &user_buffer[i])) 764 return -EFAULT; 765 value = hex_to_bin(c); 766 if (value >= 0) { 767 *num <<= 4; 768 *num |= value; 769 } else { 770 break; 771 } 772 } 773 return i; 774 } 775 776 static ssize_t count_trail_chars(const char __user *user_buffer, size_t maxlen) 777 { 778 size_t i; 779 780 for (i = 0; i < maxlen; i++) { 781 char c; 782 783 if (get_user(c, &user_buffer[i])) 784 return -EFAULT; 785 switch (c) { 786 case '\"': 787 case '\n': 788 case '\r': 789 case '\t': 790 case ' ': 791 case '=': 792 break; 793 default: 794 goto done; 795 } 796 } 797 done: 798 return i; 799 } 800 801 static ssize_t num_arg(const char __user *user_buffer, size_t maxlen, 802 unsigned long *num) 803 { 804 size_t i; 805 *num = 0; 806 807 for (i = 0; i < maxlen; i++) { 808 char c; 809 810 if (get_user(c, &user_buffer[i])) 811 return -EFAULT; 812 if ((c >= '0') && (c <= '9')) { 813 *num *= 10; 814 *num += c - '0'; 815 } else 816 break; 817 } 818 return i; 819 } 820 821 static ssize_t strn_len(const char __user *user_buffer, size_t maxlen) 822 { 823 size_t i; 824 825 for (i = 0; i < maxlen; i++) { 826 char c; 827 828 if (get_user(c, &user_buffer[i])) 829 return -EFAULT; 830 switch (c) { 831 case '\"': 832 case '\n': 833 case '\r': 834 case '\t': 835 case ' ': 836 case '=': 837 goto done_str; 838 default: 839 break; 840 } 841 } 842 done_str: 843 return i; 844 } 845 846 /* Parses imix entries from user buffer. 847 * The user buffer should consist of imix entries separated by spaces 848 * where each entry consists of size and weight delimited by commas. 849 * "size1,weight_1 size2,weight_2 ... size_n,weight_n" for example. 850 */ 851 static ssize_t get_imix_entries(const char __user *buffer, 852 size_t maxlen, 853 struct pktgen_dev *pkt_dev) 854 { 855 size_t i = 0, max; 856 ssize_t len; 857 char c; 858 859 pkt_dev->n_imix_entries = 0; 860 861 do { 862 unsigned long weight; 863 unsigned long size; 864 865 if (pkt_dev->n_imix_entries >= MAX_IMIX_ENTRIES) 866 return -E2BIG; 867 868 if (i >= maxlen) 869 return -EINVAL; 870 871 max = min(10, maxlen - i); 872 len = num_arg(&buffer[i], max, &size); 873 if (len < 0) 874 return len; 875 i += len; 876 if (i >= maxlen) 877 return -EINVAL; 878 if (get_user(c, &buffer[i])) 879 return -EFAULT; 880 /* Check for comma between size_i and weight_i */ 881 if (c != ',') 882 return -EINVAL; 883 i++; 884 if (i >= maxlen) 885 return -EINVAL; 886 887 if (size < 14 + 20 + 8) 888 size = 14 + 20 + 8; 889 890 max = min(10, maxlen - i); 891 len = num_arg(&buffer[i], max, &weight); 892 if (len < 0) 893 return len; 894 if (weight <= 0) 895 return -EINVAL; 896 897 pkt_dev->imix_entries[pkt_dev->n_imix_entries].size = size; 898 pkt_dev->imix_entries[pkt_dev->n_imix_entries].weight = weight; 899 900 i += len; 901 pkt_dev->n_imix_entries++; 902 903 if (i >= maxlen) 904 break; 905 if (get_user(c, &buffer[i])) 906 return -EFAULT; 907 i++; 908 } while (c == ' '); 909 910 return i; 911 } 912 913 static ssize_t get_labels(const char __user *buffer, 914 size_t maxlen, struct pktgen_dev *pkt_dev) 915 { 916 unsigned int n = 0; 917 size_t i = 0, max; 918 ssize_t len; 919 char c; 920 921 pkt_dev->nr_labels = 0; 922 do { 923 __u32 tmp; 924 925 if (n >= MAX_MPLS_LABELS) 926 return -E2BIG; 927 928 if (i >= maxlen) 929 return -EINVAL; 930 931 max = min(8, maxlen - i); 932 len = hex32_arg(&buffer[i], max, &tmp); 933 if (len < 0) 934 return len; 935 936 /* return empty list in case of invalid input or zero value */ 937 if (len == 0 || tmp == 0) 938 return maxlen; 939 940 pkt_dev->labels[n] = htonl(tmp); 941 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM) 942 pkt_dev->flags |= F_MPLS_RND; 943 i += len; 944 n++; 945 if (i >= maxlen) 946 break; 947 if (get_user(c, &buffer[i])) 948 return -EFAULT; 949 i++; 950 } while (c == ','); 951 952 pkt_dev->nr_labels = n; 953 return i; 954 } 955 956 static __u32 pktgen_read_flag(const char *f, bool *disable) 957 { 958 __u32 i; 959 960 if (f[0] == '!') { 961 *disable = true; 962 f++; 963 } 964 965 for (i = 0; i < NR_PKT_FLAGS; i++) { 966 if (!IS_ENABLED(CONFIG_XFRM) && i == IPSEC_SHIFT) 967 continue; 968 969 /* allow only disabling ipv6 flag */ 970 if (!*disable && i == IPV6_SHIFT) 971 continue; 972 973 if (strcmp(f, pkt_flag_names[i]) == 0) 974 return 1 << i; 975 } 976 977 if (strcmp(f, "FLOW_RND") == 0) { 978 *disable = !*disable; 979 return F_FLOW_SEQ; 980 } 981 982 return 0; 983 } 984 985 static ssize_t pktgen_if_write(struct file *file, 986 const char __user *user_buffer, size_t count, 987 loff_t *offset) 988 { 989 struct seq_file *seq = file->private_data; 990 struct pktgen_dev *pkt_dev = seq->private; 991 size_t i, max; 992 ssize_t len; 993 char name[16], valstr[32]; 994 unsigned long value = 0; 995 char *pg_result = NULL; 996 char buf[128]; 997 998 pg_result = &(pkt_dev->result[0]); 999 1000 if (count < 1) { 1001 pr_warn("wrong command format\n"); 1002 return -EINVAL; 1003 } 1004 1005 max = count; 1006 len = count_trail_chars(user_buffer, max); 1007 if (len < 0) { 1008 pr_warn("illegal format\n"); 1009 return len; 1010 } 1011 i = len; 1012 1013 /* Read variable name */ 1014 max = min(sizeof(name) - 1, count - i); 1015 len = strn_len(&user_buffer[i], max); 1016 if (len < 0) 1017 return len; 1018 1019 memset(name, 0, sizeof(name)); 1020 if (copy_from_user(name, &user_buffer[i], len)) 1021 return -EFAULT; 1022 i += len; 1023 1024 max = count - i; 1025 len = count_trail_chars(&user_buffer[i], max); 1026 if (len < 0) 1027 return len; 1028 1029 i += len; 1030 1031 if (debug) { 1032 size_t copy = min_t(size_t, count + 1, 1024); 1033 char *tp = strndup_user(user_buffer, copy); 1034 1035 if (IS_ERR(tp)) 1036 return PTR_ERR(tp); 1037 1038 pr_debug("%s,%zu buffer -:%s:-\n", name, count, tp); 1039 kfree(tp); 1040 } 1041 1042 if (!strcmp(name, "min_pkt_size")) { 1043 max = min(10, count - i); 1044 len = num_arg(&user_buffer[i], max, &value); 1045 if (len < 0) 1046 return len; 1047 1048 if (value < 14 + 20 + 8) 1049 value = 14 + 20 + 8; 1050 if (value != pkt_dev->min_pkt_size) { 1051 pkt_dev->min_pkt_size = value; 1052 pkt_dev->cur_pkt_size = value; 1053 } 1054 sprintf(pg_result, "OK: min_pkt_size=%d", 1055 pkt_dev->min_pkt_size); 1056 return count; 1057 } 1058 1059 if (!strcmp(name, "max_pkt_size")) { 1060 max = min(10, count - i); 1061 len = num_arg(&user_buffer[i], max, &value); 1062 if (len < 0) 1063 return len; 1064 1065 if (value < 14 + 20 + 8) 1066 value = 14 + 20 + 8; 1067 if (value != pkt_dev->max_pkt_size) { 1068 pkt_dev->max_pkt_size = value; 1069 pkt_dev->cur_pkt_size = value; 1070 } 1071 sprintf(pg_result, "OK: max_pkt_size=%d", 1072 pkt_dev->max_pkt_size); 1073 return count; 1074 } 1075 1076 /* Shortcut for min = max */ 1077 1078 if (!strcmp(name, "pkt_size")) { 1079 max = min(10, count - i); 1080 len = num_arg(&user_buffer[i], max, &value); 1081 if (len < 0) 1082 return len; 1083 1084 if (value < 14 + 20 + 8) 1085 value = 14 + 20 + 8; 1086 if (value != pkt_dev->min_pkt_size) { 1087 pkt_dev->min_pkt_size = value; 1088 pkt_dev->max_pkt_size = value; 1089 pkt_dev->cur_pkt_size = value; 1090 } 1091 sprintf(pg_result, "OK: pkt_size=%d", pkt_dev->min_pkt_size); 1092 return count; 1093 } 1094 1095 if (!strcmp(name, "imix_weights")) { 1096 if (pkt_dev->clone_skb > 0) 1097 return -EINVAL; 1098 1099 max = count - i; 1100 len = get_imix_entries(&user_buffer[i], max, pkt_dev); 1101 if (len < 0) 1102 return len; 1103 1104 fill_imix_distribution(pkt_dev); 1105 1106 return count; 1107 } 1108 1109 if (!strcmp(name, "debug")) { 1110 max = min(10, count - i); 1111 len = num_arg(&user_buffer[i], max, &value); 1112 if (len < 0) 1113 return len; 1114 1115 debug = value; 1116 sprintf(pg_result, "OK: debug=%u", debug); 1117 return count; 1118 } 1119 1120 if (!strcmp(name, "frags")) { 1121 max = min(10, count - i); 1122 len = num_arg(&user_buffer[i], max, &value); 1123 if (len < 0) 1124 return len; 1125 1126 pkt_dev->nfrags = value; 1127 sprintf(pg_result, "OK: frags=%d", pkt_dev->nfrags); 1128 return count; 1129 } 1130 if (!strcmp(name, "delay")) { 1131 max = min(10, count - i); 1132 len = num_arg(&user_buffer[i], max, &value); 1133 if (len < 0) 1134 return len; 1135 1136 if (value == 0x7FFFFFFF) 1137 pkt_dev->delay = ULLONG_MAX; 1138 else 1139 pkt_dev->delay = (u64)value; 1140 1141 sprintf(pg_result, "OK: delay=%llu", 1142 (unsigned long long) pkt_dev->delay); 1143 return count; 1144 } 1145 if (!strcmp(name, "rate")) { 1146 max = min(10, count - i); 1147 len = num_arg(&user_buffer[i], max, &value); 1148 if (len < 0) 1149 return len; 1150 1151 if (!value) 1152 return -EINVAL; 1153 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value; 1154 if (debug) 1155 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1156 1157 sprintf(pg_result, "OK: rate=%lu", value); 1158 return count; 1159 } 1160 if (!strcmp(name, "ratep")) { 1161 max = min(10, count - i); 1162 len = num_arg(&user_buffer[i], max, &value); 1163 if (len < 0) 1164 return len; 1165 1166 if (!value) 1167 return -EINVAL; 1168 pkt_dev->delay = NSEC_PER_SEC/value; 1169 if (debug) 1170 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1171 1172 sprintf(pg_result, "OK: rate=%lu", value); 1173 return count; 1174 } 1175 if (!strcmp(name, "udp_src_min")) { 1176 max = min(10, count - i); 1177 len = num_arg(&user_buffer[i], max, &value); 1178 if (len < 0) 1179 return len; 1180 1181 if (value != pkt_dev->udp_src_min) { 1182 pkt_dev->udp_src_min = value; 1183 pkt_dev->cur_udp_src = value; 1184 } 1185 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min); 1186 return count; 1187 } 1188 if (!strcmp(name, "udp_dst_min")) { 1189 max = min(10, count - i); 1190 len = num_arg(&user_buffer[i], max, &value); 1191 if (len < 0) 1192 return len; 1193 1194 if (value != pkt_dev->udp_dst_min) { 1195 pkt_dev->udp_dst_min = value; 1196 pkt_dev->cur_udp_dst = value; 1197 } 1198 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min); 1199 return count; 1200 } 1201 if (!strcmp(name, "udp_src_max")) { 1202 max = min(10, count - i); 1203 len = num_arg(&user_buffer[i], max, &value); 1204 if (len < 0) 1205 return len; 1206 1207 if (value != pkt_dev->udp_src_max) { 1208 pkt_dev->udp_src_max = value; 1209 pkt_dev->cur_udp_src = value; 1210 } 1211 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max); 1212 return count; 1213 } 1214 if (!strcmp(name, "udp_dst_max")) { 1215 max = min(10, count - i); 1216 len = num_arg(&user_buffer[i], max, &value); 1217 if (len < 0) 1218 return len; 1219 1220 if (value != pkt_dev->udp_dst_max) { 1221 pkt_dev->udp_dst_max = value; 1222 pkt_dev->cur_udp_dst = value; 1223 } 1224 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max); 1225 return count; 1226 } 1227 if (!strcmp(name, "clone_skb")) { 1228 max = min(10, count - i); 1229 len = num_arg(&user_buffer[i], max, &value); 1230 if (len < 0) 1231 return len; 1232 /* clone_skb is not supported for netif_receive xmit_mode and 1233 * IMIX mode. 1234 */ 1235 if ((value > 0) && 1236 ((pkt_dev->xmit_mode == M_NETIF_RECEIVE) || 1237 !(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))) 1238 return -EOPNOTSUPP; 1239 if (value > 0 && (pkt_dev->n_imix_entries > 0 || 1240 !(pkt_dev->flags & F_SHARED))) 1241 return -EINVAL; 1242 1243 pkt_dev->clone_skb = value; 1244 1245 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb); 1246 return count; 1247 } 1248 if (!strcmp(name, "count")) { 1249 max = min(10, count - i); 1250 len = num_arg(&user_buffer[i], max, &value); 1251 if (len < 0) 1252 return len; 1253 1254 pkt_dev->count = value; 1255 sprintf(pg_result, "OK: count=%llu", 1256 (unsigned long long)pkt_dev->count); 1257 return count; 1258 } 1259 if (!strcmp(name, "src_mac_count")) { 1260 max = min(10, count - i); 1261 len = num_arg(&user_buffer[i], max, &value); 1262 if (len < 0) 1263 return len; 1264 1265 if (pkt_dev->src_mac_count != value) { 1266 pkt_dev->src_mac_count = value; 1267 pkt_dev->cur_src_mac_offset = 0; 1268 } 1269 sprintf(pg_result, "OK: src_mac_count=%d", 1270 pkt_dev->src_mac_count); 1271 return count; 1272 } 1273 if (!strcmp(name, "dst_mac_count")) { 1274 max = min(10, count - i); 1275 len = num_arg(&user_buffer[i], max, &value); 1276 if (len < 0) 1277 return len; 1278 1279 if (pkt_dev->dst_mac_count != value) { 1280 pkt_dev->dst_mac_count = value; 1281 pkt_dev->cur_dst_mac_offset = 0; 1282 } 1283 sprintf(pg_result, "OK: dst_mac_count=%d", 1284 pkt_dev->dst_mac_count); 1285 return count; 1286 } 1287 if (!strcmp(name, "burst")) { 1288 max = min(10, count - i); 1289 len = num_arg(&user_buffer[i], max, &value); 1290 if (len < 0) 1291 return len; 1292 1293 if ((value > 1) && 1294 ((pkt_dev->xmit_mode == M_QUEUE_XMIT) || 1295 ((pkt_dev->xmit_mode == M_START_XMIT) && 1296 (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))))) 1297 return -EOPNOTSUPP; 1298 1299 if (value > 1 && !(pkt_dev->flags & F_SHARED)) 1300 return -EINVAL; 1301 1302 pkt_dev->burst = value < 1 ? 1 : value; 1303 sprintf(pg_result, "OK: burst=%u", pkt_dev->burst); 1304 return count; 1305 } 1306 if (!strcmp(name, "node")) { 1307 max = min(10, count - i); 1308 len = num_arg(&user_buffer[i], max, &value); 1309 if (len < 0) 1310 return len; 1311 1312 if (node_possible(value)) { 1313 pkt_dev->node = value; 1314 sprintf(pg_result, "OK: node=%d", pkt_dev->node); 1315 if (pkt_dev->page) { 1316 put_page(pkt_dev->page); 1317 pkt_dev->page = NULL; 1318 } 1319 } else { 1320 sprintf(pg_result, "ERROR: node not possible"); 1321 } 1322 return count; 1323 } 1324 if (!strcmp(name, "xmit_mode")) { 1325 char f[32]; 1326 1327 max = min(sizeof(f) - 1, count - i); 1328 len = strn_len(&user_buffer[i], max); 1329 if (len < 0) 1330 return len; 1331 1332 memset(f, 0, sizeof(f)); 1333 if (copy_from_user(f, &user_buffer[i], len)) 1334 return -EFAULT; 1335 1336 if (strcmp(f, "start_xmit") == 0) { 1337 pkt_dev->xmit_mode = M_START_XMIT; 1338 } else if (strcmp(f, "netif_receive") == 0) { 1339 /* clone_skb set earlier, not supported in this mode */ 1340 if (pkt_dev->clone_skb > 0) 1341 return -EOPNOTSUPP; 1342 1343 pkt_dev->xmit_mode = M_NETIF_RECEIVE; 1344 1345 /* make sure new packet is allocated every time 1346 * pktgen_xmit() is called 1347 */ 1348 pkt_dev->last_ok = 1; 1349 } else if (strcmp(f, "queue_xmit") == 0) { 1350 pkt_dev->xmit_mode = M_QUEUE_XMIT; 1351 pkt_dev->last_ok = 1; 1352 } else { 1353 sprintf(pg_result, 1354 "xmit_mode -:%s:- unknown\nAvailable modes: %s", 1355 f, "start_xmit, netif_receive\n"); 1356 return count; 1357 } 1358 sprintf(pg_result, "OK: xmit_mode=%s", f); 1359 return count; 1360 } 1361 if (!strcmp(name, "flag")) { 1362 bool disable = false; 1363 __u32 flag; 1364 char f[32]; 1365 char *end; 1366 1367 max = min(sizeof(f) - 1, count - i); 1368 len = strn_len(&user_buffer[i], max); 1369 if (len < 0) 1370 return len; 1371 1372 memset(f, 0, 32); 1373 if (copy_from_user(f, &user_buffer[i], len)) 1374 return -EFAULT; 1375 1376 flag = pktgen_read_flag(f, &disable); 1377 if (flag) { 1378 if (disable) { 1379 /* If "clone_skb", or "burst" parameters are 1380 * configured, it means that the skb still 1381 * needs to be referenced by the pktgen, so 1382 * the skb must be shared. 1383 */ 1384 if (flag == F_SHARED && (pkt_dev->clone_skb || 1385 pkt_dev->burst > 1)) 1386 return -EINVAL; 1387 pkt_dev->flags &= ~flag; 1388 } else { 1389 pkt_dev->flags |= flag; 1390 } 1391 1392 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags); 1393 return count; 1394 } 1395 1396 /* Unknown flag */ 1397 end = pkt_dev->result + sizeof(pkt_dev->result); 1398 pg_result += sprintf(pg_result, 1399 "Flag -:%s:- unknown\n" 1400 "Available flags, (prepend ! to un-set flag):\n", f); 1401 1402 for (int n = 0; n < NR_PKT_FLAGS && pg_result < end; n++) { 1403 if (!IS_ENABLED(CONFIG_XFRM) && n == IPSEC_SHIFT) 1404 continue; 1405 pg_result += snprintf(pg_result, end - pg_result, 1406 "%s, ", pkt_flag_names[n]); 1407 } 1408 if (!WARN_ON_ONCE(pg_result >= end)) { 1409 /* Remove the comma and whitespace at the end */ 1410 *(pg_result - 2) = '\0'; 1411 } 1412 1413 return count; 1414 } 1415 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) { 1416 max = min(sizeof(pkt_dev->dst_min) - 1, count - i); 1417 len = strn_len(&user_buffer[i], max); 1418 if (len < 0) 1419 return len; 1420 1421 if (copy_from_user(buf, &user_buffer[i], len)) 1422 return -EFAULT; 1423 buf[len] = 0; 1424 if (strcmp(buf, pkt_dev->dst_min) != 0) { 1425 strscpy_pad(pkt_dev->dst_min, buf); 1426 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 1427 pkt_dev->cur_daddr = pkt_dev->daddr_min; 1428 } 1429 if (debug) 1430 pr_debug("dst_min set to: %s\n", pkt_dev->dst_min); 1431 1432 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min); 1433 return count; 1434 } 1435 if (!strcmp(name, "dst_max")) { 1436 max = min(sizeof(pkt_dev->dst_max) - 1, count - i); 1437 len = strn_len(&user_buffer[i], max); 1438 if (len < 0) 1439 return len; 1440 1441 if (copy_from_user(buf, &user_buffer[i], len)) 1442 return -EFAULT; 1443 buf[len] = 0; 1444 if (strcmp(buf, pkt_dev->dst_max) != 0) { 1445 strscpy_pad(pkt_dev->dst_max, buf); 1446 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 1447 pkt_dev->cur_daddr = pkt_dev->daddr_max; 1448 } 1449 if (debug) 1450 pr_debug("dst_max set to: %s\n", pkt_dev->dst_max); 1451 1452 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max); 1453 return count; 1454 } 1455 if (!strcmp(name, "dst6")) { 1456 max = min(sizeof(buf) - 1, count - i); 1457 len = strn_len(&user_buffer[i], max); 1458 if (len < 0) 1459 return len; 1460 1461 pkt_dev->flags |= F_IPV6; 1462 1463 if (copy_from_user(buf, &user_buffer[i], len)) 1464 return -EFAULT; 1465 buf[len] = 0; 1466 1467 in6_pton(buf, -1, pkt_dev->in6_daddr.s6_addr, -1, NULL); 1468 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr); 1469 1470 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr; 1471 1472 if (debug) 1473 pr_debug("dst6 set to: %s\n", buf); 1474 1475 sprintf(pg_result, "OK: dst6=%s", buf); 1476 return count; 1477 } 1478 if (!strcmp(name, "dst6_min")) { 1479 max = min(sizeof(buf) - 1, count - i); 1480 len = strn_len(&user_buffer[i], max); 1481 if (len < 0) 1482 return len; 1483 1484 pkt_dev->flags |= F_IPV6; 1485 1486 if (copy_from_user(buf, &user_buffer[i], len)) 1487 return -EFAULT; 1488 buf[len] = 0; 1489 1490 in6_pton(buf, -1, pkt_dev->min_in6_daddr.s6_addr, -1, NULL); 1491 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr); 1492 1493 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr; 1494 if (debug) 1495 pr_debug("dst6_min set to: %s\n", buf); 1496 1497 sprintf(pg_result, "OK: dst6_min=%s", buf); 1498 return count; 1499 } 1500 if (!strcmp(name, "dst6_max")) { 1501 max = min(sizeof(buf) - 1, count - i); 1502 len = strn_len(&user_buffer[i], max); 1503 if (len < 0) 1504 return len; 1505 1506 pkt_dev->flags |= F_IPV6; 1507 1508 if (copy_from_user(buf, &user_buffer[i], len)) 1509 return -EFAULT; 1510 buf[len] = 0; 1511 1512 in6_pton(buf, -1, pkt_dev->max_in6_daddr.s6_addr, -1, NULL); 1513 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr); 1514 1515 if (debug) 1516 pr_debug("dst6_max set to: %s\n", buf); 1517 1518 sprintf(pg_result, "OK: dst6_max=%s", buf); 1519 return count; 1520 } 1521 if (!strcmp(name, "src6")) { 1522 max = min(sizeof(buf) - 1, count - i); 1523 len = strn_len(&user_buffer[i], max); 1524 if (len < 0) 1525 return len; 1526 1527 pkt_dev->flags |= F_IPV6; 1528 1529 if (copy_from_user(buf, &user_buffer[i], len)) 1530 return -EFAULT; 1531 buf[len] = 0; 1532 1533 in6_pton(buf, -1, pkt_dev->in6_saddr.s6_addr, -1, NULL); 1534 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr); 1535 1536 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr; 1537 1538 if (debug) 1539 pr_debug("src6 set to: %s\n", buf); 1540 1541 sprintf(pg_result, "OK: src6=%s", buf); 1542 return count; 1543 } 1544 if (!strcmp(name, "src_min")) { 1545 max = min(sizeof(pkt_dev->src_min) - 1, count - i); 1546 len = strn_len(&user_buffer[i], max); 1547 if (len < 0) 1548 return len; 1549 1550 if (copy_from_user(buf, &user_buffer[i], len)) 1551 return -EFAULT; 1552 buf[len] = 0; 1553 if (strcmp(buf, pkt_dev->src_min) != 0) { 1554 strscpy_pad(pkt_dev->src_min, buf); 1555 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 1556 pkt_dev->cur_saddr = pkt_dev->saddr_min; 1557 } 1558 if (debug) 1559 pr_debug("src_min set to: %s\n", pkt_dev->src_min); 1560 1561 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min); 1562 return count; 1563 } 1564 if (!strcmp(name, "src_max")) { 1565 max = min(sizeof(pkt_dev->src_max) - 1, count - i); 1566 len = strn_len(&user_buffer[i], max); 1567 if (len < 0) 1568 return len; 1569 1570 if (copy_from_user(buf, &user_buffer[i], len)) 1571 return -EFAULT; 1572 buf[len] = 0; 1573 if (strcmp(buf, pkt_dev->src_max) != 0) { 1574 strscpy_pad(pkt_dev->src_max, buf); 1575 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 1576 pkt_dev->cur_saddr = pkt_dev->saddr_max; 1577 } 1578 if (debug) 1579 pr_debug("src_max set to: %s\n", pkt_dev->src_max); 1580 1581 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max); 1582 return count; 1583 } 1584 if (!strcmp(name, "dst_mac")) { 1585 max = min(sizeof(valstr) - 1, count - i); 1586 len = strn_len(&user_buffer[i], max); 1587 if (len < 0) 1588 return len; 1589 1590 memset(valstr, 0, sizeof(valstr)); 1591 if (copy_from_user(valstr, &user_buffer[i], len)) 1592 return -EFAULT; 1593 1594 if (!mac_pton(valstr, pkt_dev->dst_mac)) 1595 return -EINVAL; 1596 /* Set up Dest MAC */ 1597 ether_addr_copy(&pkt_dev->hh[0], pkt_dev->dst_mac); 1598 1599 sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac); 1600 return count; 1601 } 1602 if (!strcmp(name, "src_mac")) { 1603 max = min(sizeof(valstr) - 1, count - i); 1604 len = strn_len(&user_buffer[i], max); 1605 if (len < 0) 1606 return len; 1607 1608 memset(valstr, 0, sizeof(valstr)); 1609 if (copy_from_user(valstr, &user_buffer[i], len)) 1610 return -EFAULT; 1611 1612 if (!mac_pton(valstr, pkt_dev->src_mac)) 1613 return -EINVAL; 1614 /* Set up Src MAC */ 1615 ether_addr_copy(&pkt_dev->hh[6], pkt_dev->src_mac); 1616 1617 sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac); 1618 return count; 1619 } 1620 1621 if (!strcmp(name, "clear_counters")) { 1622 pktgen_clear_counters(pkt_dev); 1623 sprintf(pg_result, "OK: Clearing counters.\n"); 1624 return count; 1625 } 1626 1627 if (!strcmp(name, "flows")) { 1628 max = min(10, count - i); 1629 len = num_arg(&user_buffer[i], max, &value); 1630 if (len < 0) 1631 return len; 1632 1633 if (value > MAX_CFLOWS) 1634 value = MAX_CFLOWS; 1635 1636 WRITE_ONCE(pkt_dev->cflows, value); 1637 sprintf(pg_result, "OK: flows=%u", (unsigned int)value); 1638 return count; 1639 } 1640 #ifdef CONFIG_XFRM 1641 if (!strcmp(name, "spi")) { 1642 max = min(10, count - i); 1643 len = num_arg(&user_buffer[i], max, &value); 1644 if (len < 0) 1645 return len; 1646 1647 pkt_dev->spi = value; 1648 sprintf(pg_result, "OK: spi=%u", pkt_dev->spi); 1649 return count; 1650 } 1651 #endif 1652 if (!strcmp(name, "flowlen")) { 1653 max = min(10, count - i); 1654 len = num_arg(&user_buffer[i], max, &value); 1655 if (len < 0) 1656 return len; 1657 1658 pkt_dev->lflow = value; 1659 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow); 1660 return count; 1661 } 1662 1663 if (!strcmp(name, "queue_map_min")) { 1664 max = min(5, count - i); 1665 len = num_arg(&user_buffer[i], max, &value); 1666 if (len < 0) 1667 return len; 1668 1669 pkt_dev->queue_map_min = value; 1670 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min); 1671 return count; 1672 } 1673 1674 if (!strcmp(name, "queue_map_max")) { 1675 max = min(5, count - i); 1676 len = num_arg(&user_buffer[i], max, &value); 1677 if (len < 0) 1678 return len; 1679 1680 pkt_dev->queue_map_max = value; 1681 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max); 1682 return count; 1683 } 1684 1685 if (!strcmp(name, "mpls")) { 1686 unsigned int n, cnt; 1687 1688 max = count - i; 1689 len = get_labels(&user_buffer[i], max, pkt_dev); 1690 if (len < 0) 1691 return len; 1692 1693 cnt = sprintf(pg_result, "OK: mpls="); 1694 for (n = 0; n < pkt_dev->nr_labels; n++) 1695 cnt += sprintf(pg_result + cnt, 1696 "%08x%s", ntohl(pkt_dev->labels[n]), 1697 n == pkt_dev->nr_labels-1 ? "" : ","); 1698 1699 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) { 1700 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1701 pkt_dev->svlan_id = 0xffff; 1702 1703 if (debug) 1704 pr_debug("VLAN/SVLAN auto turned off\n"); 1705 } 1706 return count; 1707 } 1708 1709 if (!strcmp(name, "vlan_id")) { 1710 max = min(4, count - i); 1711 len = num_arg(&user_buffer[i], max, &value); 1712 if (len < 0) 1713 return len; 1714 1715 if (value <= 4095) { 1716 pkt_dev->vlan_id = value; /* turn on VLAN */ 1717 1718 if (debug) 1719 pr_debug("VLAN turned on\n"); 1720 1721 if (debug && pkt_dev->nr_labels) 1722 pr_debug("MPLS auto turned off\n"); 1723 1724 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1725 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id); 1726 } else { 1727 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1728 pkt_dev->svlan_id = 0xffff; 1729 1730 if (debug) 1731 pr_debug("VLAN/SVLAN turned off\n"); 1732 } 1733 return count; 1734 } 1735 1736 if (!strcmp(name, "vlan_p")) { 1737 max = min(1, count - i); 1738 len = num_arg(&user_buffer[i], max, &value); 1739 if (len < 0) 1740 return len; 1741 1742 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) { 1743 pkt_dev->vlan_p = value; 1744 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p); 1745 } else { 1746 sprintf(pg_result, "ERROR: vlan_p must be 0-7"); 1747 } 1748 return count; 1749 } 1750 1751 if (!strcmp(name, "vlan_cfi")) { 1752 max = min(1, count - i); 1753 len = num_arg(&user_buffer[i], max, &value); 1754 if (len < 0) 1755 return len; 1756 1757 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) { 1758 pkt_dev->vlan_cfi = value; 1759 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi); 1760 } else { 1761 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1"); 1762 } 1763 return count; 1764 } 1765 1766 if (!strcmp(name, "svlan_id")) { 1767 max = min(4, count - i); 1768 len = num_arg(&user_buffer[i], max, &value); 1769 if (len < 0) 1770 return len; 1771 1772 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) { 1773 pkt_dev->svlan_id = value; /* turn on SVLAN */ 1774 1775 if (debug) 1776 pr_debug("SVLAN turned on\n"); 1777 1778 if (debug && pkt_dev->nr_labels) 1779 pr_debug("MPLS auto turned off\n"); 1780 1781 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1782 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id); 1783 } else { 1784 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1785 pkt_dev->svlan_id = 0xffff; 1786 1787 if (debug) 1788 pr_debug("VLAN/SVLAN turned off\n"); 1789 } 1790 return count; 1791 } 1792 1793 if (!strcmp(name, "svlan_p")) { 1794 max = min(1, count - i); 1795 len = num_arg(&user_buffer[i], max, &value); 1796 if (len < 0) 1797 return len; 1798 1799 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) { 1800 pkt_dev->svlan_p = value; 1801 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p); 1802 } else { 1803 sprintf(pg_result, "ERROR: svlan_p must be 0-7"); 1804 } 1805 return count; 1806 } 1807 1808 if (!strcmp(name, "svlan_cfi")) { 1809 max = min(1, count - i); 1810 len = num_arg(&user_buffer[i], max, &value); 1811 if (len < 0) 1812 return len; 1813 1814 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) { 1815 pkt_dev->svlan_cfi = value; 1816 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi); 1817 } else { 1818 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1"); 1819 } 1820 return count; 1821 } 1822 1823 if (!strcmp(name, "tos")) { 1824 __u32 tmp_value; 1825 1826 max = min(2, count - i); 1827 len = hex32_arg(&user_buffer[i], max, &tmp_value); 1828 if (len < 0) 1829 return len; 1830 1831 if (len == 2) { 1832 pkt_dev->tos = tmp_value; 1833 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos); 1834 } else { 1835 sprintf(pg_result, "ERROR: tos must be 00-ff"); 1836 } 1837 return count; 1838 } 1839 1840 if (!strcmp(name, "traffic_class")) { 1841 __u32 tmp_value; 1842 1843 max = min(2, count - i); 1844 len = hex32_arg(&user_buffer[i], max, &tmp_value); 1845 if (len < 0) 1846 return len; 1847 1848 if (len == 2) { 1849 pkt_dev->traffic_class = tmp_value; 1850 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class); 1851 } else { 1852 sprintf(pg_result, "ERROR: traffic_class must be 00-ff"); 1853 } 1854 return count; 1855 } 1856 1857 if (!strcmp(name, "skb_priority")) { 1858 max = min(9, count - i); 1859 len = num_arg(&user_buffer[i], max, &value); 1860 if (len < 0) 1861 return len; 1862 1863 pkt_dev->skb_priority = value; 1864 sprintf(pg_result, "OK: skb_priority=%i", 1865 pkt_dev->skb_priority); 1866 return count; 1867 } 1868 1869 sprintf(pkt_dev->result, "No such parameter \"%s\"", name); 1870 return -EINVAL; 1871 } 1872 1873 static int pktgen_if_open(struct inode *inode, struct file *file) 1874 { 1875 return single_open(file, pktgen_if_show, pde_data(inode)); 1876 } 1877 1878 static const struct proc_ops pktgen_if_proc_ops = { 1879 .proc_open = pktgen_if_open, 1880 .proc_read = seq_read, 1881 .proc_lseek = seq_lseek, 1882 .proc_write = pktgen_if_write, 1883 .proc_release = single_release, 1884 }; 1885 1886 static int pktgen_thread_show(struct seq_file *seq, void *v) 1887 { 1888 struct pktgen_thread *t = seq->private; 1889 const struct pktgen_dev *pkt_dev; 1890 1891 BUG_ON(!t); 1892 1893 seq_puts(seq, "Running: "); 1894 1895 rcu_read_lock(); 1896 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 1897 if (pkt_dev->running) 1898 seq_printf(seq, "%s ", pkt_dev->odevname); 1899 1900 seq_puts(seq, "\nStopped: "); 1901 1902 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 1903 if (!pkt_dev->running) 1904 seq_printf(seq, "%s ", pkt_dev->odevname); 1905 1906 if (t->result[0]) 1907 seq_printf(seq, "\nResult: %s\n", t->result); 1908 else 1909 seq_puts(seq, "\nResult: NA\n"); 1910 1911 rcu_read_unlock(); 1912 1913 return 0; 1914 } 1915 1916 static ssize_t pktgen_thread_write(struct file *file, 1917 const char __user *user_buffer, 1918 size_t count, loff_t *offset) 1919 { 1920 struct seq_file *seq = file->private_data; 1921 struct pktgen_thread *t = seq->private; 1922 size_t i, max; 1923 ssize_t len, ret; 1924 char name[40]; 1925 char *pg_result; 1926 1927 if (count < 1) { 1928 // sprintf(pg_result, "Wrong command format"); 1929 return -EINVAL; 1930 } 1931 1932 max = count; 1933 len = count_trail_chars(user_buffer, max); 1934 if (len < 0) 1935 return len; 1936 1937 i = len; 1938 1939 /* Read variable name */ 1940 max = min(sizeof(name) - 1, count - i); 1941 len = strn_len(&user_buffer[i], max); 1942 if (len < 0) 1943 return len; 1944 1945 memset(name, 0, sizeof(name)); 1946 if (copy_from_user(name, &user_buffer[i], len)) 1947 return -EFAULT; 1948 i += len; 1949 1950 max = count - i; 1951 len = count_trail_chars(&user_buffer[i], max); 1952 if (len < 0) 1953 return len; 1954 1955 i += len; 1956 1957 if (debug) 1958 pr_debug("t=%s, count=%lu\n", name, (unsigned long)count); 1959 1960 if (!t) { 1961 pr_err("ERROR: No thread\n"); 1962 ret = -EINVAL; 1963 goto out; 1964 } 1965 1966 pg_result = &(t->result[0]); 1967 1968 if (!strcmp(name, "add_device")) { 1969 char f[32]; 1970 1971 memset(f, 0, 32); 1972 max = min(sizeof(f) - 1, count - i); 1973 len = strn_len(&user_buffer[i], max); 1974 if (len < 0) { 1975 ret = len; 1976 goto out; 1977 } 1978 if (copy_from_user(f, &user_buffer[i], len)) 1979 return -EFAULT; 1980 1981 mutex_lock(&pktgen_thread_lock); 1982 ret = pktgen_add_device(t, f); 1983 mutex_unlock(&pktgen_thread_lock); 1984 if (!ret) { 1985 ret = count; 1986 sprintf(pg_result, "OK: add_device=%s", f); 1987 } else 1988 sprintf(pg_result, "ERROR: can not add device %s", f); 1989 goto out; 1990 } 1991 1992 if (!strcmp(name, "rem_device_all")) { 1993 mutex_lock(&pktgen_thread_lock); 1994 t->control |= T_REMDEVALL; 1995 mutex_unlock(&pktgen_thread_lock); 1996 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */ 1997 ret = count; 1998 sprintf(pg_result, "OK: rem_device_all"); 1999 goto out; 2000 } 2001 2002 if (!strcmp(name, "max_before_softirq")) { 2003 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use"); 2004 ret = count; 2005 goto out; 2006 } 2007 2008 ret = -EINVAL; 2009 out: 2010 return ret; 2011 } 2012 2013 static int pktgen_thread_open(struct inode *inode, struct file *file) 2014 { 2015 return single_open(file, pktgen_thread_show, pde_data(inode)); 2016 } 2017 2018 static const struct proc_ops pktgen_thread_proc_ops = { 2019 .proc_open = pktgen_thread_open, 2020 .proc_read = seq_read, 2021 .proc_lseek = seq_lseek, 2022 .proc_write = pktgen_thread_write, 2023 .proc_release = single_release, 2024 }; 2025 2026 /* Think find or remove for NN */ 2027 static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn, 2028 const char *ifname, int remove) 2029 { 2030 struct pktgen_thread *t; 2031 struct pktgen_dev *pkt_dev = NULL; 2032 bool exact = (remove == FIND); 2033 2034 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 2035 pkt_dev = pktgen_find_dev(t, ifname, exact); 2036 if (pkt_dev) { 2037 if (remove) { 2038 pkt_dev->removal_mark = 1; 2039 t->control |= T_REMDEV; 2040 } 2041 break; 2042 } 2043 } 2044 return pkt_dev; 2045 } 2046 2047 /* 2048 * mark a device for removal 2049 */ 2050 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname) 2051 { 2052 struct pktgen_dev *pkt_dev = NULL; 2053 const int max_tries = 10, msec_per_try = 125; 2054 int i = 0; 2055 2056 mutex_lock(&pktgen_thread_lock); 2057 pr_debug("%s: marking %s for removal\n", __func__, ifname); 2058 2059 while (1) { 2060 2061 pkt_dev = __pktgen_NN_threads(pn, ifname, REMOVE); 2062 if (pkt_dev == NULL) 2063 break; /* success */ 2064 2065 mutex_unlock(&pktgen_thread_lock); 2066 pr_debug("%s: waiting for %s to disappear....\n", 2067 __func__, ifname); 2068 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try)); 2069 mutex_lock(&pktgen_thread_lock); 2070 2071 if (++i >= max_tries) { 2072 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n", 2073 __func__, msec_per_try * i, ifname); 2074 break; 2075 } 2076 2077 } 2078 2079 mutex_unlock(&pktgen_thread_lock); 2080 } 2081 2082 static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *dev) 2083 { 2084 struct pktgen_thread *t; 2085 2086 mutex_lock(&pktgen_thread_lock); 2087 2088 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 2089 struct pktgen_dev *pkt_dev; 2090 2091 if_lock(t); 2092 list_for_each_entry(pkt_dev, &t->if_list, list) { 2093 if (pkt_dev->odev != dev) 2094 continue; 2095 2096 proc_remove(pkt_dev->entry); 2097 2098 pkt_dev->entry = proc_create_data(dev->name, 0600, 2099 pn->proc_dir, 2100 &pktgen_if_proc_ops, 2101 pkt_dev); 2102 if (!pkt_dev->entry) 2103 pr_err("can't move proc entry for '%s'\n", 2104 dev->name); 2105 break; 2106 } 2107 if_unlock(t); 2108 } 2109 mutex_unlock(&pktgen_thread_lock); 2110 } 2111 2112 static int pktgen_device_event(struct notifier_block *unused, 2113 unsigned long event, void *ptr) 2114 { 2115 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 2116 struct pktgen_net *pn = net_generic(dev_net(dev), pg_net_id); 2117 2118 if (pn->pktgen_exiting) 2119 return NOTIFY_DONE; 2120 2121 /* It is OK that we do not hold the group lock right now, 2122 * as we run under the RTNL lock. 2123 */ 2124 2125 switch (event) { 2126 case NETDEV_CHANGENAME: 2127 pktgen_change_name(pn, dev); 2128 break; 2129 2130 case NETDEV_UNREGISTER: 2131 pktgen_mark_device(pn, dev->name); 2132 break; 2133 } 2134 2135 return NOTIFY_DONE; 2136 } 2137 2138 static struct net_device *pktgen_dev_get_by_name(const struct pktgen_net *pn, 2139 struct pktgen_dev *pkt_dev, 2140 const char *ifname) 2141 { 2142 char b[IFNAMSIZ+5]; 2143 int i; 2144 2145 for (i = 0; ifname[i] != '@'; i++) { 2146 if (i == IFNAMSIZ) 2147 break; 2148 2149 b[i] = ifname[i]; 2150 } 2151 b[i] = 0; 2152 2153 return dev_get_by_name(pn->net, b); 2154 } 2155 2156 2157 /* Associate pktgen_dev with a device. */ 2158 2159 static int pktgen_setup_dev(const struct pktgen_net *pn, 2160 struct pktgen_dev *pkt_dev, const char *ifname) 2161 { 2162 struct net_device *odev; 2163 int err; 2164 2165 /* Clean old setups */ 2166 if (pkt_dev->odev) { 2167 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); 2168 pkt_dev->odev = NULL; 2169 } 2170 2171 odev = pktgen_dev_get_by_name(pn, pkt_dev, ifname); 2172 if (!odev) { 2173 pr_err("no such netdevice: \"%s\"\n", ifname); 2174 return -ENODEV; 2175 } 2176 2177 if (odev->type != ARPHRD_ETHER && odev->type != ARPHRD_LOOPBACK) { 2178 pr_err("not an ethernet or loopback device: \"%s\"\n", ifname); 2179 err = -EINVAL; 2180 } else if (!netif_running(odev)) { 2181 pr_err("device is down: \"%s\"\n", ifname); 2182 err = -ENETDOWN; 2183 } else { 2184 pkt_dev->odev = odev; 2185 netdev_tracker_alloc(odev, &pkt_dev->dev_tracker, GFP_KERNEL); 2186 return 0; 2187 } 2188 2189 dev_put(odev); 2190 return err; 2191 } 2192 2193 /* Read pkt_dev from the interface and set up internal pktgen_dev 2194 * structure to have the right information to create/send packets 2195 */ 2196 static void pktgen_setup_inject(struct pktgen_dev *pkt_dev) 2197 { 2198 int ntxq; 2199 2200 if (!pkt_dev->odev) { 2201 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n"); 2202 sprintf(pkt_dev->result, 2203 "ERROR: pkt_dev->odev == NULL in setup_inject.\n"); 2204 return; 2205 } 2206 2207 /* make sure that we don't pick a non-existing transmit queue */ 2208 ntxq = pkt_dev->odev->real_num_tx_queues; 2209 2210 if (ntxq <= pkt_dev->queue_map_min) { 2211 pr_warn("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2212 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq, 2213 pkt_dev->odevname); 2214 pkt_dev->queue_map_min = (ntxq ?: 1) - 1; 2215 } 2216 if (pkt_dev->queue_map_max >= ntxq) { 2217 pr_warn("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2218 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq, 2219 pkt_dev->odevname); 2220 pkt_dev->queue_map_max = (ntxq ?: 1) - 1; 2221 } 2222 2223 /* Default to the interface's mac if not explicitly set. */ 2224 2225 if (is_zero_ether_addr(pkt_dev->src_mac)) 2226 ether_addr_copy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr); 2227 2228 /* Set up Dest MAC */ 2229 ether_addr_copy(&(pkt_dev->hh[0]), pkt_dev->dst_mac); 2230 2231 if (pkt_dev->flags & F_IPV6) { 2232 int i, set = 0, err = 1; 2233 struct inet6_dev *idev; 2234 2235 if (pkt_dev->min_pkt_size == 0) { 2236 pkt_dev->min_pkt_size = 14 + sizeof(struct ipv6hdr) 2237 + sizeof(struct udphdr) 2238 + sizeof(struct pktgen_hdr) 2239 + pkt_dev->pkt_overhead; 2240 } 2241 2242 for (i = 0; i < sizeof(struct in6_addr); i++) 2243 if (pkt_dev->cur_in6_saddr.s6_addr[i]) { 2244 set = 1; 2245 break; 2246 } 2247 2248 if (!set) { 2249 2250 /* 2251 * Use linklevel address if unconfigured. 2252 * 2253 * use ipv6_get_lladdr if/when it's get exported 2254 */ 2255 2256 rcu_read_lock(); 2257 idev = __in6_dev_get(pkt_dev->odev); 2258 if (idev) { 2259 struct inet6_ifaddr *ifp; 2260 2261 read_lock_bh(&idev->lock); 2262 list_for_each_entry(ifp, &idev->addr_list, if_list) { 2263 if ((ifp->scope & IFA_LINK) && 2264 !(ifp->flags & IFA_F_TENTATIVE)) { 2265 pkt_dev->cur_in6_saddr = ifp->addr; 2266 err = 0; 2267 break; 2268 } 2269 } 2270 read_unlock_bh(&idev->lock); 2271 } 2272 rcu_read_unlock(); 2273 if (err) 2274 pr_err("ERROR: IPv6 link address not available\n"); 2275 } 2276 } else { 2277 if (pkt_dev->min_pkt_size == 0) { 2278 pkt_dev->min_pkt_size = 14 + sizeof(struct iphdr) 2279 + sizeof(struct udphdr) 2280 + sizeof(struct pktgen_hdr) 2281 + pkt_dev->pkt_overhead; 2282 } 2283 2284 pkt_dev->saddr_min = 0; 2285 pkt_dev->saddr_max = 0; 2286 if (strlen(pkt_dev->src_min) == 0) { 2287 2288 struct in_device *in_dev; 2289 2290 rcu_read_lock(); 2291 in_dev = __in_dev_get_rcu(pkt_dev->odev); 2292 if (in_dev) { 2293 const struct in_ifaddr *ifa; 2294 2295 ifa = rcu_dereference(in_dev->ifa_list); 2296 if (ifa) { 2297 pkt_dev->saddr_min = ifa->ifa_address; 2298 pkt_dev->saddr_max = pkt_dev->saddr_min; 2299 } 2300 } 2301 rcu_read_unlock(); 2302 } else { 2303 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 2304 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 2305 } 2306 2307 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 2308 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 2309 } 2310 /* Initialize current values. */ 2311 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size; 2312 if (pkt_dev->min_pkt_size > pkt_dev->max_pkt_size) 2313 pkt_dev->max_pkt_size = pkt_dev->min_pkt_size; 2314 2315 pkt_dev->cur_dst_mac_offset = 0; 2316 pkt_dev->cur_src_mac_offset = 0; 2317 pkt_dev->cur_saddr = pkt_dev->saddr_min; 2318 pkt_dev->cur_daddr = pkt_dev->daddr_min; 2319 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2320 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2321 pkt_dev->nflows = 0; 2322 } 2323 2324 2325 static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) 2326 { 2327 ktime_t start_time, end_time; 2328 s64 remaining; 2329 struct hrtimer_sleeper t; 2330 2331 hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 2332 hrtimer_set_expires(&t.timer, spin_until); 2333 2334 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer)); 2335 if (remaining <= 0) 2336 goto out; 2337 2338 start_time = ktime_get(); 2339 if (remaining < 100000) { 2340 /* for small delays (<100us), just loop until limit is reached */ 2341 do { 2342 end_time = ktime_get(); 2343 } while (ktime_compare(end_time, spin_until) < 0); 2344 } else { 2345 do { 2346 set_current_state(TASK_INTERRUPTIBLE); 2347 hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS); 2348 2349 if (likely(t.task)) 2350 schedule(); 2351 2352 hrtimer_cancel(&t.timer); 2353 } while (t.task && pkt_dev->running && !signal_pending(current)); 2354 __set_current_state(TASK_RUNNING); 2355 end_time = ktime_get(); 2356 } 2357 2358 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time)); 2359 out: 2360 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay); 2361 destroy_hrtimer_on_stack(&t.timer); 2362 } 2363 2364 static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev) 2365 { 2366 pkt_dev->pkt_overhead = 0; 2367 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32); 2368 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev); 2369 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev); 2370 } 2371 2372 static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow) 2373 { 2374 return !!(pkt_dev->flows[flow].flags & F_INIT); 2375 } 2376 2377 static inline int f_pick(struct pktgen_dev *pkt_dev, unsigned int cflows) 2378 { 2379 int flow = pkt_dev->curfl; 2380 2381 if (pkt_dev->flags & F_FLOW_SEQ) { 2382 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) { 2383 /* reset time */ 2384 pkt_dev->flows[flow].count = 0; 2385 pkt_dev->flows[flow].flags = 0; 2386 pkt_dev->curfl += 1; 2387 if (pkt_dev->curfl >= cflows) 2388 pkt_dev->curfl = 0; /*reset */ 2389 } 2390 } else { 2391 flow = get_random_u32_below(cflows); 2392 pkt_dev->curfl = flow; 2393 2394 if (pkt_dev->flows[flow].count > pkt_dev->lflow) { 2395 pkt_dev->flows[flow].count = 0; 2396 pkt_dev->flows[flow].flags = 0; 2397 } 2398 } 2399 2400 return pkt_dev->curfl; 2401 } 2402 2403 2404 /* If there was already an IPSEC SA, we keep it as is, else 2405 * we go look for it ... 2406 */ 2407 #define DUMMY_MARK 0 2408 static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow) 2409 { 2410 #ifdef CONFIG_XFRM 2411 struct xfrm_state *x = pkt_dev->flows[flow].x; 2412 struct pktgen_net *pn = net_generic(dev_net(pkt_dev->odev), pg_net_id); 2413 2414 if (!x) { 2415 2416 if (pkt_dev->spi) { 2417 /* We need as quick as possible to find the right SA 2418 * Searching with minimum criteria to achieve, this. 2419 */ 2420 x = xfrm_state_lookup_byspi(pn->net, htonl(pkt_dev->spi), AF_INET); 2421 } else { 2422 /* slow path: we don't already have xfrm_state */ 2423 x = xfrm_stateonly_find(pn->net, DUMMY_MARK, 0, 2424 (xfrm_address_t *)&pkt_dev->cur_daddr, 2425 (xfrm_address_t *)&pkt_dev->cur_saddr, 2426 AF_INET, 2427 pkt_dev->ipsmode, 2428 pkt_dev->ipsproto, 0); 2429 } 2430 if (x) { 2431 pkt_dev->flows[flow].x = x; 2432 set_pkt_overhead(pkt_dev); 2433 pkt_dev->pkt_overhead += x->props.header_len; 2434 } 2435 2436 } 2437 #endif 2438 } 2439 static void set_cur_queue_map(struct pktgen_dev *pkt_dev) 2440 { 2441 if (pkt_dev->flags & F_QUEUE_MAP_CPU) 2442 pkt_dev->cur_queue_map = smp_processor_id(); 2443 2444 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) { 2445 __u16 t; 2446 2447 if (pkt_dev->flags & F_QUEUE_MAP_RND) { 2448 t = get_random_u32_inclusive(pkt_dev->queue_map_min, 2449 pkt_dev->queue_map_max); 2450 } else { 2451 t = pkt_dev->cur_queue_map + 1; 2452 if (t > pkt_dev->queue_map_max) 2453 t = pkt_dev->queue_map_min; 2454 } 2455 pkt_dev->cur_queue_map = t; 2456 } 2457 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues; 2458 } 2459 2460 /* Increment/randomize headers according to flags and current values 2461 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst 2462 */ 2463 static void mod_cur_headers(struct pktgen_dev *pkt_dev) 2464 { 2465 unsigned int cflows; 2466 __u32 imn; 2467 __u32 imx; 2468 int flow = 0; 2469 2470 cflows = READ_ONCE(pkt_dev->cflows); 2471 if (cflows) 2472 flow = f_pick(pkt_dev, cflows); 2473 2474 /* Deal with source MAC */ 2475 if (pkt_dev->src_mac_count > 1) { 2476 __u32 mc; 2477 __u32 tmp; 2478 2479 if (pkt_dev->flags & F_MACSRC_RND) 2480 mc = get_random_u32_below(pkt_dev->src_mac_count); 2481 else { 2482 mc = pkt_dev->cur_src_mac_offset++; 2483 if (pkt_dev->cur_src_mac_offset >= 2484 pkt_dev->src_mac_count) 2485 pkt_dev->cur_src_mac_offset = 0; 2486 } 2487 2488 tmp = pkt_dev->src_mac[5] + (mc & 0xFF); 2489 pkt_dev->hh[11] = tmp; 2490 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2491 pkt_dev->hh[10] = tmp; 2492 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2493 pkt_dev->hh[9] = tmp; 2494 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2495 pkt_dev->hh[8] = tmp; 2496 tmp = (pkt_dev->src_mac[1] + (tmp >> 8)); 2497 pkt_dev->hh[7] = tmp; 2498 } 2499 2500 /* Deal with Destination MAC */ 2501 if (pkt_dev->dst_mac_count > 1) { 2502 __u32 mc; 2503 __u32 tmp; 2504 2505 if (pkt_dev->flags & F_MACDST_RND) 2506 mc = get_random_u32_below(pkt_dev->dst_mac_count); 2507 2508 else { 2509 mc = pkt_dev->cur_dst_mac_offset++; 2510 if (pkt_dev->cur_dst_mac_offset >= 2511 pkt_dev->dst_mac_count) { 2512 pkt_dev->cur_dst_mac_offset = 0; 2513 } 2514 } 2515 2516 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF); 2517 pkt_dev->hh[5] = tmp; 2518 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2519 pkt_dev->hh[4] = tmp; 2520 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2521 pkt_dev->hh[3] = tmp; 2522 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2523 pkt_dev->hh[2] = tmp; 2524 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8)); 2525 pkt_dev->hh[1] = tmp; 2526 } 2527 2528 if (pkt_dev->flags & F_MPLS_RND) { 2529 unsigned int i; 2530 2531 for (i = 0; i < pkt_dev->nr_labels; i++) 2532 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM) 2533 pkt_dev->labels[i] = MPLS_STACK_BOTTOM | 2534 ((__force __be32)get_random_u32() & 2535 htonl(0x000fffff)); 2536 } 2537 2538 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) { 2539 pkt_dev->vlan_id = get_random_u32_below(4096); 2540 } 2541 2542 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) { 2543 pkt_dev->svlan_id = get_random_u32_below(4096); 2544 } 2545 2546 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) { 2547 if (pkt_dev->flags & F_UDPSRC_RND) 2548 pkt_dev->cur_udp_src = get_random_u32_inclusive(pkt_dev->udp_src_min, 2549 pkt_dev->udp_src_max - 1); 2550 2551 else { 2552 pkt_dev->cur_udp_src++; 2553 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max) 2554 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2555 } 2556 } 2557 2558 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) { 2559 if (pkt_dev->flags & F_UDPDST_RND) { 2560 pkt_dev->cur_udp_dst = get_random_u32_inclusive(pkt_dev->udp_dst_min, 2561 pkt_dev->udp_dst_max - 1); 2562 } else { 2563 pkt_dev->cur_udp_dst++; 2564 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max) 2565 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2566 } 2567 } 2568 2569 if (!(pkt_dev->flags & F_IPV6)) { 2570 2571 imn = ntohl(pkt_dev->saddr_min); 2572 imx = ntohl(pkt_dev->saddr_max); 2573 if (imn < imx) { 2574 __u32 t; 2575 2576 if (pkt_dev->flags & F_IPSRC_RND) 2577 t = get_random_u32_inclusive(imn, imx - 1); 2578 else { 2579 t = ntohl(pkt_dev->cur_saddr); 2580 t++; 2581 if (t > imx) 2582 t = imn; 2583 2584 } 2585 pkt_dev->cur_saddr = htonl(t); 2586 } 2587 2588 if (cflows && f_seen(pkt_dev, flow)) { 2589 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr; 2590 } else { 2591 imn = ntohl(pkt_dev->daddr_min); 2592 imx = ntohl(pkt_dev->daddr_max); 2593 if (imn < imx) { 2594 __u32 t; 2595 __be32 s; 2596 2597 if (pkt_dev->flags & F_IPDST_RND) { 2598 2599 do { 2600 t = get_random_u32_inclusive(imn, imx - 1); 2601 s = htonl(t); 2602 } while (ipv4_is_loopback(s) || 2603 ipv4_is_multicast(s) || 2604 ipv4_is_lbcast(s) || 2605 ipv4_is_zeronet(s) || 2606 ipv4_is_local_multicast(s)); 2607 pkt_dev->cur_daddr = s; 2608 } else { 2609 t = ntohl(pkt_dev->cur_daddr); 2610 t++; 2611 if (t > imx) { 2612 t = imn; 2613 } 2614 pkt_dev->cur_daddr = htonl(t); 2615 } 2616 } 2617 if (cflows) { 2618 pkt_dev->flows[flow].flags |= F_INIT; 2619 pkt_dev->flows[flow].cur_daddr = 2620 pkt_dev->cur_daddr; 2621 if (pkt_dev->flags & F_IPSEC) 2622 get_ipsec_sa(pkt_dev, flow); 2623 pkt_dev->nflows++; 2624 } 2625 } 2626 } else { /* IPV6 * */ 2627 2628 if (!ipv6_addr_any(&pkt_dev->min_in6_daddr)) { 2629 int i; 2630 2631 /* Only random destinations yet */ 2632 2633 for (i = 0; i < 4; i++) { 2634 pkt_dev->cur_in6_daddr.s6_addr32[i] = 2635 (((__force __be32)get_random_u32() | 2636 pkt_dev->min_in6_daddr.s6_addr32[i]) & 2637 pkt_dev->max_in6_daddr.s6_addr32[i]); 2638 } 2639 } 2640 } 2641 2642 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) { 2643 __u32 t; 2644 2645 if (pkt_dev->flags & F_TXSIZE_RND) { 2646 t = get_random_u32_inclusive(pkt_dev->min_pkt_size, 2647 pkt_dev->max_pkt_size - 1); 2648 } else { 2649 t = pkt_dev->cur_pkt_size + 1; 2650 if (t > pkt_dev->max_pkt_size) 2651 t = pkt_dev->min_pkt_size; 2652 } 2653 pkt_dev->cur_pkt_size = t; 2654 } else if (pkt_dev->n_imix_entries > 0) { 2655 struct imix_pkt *entry; 2656 __u32 t = get_random_u32_below(IMIX_PRECISION); 2657 __u8 entry_index = pkt_dev->imix_distribution[t]; 2658 2659 entry = &pkt_dev->imix_entries[entry_index]; 2660 entry->count_so_far++; 2661 pkt_dev->cur_pkt_size = entry->size; 2662 } 2663 2664 set_cur_queue_map(pkt_dev); 2665 2666 pkt_dev->flows[flow].count++; 2667 } 2668 2669 static void fill_imix_distribution(struct pktgen_dev *pkt_dev) 2670 { 2671 int cumulative_probabilites[MAX_IMIX_ENTRIES]; 2672 int j = 0; 2673 __u64 cumulative_prob = 0; 2674 __u64 total_weight = 0; 2675 int i = 0; 2676 2677 for (i = 0; i < pkt_dev->n_imix_entries; i++) 2678 total_weight += pkt_dev->imix_entries[i].weight; 2679 2680 /* Fill cumulative_probabilites with sum of normalized probabilities */ 2681 for (i = 0; i < pkt_dev->n_imix_entries - 1; i++) { 2682 cumulative_prob += div64_u64(pkt_dev->imix_entries[i].weight * 2683 IMIX_PRECISION, 2684 total_weight); 2685 cumulative_probabilites[i] = cumulative_prob; 2686 } 2687 cumulative_probabilites[pkt_dev->n_imix_entries - 1] = 100; 2688 2689 for (i = 0; i < IMIX_PRECISION; i++) { 2690 if (i == cumulative_probabilites[j]) 2691 j++; 2692 pkt_dev->imix_distribution[i] = j; 2693 } 2694 } 2695 2696 #ifdef CONFIG_XFRM 2697 static u32 pktgen_dst_metrics[RTAX_MAX + 1] = { 2698 2699 [RTAX_HOPLIMIT] = 0x5, /* Set a static hoplimit */ 2700 }; 2701 2702 static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev) 2703 { 2704 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2705 int err = 0; 2706 struct net *net = dev_net(pkt_dev->odev); 2707 2708 if (!x) 2709 return 0; 2710 /* XXX: we dont support tunnel mode for now until 2711 * we resolve the dst issue 2712 */ 2713 if ((x->props.mode != XFRM_MODE_TRANSPORT) && (pkt_dev->spi == 0)) 2714 return 0; 2715 2716 /* But when user specify an valid SPI, transformation 2717 * supports both transport/tunnel mode + ESP/AH type. 2718 */ 2719 if ((x->props.mode == XFRM_MODE_TUNNEL) && (pkt_dev->spi != 0)) 2720 skb->_skb_refdst = (unsigned long)&pkt_dev->xdst.u.dst | SKB_DST_NOREF; 2721 2722 rcu_read_lock_bh(); 2723 err = pktgen_xfrm_outer_mode_output(x, skb); 2724 rcu_read_unlock_bh(); 2725 if (err) { 2726 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR); 2727 goto error; 2728 } 2729 err = x->type->output(x, skb); 2730 if (err) { 2731 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR); 2732 goto error; 2733 } 2734 spin_lock_bh(&x->lock); 2735 x->curlft.bytes += skb->len; 2736 x->curlft.packets++; 2737 spin_unlock_bh(&x->lock); 2738 error: 2739 return err; 2740 } 2741 2742 static void free_SAs(struct pktgen_dev *pkt_dev) 2743 { 2744 if (pkt_dev->cflows) { 2745 /* let go of the SAs if we have them */ 2746 int i; 2747 2748 for (i = 0; i < pkt_dev->cflows; i++) { 2749 struct xfrm_state *x = pkt_dev->flows[i].x; 2750 2751 if (x) { 2752 xfrm_state_put(x); 2753 pkt_dev->flows[i].x = NULL; 2754 } 2755 } 2756 } 2757 } 2758 2759 static int process_ipsec(struct pktgen_dev *pkt_dev, 2760 struct sk_buff *skb, __be16 protocol) 2761 { 2762 if (pkt_dev->flags & F_IPSEC) { 2763 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2764 int nhead = 0; 2765 2766 if (x) { 2767 struct ethhdr *eth; 2768 struct iphdr *iph; 2769 int ret; 2770 2771 nhead = x->props.header_len - skb_headroom(skb); 2772 if (nhead > 0) { 2773 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC); 2774 if (ret < 0) { 2775 pr_err("Error expanding ipsec packet %d\n", 2776 ret); 2777 goto err; 2778 } 2779 } 2780 2781 /* ipsec is not expecting ll header */ 2782 skb_pull(skb, ETH_HLEN); 2783 ret = pktgen_output_ipsec(skb, pkt_dev); 2784 if (ret) { 2785 pr_err("Error creating ipsec packet %d\n", ret); 2786 goto err; 2787 } 2788 /* restore ll */ 2789 eth = skb_push(skb, ETH_HLEN); 2790 memcpy(eth, pkt_dev->hh, 2 * ETH_ALEN); 2791 eth->h_proto = protocol; 2792 2793 /* Update IPv4 header len as well as checksum value */ 2794 iph = ip_hdr(skb); 2795 iph->tot_len = htons(skb->len - ETH_HLEN); 2796 ip_send_check(iph); 2797 } 2798 } 2799 return 1; 2800 err: 2801 kfree_skb(skb); 2802 return 0; 2803 } 2804 #endif 2805 2806 static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev) 2807 { 2808 unsigned int i; 2809 2810 for (i = 0; i < pkt_dev->nr_labels; i++) 2811 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM; 2812 2813 mpls--; 2814 *mpls |= MPLS_STACK_BOTTOM; 2815 } 2816 2817 static inline __be16 build_tci(unsigned int id, unsigned int cfi, 2818 unsigned int prio) 2819 { 2820 return htons(id | (cfi << 12) | (prio << 13)); 2821 } 2822 2823 static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb, 2824 int datalen) 2825 { 2826 struct timespec64 timestamp; 2827 struct pktgen_hdr *pgh; 2828 2829 pgh = skb_put(skb, sizeof(*pgh)); 2830 datalen -= sizeof(*pgh); 2831 2832 if (pkt_dev->nfrags <= 0) { 2833 skb_put_zero(skb, datalen); 2834 } else { 2835 int frags = pkt_dev->nfrags; 2836 int i, len; 2837 int frag_len; 2838 2839 2840 if (frags > MAX_SKB_FRAGS) 2841 frags = MAX_SKB_FRAGS; 2842 len = datalen - frags * PAGE_SIZE; 2843 if (len > 0) { 2844 skb_put_zero(skb, len); 2845 datalen = frags * PAGE_SIZE; 2846 } 2847 2848 i = 0; 2849 frag_len = min_t(int, datalen / frags, PAGE_SIZE); 2850 while (datalen > 0) { 2851 if (unlikely(!pkt_dev->page)) { 2852 int node = numa_node_id(); 2853 2854 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE)) 2855 node = pkt_dev->node; 2856 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); 2857 if (!pkt_dev->page) 2858 break; 2859 } 2860 get_page(pkt_dev->page); 2861 2862 /*last fragment, fill rest of data*/ 2863 if (i == (frags - 1)) 2864 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i], 2865 pkt_dev->page, 0, 2866 min(datalen, PAGE_SIZE)); 2867 else 2868 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i], 2869 pkt_dev->page, 0, frag_len); 2870 2871 datalen -= skb_frag_size(&skb_shinfo(skb)->frags[i]); 2872 skb->len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2873 skb->data_len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2874 i++; 2875 skb_shinfo(skb)->nr_frags = i; 2876 } 2877 } 2878 2879 /* Stamp the time, and sequence number, 2880 * convert them to network byte order 2881 */ 2882 pgh->pgh_magic = htonl(PKTGEN_MAGIC); 2883 pgh->seq_num = htonl(pkt_dev->seq_num); 2884 2885 if (pkt_dev->flags & F_NO_TIMESTAMP) { 2886 pgh->tv_sec = 0; 2887 pgh->tv_usec = 0; 2888 } else { 2889 /* 2890 * pgh->tv_sec wraps in y2106 when interpreted as unsigned 2891 * as done by wireshark, or y2038 when interpreted as signed. 2892 * This is probably harmless, but if anyone wants to improve 2893 * it, we could introduce a variant that puts 64-bit nanoseconds 2894 * into the respective header bytes. 2895 * This would also be slightly faster to read. 2896 */ 2897 ktime_get_real_ts64(×tamp); 2898 pgh->tv_sec = htonl(timestamp.tv_sec); 2899 pgh->tv_usec = htonl(timestamp.tv_nsec / NSEC_PER_USEC); 2900 } 2901 } 2902 2903 static struct sk_buff *pktgen_alloc_skb(struct net_device *dev, 2904 struct pktgen_dev *pkt_dev) 2905 { 2906 unsigned int extralen = LL_RESERVED_SPACE(dev); 2907 struct sk_buff *skb = NULL; 2908 unsigned int size; 2909 2910 size = pkt_dev->cur_pkt_size + 64 + extralen + pkt_dev->pkt_overhead; 2911 if (pkt_dev->flags & F_NODE) { 2912 int node = pkt_dev->node >= 0 ? pkt_dev->node : numa_node_id(); 2913 2914 skb = __alloc_skb(NET_SKB_PAD + size, GFP_NOWAIT, 0, node); 2915 if (likely(skb)) { 2916 skb_reserve(skb, NET_SKB_PAD); 2917 skb->dev = dev; 2918 } 2919 } else { 2920 skb = __netdev_alloc_skb(dev, size, GFP_NOWAIT); 2921 } 2922 2923 /* the caller pre-fetches from skb->data and reserves for the mac hdr */ 2924 if (likely(skb)) 2925 skb_reserve(skb, extralen - 16); 2926 2927 return skb; 2928 } 2929 2930 static struct sk_buff *fill_packet_ipv4(struct net_device *odev, 2931 struct pktgen_dev *pkt_dev) 2932 { 2933 struct sk_buff *skb = NULL; 2934 __u8 *eth; 2935 struct udphdr *udph; 2936 int datalen, iplen; 2937 struct iphdr *iph; 2938 __be16 protocol = htons(ETH_P_IP); 2939 __be32 *mpls; 2940 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2941 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 2942 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 2943 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 2944 u16 queue_map; 2945 2946 if (pkt_dev->nr_labels) 2947 protocol = htons(ETH_P_MPLS_UC); 2948 2949 if (pkt_dev->vlan_id != 0xffff) 2950 protocol = htons(ETH_P_8021Q); 2951 2952 /* Update any of the values, used when we're incrementing various 2953 * fields. 2954 */ 2955 mod_cur_headers(pkt_dev); 2956 queue_map = pkt_dev->cur_queue_map; 2957 2958 skb = pktgen_alloc_skb(odev, pkt_dev); 2959 if (!skb) { 2960 sprintf(pkt_dev->result, "No memory"); 2961 return NULL; 2962 } 2963 2964 prefetchw(skb->data); 2965 skb_reserve(skb, 16); 2966 2967 /* Reserve for ethernet and IP header */ 2968 eth = skb_push(skb, 14); 2969 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32)); 2970 if (pkt_dev->nr_labels) 2971 mpls_push(mpls, pkt_dev); 2972 2973 if (pkt_dev->vlan_id != 0xffff) { 2974 if (pkt_dev->svlan_id != 0xffff) { 2975 svlan_tci = skb_put(skb, sizeof(__be16)); 2976 *svlan_tci = build_tci(pkt_dev->svlan_id, 2977 pkt_dev->svlan_cfi, 2978 pkt_dev->svlan_p); 2979 svlan_encapsulated_proto = skb_put(skb, 2980 sizeof(__be16)); 2981 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 2982 } 2983 vlan_tci = skb_put(skb, sizeof(__be16)); 2984 *vlan_tci = build_tci(pkt_dev->vlan_id, 2985 pkt_dev->vlan_cfi, 2986 pkt_dev->vlan_p); 2987 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16)); 2988 *vlan_encapsulated_proto = htons(ETH_P_IP); 2989 } 2990 2991 skb_reset_mac_header(skb); 2992 skb_set_network_header(skb, skb->len); 2993 iph = skb_put(skb, sizeof(struct iphdr)); 2994 2995 skb_set_transport_header(skb, skb->len); 2996 udph = skb_put(skb, sizeof(struct udphdr)); 2997 skb_set_queue_mapping(skb, queue_map); 2998 skb->priority = pkt_dev->skb_priority; 2999 3000 memcpy(eth, pkt_dev->hh, 12); 3001 *(__be16 *)ð[12] = protocol; 3002 3003 /* Eth + IPh + UDPh + mpls */ 3004 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 - 3005 pkt_dev->pkt_overhead; 3006 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) 3007 datalen = sizeof(struct pktgen_hdr); 3008 3009 udph->source = htons(pkt_dev->cur_udp_src); 3010 udph->dest = htons(pkt_dev->cur_udp_dst); 3011 udp_set_len_short(udph, datalen + 8); /* DATA + udphdr */ 3012 udph->check = 0; 3013 3014 iph->ihl = 5; 3015 iph->version = 4; 3016 iph->ttl = 32; 3017 iph->tos = pkt_dev->tos; 3018 iph->protocol = IPPROTO_UDP; /* UDP */ 3019 iph->saddr = pkt_dev->cur_saddr; 3020 iph->daddr = pkt_dev->cur_daddr; 3021 iph->id = htons(pkt_dev->ip_id); 3022 pkt_dev->ip_id++; 3023 iph->frag_off = 0; 3024 iplen = 20 + 8 + datalen; 3025 iph->tot_len = htons(iplen); 3026 ip_send_check(iph); 3027 skb->protocol = protocol; 3028 skb->dev = odev; 3029 skb->pkt_type = PACKET_HOST; 3030 3031 pktgen_finalize_skb(pkt_dev, skb, datalen); 3032 3033 if (!(pkt_dev->flags & F_UDPCSUM)) { 3034 skb->ip_summed = CHECKSUM_NONE; 3035 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)) { 3036 skb->ip_summed = CHECKSUM_PARTIAL; 3037 skb->csum = 0; 3038 udp4_hwcsum(skb, iph->saddr, iph->daddr); 3039 } else { 3040 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), datalen + 8, 0); 3041 3042 /* add protocol-dependent pseudo-header */ 3043 udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 3044 datalen + 8, IPPROTO_UDP, csum); 3045 3046 if (udph->check == 0) 3047 udph->check = CSUM_MANGLED_0; 3048 } 3049 3050 #ifdef CONFIG_XFRM 3051 if (!process_ipsec(pkt_dev, skb, protocol)) 3052 return NULL; 3053 #endif 3054 3055 return skb; 3056 } 3057 3058 static struct sk_buff *fill_packet_ipv6(struct net_device *odev, 3059 struct pktgen_dev *pkt_dev) 3060 { 3061 struct sk_buff *skb = NULL; 3062 __u8 *eth; 3063 struct udphdr *udph; 3064 int datalen, udplen; 3065 struct ipv6hdr *iph; 3066 __be16 protocol = htons(ETH_P_IPV6); 3067 __be32 *mpls; 3068 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 3069 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 3070 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 3071 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 3072 u16 queue_map; 3073 3074 if (pkt_dev->nr_labels) 3075 protocol = htons(ETH_P_MPLS_UC); 3076 3077 if (pkt_dev->vlan_id != 0xffff) 3078 protocol = htons(ETH_P_8021Q); 3079 3080 /* Update any of the values, used when we're incrementing various 3081 * fields. 3082 */ 3083 mod_cur_headers(pkt_dev); 3084 queue_map = pkt_dev->cur_queue_map; 3085 3086 skb = pktgen_alloc_skb(odev, pkt_dev); 3087 if (!skb) { 3088 sprintf(pkt_dev->result, "No memory"); 3089 return NULL; 3090 } 3091 3092 prefetchw(skb->data); 3093 skb_reserve(skb, 16); 3094 3095 /* Reserve for ethernet and IP header */ 3096 eth = skb_push(skb, 14); 3097 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32)); 3098 if (pkt_dev->nr_labels) 3099 mpls_push(mpls, pkt_dev); 3100 3101 if (pkt_dev->vlan_id != 0xffff) { 3102 if (pkt_dev->svlan_id != 0xffff) { 3103 svlan_tci = skb_put(skb, sizeof(__be16)); 3104 *svlan_tci = build_tci(pkt_dev->svlan_id, 3105 pkt_dev->svlan_cfi, 3106 pkt_dev->svlan_p); 3107 svlan_encapsulated_proto = skb_put(skb, 3108 sizeof(__be16)); 3109 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 3110 } 3111 vlan_tci = skb_put(skb, sizeof(__be16)); 3112 *vlan_tci = build_tci(pkt_dev->vlan_id, 3113 pkt_dev->vlan_cfi, 3114 pkt_dev->vlan_p); 3115 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16)); 3116 *vlan_encapsulated_proto = htons(ETH_P_IPV6); 3117 } 3118 3119 skb_reset_mac_header(skb); 3120 skb_set_network_header(skb, skb->len); 3121 iph = skb_put(skb, sizeof(struct ipv6hdr)); 3122 3123 skb_set_transport_header(skb, skb->len); 3124 udph = skb_put(skb, sizeof(struct udphdr)); 3125 skb_set_queue_mapping(skb, queue_map); 3126 skb->priority = pkt_dev->skb_priority; 3127 3128 memcpy(eth, pkt_dev->hh, 12); 3129 *(__be16 *) ð[12] = protocol; 3130 3131 /* Eth + IPh + UDPh + mpls */ 3132 datalen = pkt_dev->cur_pkt_size - 14 - 3133 sizeof(struct ipv6hdr) - sizeof(struct udphdr) - 3134 pkt_dev->pkt_overhead; 3135 3136 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) { 3137 datalen = sizeof(struct pktgen_hdr); 3138 net_info_ratelimited("increased datalen to %d\n", datalen); 3139 } 3140 3141 udplen = datalen + sizeof(struct udphdr); 3142 udph->source = htons(pkt_dev->cur_udp_src); 3143 udph->dest = htons(pkt_dev->cur_udp_dst); 3144 udp_set_len_short(udph, udplen); 3145 udph->check = 0; 3146 3147 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */ 3148 3149 if (pkt_dev->traffic_class) { 3150 /* Version + traffic class + flow (0) */ 3151 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20)); 3152 } 3153 3154 iph->hop_limit = 32; 3155 3156 iph->payload_len = htons(udplen); 3157 iph->nexthdr = IPPROTO_UDP; 3158 3159 iph->daddr = pkt_dev->cur_in6_daddr; 3160 iph->saddr = pkt_dev->cur_in6_saddr; 3161 3162 skb->protocol = protocol; 3163 skb->dev = odev; 3164 skb->pkt_type = PACKET_HOST; 3165 3166 pktgen_finalize_skb(pkt_dev, skb, datalen); 3167 3168 if (!(pkt_dev->flags & F_UDPCSUM)) { 3169 skb->ip_summed = CHECKSUM_NONE; 3170 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM)) { 3171 skb->ip_summed = CHECKSUM_PARTIAL; 3172 skb->csum_start = skb_transport_header(skb) - skb->head; 3173 skb->csum_offset = offsetof(struct udphdr, check); 3174 udph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, 0); 3175 } else { 3176 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), udplen, 0); 3177 3178 /* add protocol-dependent pseudo-header */ 3179 udph->check = csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, csum); 3180 3181 if (udph->check == 0) 3182 udph->check = CSUM_MANGLED_0; 3183 } 3184 3185 return skb; 3186 } 3187 3188 static struct sk_buff *fill_packet(struct net_device *odev, 3189 struct pktgen_dev *pkt_dev) 3190 { 3191 if (pkt_dev->flags & F_IPV6) 3192 return fill_packet_ipv6(odev, pkt_dev); 3193 else 3194 return fill_packet_ipv4(odev, pkt_dev); 3195 } 3196 3197 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev) 3198 { 3199 pkt_dev->seq_num = 1; 3200 pkt_dev->idle_acc = 0; 3201 pkt_dev->sofar = 0; 3202 pkt_dev->tx_bytes = 0; 3203 pkt_dev->errors = 0; 3204 } 3205 3206 /* Set up structure for sending pkts, clear counters */ 3207 3208 static void pktgen_run(struct pktgen_thread *t) 3209 { 3210 struct pktgen_dev *pkt_dev; 3211 int started = 0; 3212 3213 func_enter(); 3214 3215 rcu_read_lock(); 3216 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3217 3218 /* 3219 * setup odev and create initial packet. 3220 */ 3221 pktgen_setup_inject(pkt_dev); 3222 3223 if (pkt_dev->odev) { 3224 pktgen_clear_counters(pkt_dev); 3225 pkt_dev->skb = NULL; 3226 pkt_dev->started_at = pkt_dev->next_tx = ktime_get(); 3227 3228 set_pkt_overhead(pkt_dev); 3229 3230 strscpy(pkt_dev->result, "Starting"); 3231 pkt_dev->running = 1; /* Cranke yeself! */ 3232 started++; 3233 } else 3234 strscpy(pkt_dev->result, "Error starting"); 3235 } 3236 rcu_read_unlock(); 3237 if (started) 3238 t->control &= ~(T_STOP); 3239 } 3240 3241 static void pktgen_handle_all_threads(struct pktgen_net *pn, u32 flags) 3242 { 3243 struct pktgen_thread *t; 3244 3245 mutex_lock(&pktgen_thread_lock); 3246 3247 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3248 t->control |= (flags); 3249 3250 mutex_unlock(&pktgen_thread_lock); 3251 } 3252 3253 static void pktgen_stop_all_threads(struct pktgen_net *pn) 3254 { 3255 func_enter(); 3256 3257 pktgen_handle_all_threads(pn, T_STOP); 3258 } 3259 3260 static int thread_is_running(const struct pktgen_thread *t) 3261 { 3262 const struct pktgen_dev *pkt_dev; 3263 3264 rcu_read_lock(); 3265 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 3266 if (pkt_dev->running) { 3267 rcu_read_unlock(); 3268 return 1; 3269 } 3270 rcu_read_unlock(); 3271 return 0; 3272 } 3273 3274 static int pktgen_wait_thread_run(struct pktgen_thread *t) 3275 { 3276 while (thread_is_running(t)) { 3277 3278 /* note: 't' will still be around even after the unlock/lock 3279 * cycle because pktgen_thread threads are only cleared at 3280 * net exit 3281 */ 3282 mutex_unlock(&pktgen_thread_lock); 3283 msleep_interruptible(100); 3284 mutex_lock(&pktgen_thread_lock); 3285 3286 if (signal_pending(current)) 3287 goto signal; 3288 } 3289 return 1; 3290 signal: 3291 return 0; 3292 } 3293 3294 static int pktgen_wait_all_threads_run(struct pktgen_net *pn) 3295 { 3296 struct pktgen_thread *t; 3297 int sig = 1; 3298 3299 /* prevent from racing with rmmod */ 3300 if (!try_module_get(THIS_MODULE)) 3301 return sig; 3302 3303 mutex_lock(&pktgen_thread_lock); 3304 3305 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 3306 sig = pktgen_wait_thread_run(t); 3307 if (sig == 0) 3308 break; 3309 } 3310 3311 if (sig == 0) 3312 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3313 t->control |= (T_STOP); 3314 3315 mutex_unlock(&pktgen_thread_lock); 3316 module_put(THIS_MODULE); 3317 return sig; 3318 } 3319 3320 static void pktgen_run_all_threads(struct pktgen_net *pn) 3321 { 3322 func_enter(); 3323 3324 pktgen_handle_all_threads(pn, T_RUN); 3325 3326 /* Propagate thread->control */ 3327 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3328 3329 pktgen_wait_all_threads_run(pn); 3330 } 3331 3332 static void pktgen_reset_all_threads(struct pktgen_net *pn) 3333 { 3334 func_enter(); 3335 3336 pktgen_handle_all_threads(pn, T_REMDEVALL); 3337 3338 /* Propagate thread->control */ 3339 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3340 3341 pktgen_wait_all_threads_run(pn); 3342 } 3343 3344 static void show_results(struct pktgen_dev *pkt_dev, int nr_frags) 3345 { 3346 __u64 bps, mbps, pps; 3347 char *p = pkt_dev->result; 3348 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at, 3349 pkt_dev->started_at); 3350 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc); 3351 3352 p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n", 3353 (unsigned long long)ktime_to_us(elapsed), 3354 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)), 3355 (unsigned long long)ktime_to_us(idle), 3356 (unsigned long long)pkt_dev->sofar, 3357 pkt_dev->cur_pkt_size, nr_frags); 3358 3359 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC, 3360 ktime_to_ns(elapsed)); 3361 3362 if (pkt_dev->n_imix_entries > 0) { 3363 int i; 3364 struct imix_pkt *entry; 3365 3366 bps = 0; 3367 for (i = 0; i < pkt_dev->n_imix_entries; i++) { 3368 entry = &pkt_dev->imix_entries[i]; 3369 bps += entry->size * entry->count_so_far; 3370 } 3371 bps = div64_u64(bps * 8 * NSEC_PER_SEC, ktime_to_ns(elapsed)); 3372 } else { 3373 bps = pps * 8 * pkt_dev->cur_pkt_size; 3374 } 3375 3376 mbps = bps; 3377 do_div(mbps, 1000000); 3378 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu", 3379 (unsigned long long)pps, 3380 (unsigned long long)mbps, 3381 (unsigned long long)bps, 3382 (unsigned long long)pkt_dev->errors); 3383 } 3384 3385 /* Set stopped-at timer, remove from running list, do counters & statistics */ 3386 static int pktgen_stop_device(struct pktgen_dev *pkt_dev) 3387 { 3388 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1; 3389 3390 if (!pkt_dev->running) { 3391 pr_warn("interface: %s is already stopped\n", 3392 pkt_dev->odevname); 3393 return -EINVAL; 3394 } 3395 3396 pkt_dev->running = 0; 3397 kfree_skb(pkt_dev->skb); 3398 pkt_dev->skb = NULL; 3399 pkt_dev->stopped_at = ktime_get(); 3400 3401 show_results(pkt_dev, nr_frags); 3402 3403 return 0; 3404 } 3405 3406 static struct pktgen_dev *next_to_run(struct pktgen_thread *t) 3407 { 3408 struct pktgen_dev *pkt_dev, *best = NULL; 3409 3410 rcu_read_lock(); 3411 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3412 if (!pkt_dev->running) 3413 continue; 3414 if (best == NULL) 3415 best = pkt_dev; 3416 else if (ktime_compare(pkt_dev->next_tx, best->next_tx) < 0) 3417 best = pkt_dev; 3418 } 3419 rcu_read_unlock(); 3420 3421 return best; 3422 } 3423 3424 static void pktgen_stop(struct pktgen_thread *t) 3425 { 3426 struct pktgen_dev *pkt_dev; 3427 3428 func_enter(); 3429 3430 rcu_read_lock(); 3431 3432 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3433 pktgen_stop_device(pkt_dev); 3434 } 3435 3436 rcu_read_unlock(); 3437 } 3438 3439 /* 3440 * one of our devices needs to be removed - find it 3441 * and remove it 3442 */ 3443 static void pktgen_rem_one_if(struct pktgen_thread *t) 3444 { 3445 struct list_head *q, *n; 3446 struct pktgen_dev *cur; 3447 3448 func_enter(); 3449 3450 list_for_each_safe(q, n, &t->if_list) { 3451 cur = list_entry(q, struct pktgen_dev, list); 3452 3453 if (!cur->removal_mark) 3454 continue; 3455 3456 kfree_skb(cur->skb); 3457 cur->skb = NULL; 3458 3459 pktgen_remove_device(t, cur); 3460 3461 break; 3462 } 3463 } 3464 3465 static void pktgen_rem_all_ifs(struct pktgen_thread *t) 3466 { 3467 struct list_head *q, *n; 3468 struct pktgen_dev *cur; 3469 3470 func_enter(); 3471 3472 /* Remove all devices, free mem */ 3473 3474 list_for_each_safe(q, n, &t->if_list) { 3475 cur = list_entry(q, struct pktgen_dev, list); 3476 3477 kfree_skb(cur->skb); 3478 cur->skb = NULL; 3479 3480 pktgen_remove_device(t, cur); 3481 } 3482 } 3483 3484 static void pktgen_rem_thread(struct pktgen_thread *t) 3485 { 3486 /* Remove from the thread list */ 3487 remove_proc_entry(t->tsk->comm, t->net->proc_dir); 3488 } 3489 3490 static void pktgen_resched(struct pktgen_dev *pkt_dev) 3491 { 3492 ktime_t idle_start = ktime_get(); 3493 3494 schedule(); 3495 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start)); 3496 } 3497 3498 static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev) 3499 { 3500 ktime_t idle_start = ktime_get(); 3501 3502 while (refcount_read(&(pkt_dev->skb->users)) != 1) { 3503 if (signal_pending(current)) 3504 break; 3505 3506 if (need_resched()) 3507 pktgen_resched(pkt_dev); 3508 else 3509 cpu_relax(); 3510 } 3511 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start)); 3512 } 3513 3514 static void pktgen_xmit(struct pktgen_dev *pkt_dev) 3515 { 3516 bool skb_shared = !!(READ_ONCE(pkt_dev->flags) & F_SHARED); 3517 struct net_device *odev = pkt_dev->odev; 3518 struct netdev_queue *txq; 3519 unsigned int burst = 1; 3520 struct sk_buff *skb; 3521 int clone_skb = 0; 3522 int ret; 3523 3524 /* If 'skb_shared' is false, the read of possible 3525 * new values (if any) for 'burst' and 'clone_skb' will be skipped to 3526 * prevent some concurrent changes from slipping in. And the stabilized 3527 * config will be read in during the next run of pktgen_xmit. 3528 */ 3529 if (skb_shared) { 3530 burst = READ_ONCE(pkt_dev->burst); 3531 clone_skb = READ_ONCE(pkt_dev->clone_skb); 3532 } 3533 3534 /* If device is offline, then don't send */ 3535 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) { 3536 pktgen_stop_device(pkt_dev); 3537 return; 3538 } 3539 3540 /* This is max DELAY, this has special meaning of 3541 * "never transmit" 3542 */ 3543 if (unlikely(pkt_dev->delay == ULLONG_MAX)) { 3544 pkt_dev->next_tx = ktime_add_ns(ktime_get(), ULONG_MAX); 3545 return; 3546 } 3547 3548 /* If no skb or clone count exhausted then get new one */ 3549 if (!pkt_dev->skb || (pkt_dev->last_ok && 3550 ++pkt_dev->clone_count >= clone_skb)) { 3551 /* build a new pkt */ 3552 kfree_skb(pkt_dev->skb); 3553 3554 pkt_dev->skb = fill_packet(odev, pkt_dev); 3555 if (pkt_dev->skb == NULL) { 3556 pr_err("ERROR: couldn't allocate skb in fill_packet\n"); 3557 schedule(); 3558 pkt_dev->clone_count--; /* back out increment, OOM */ 3559 return; 3560 } 3561 pkt_dev->last_pkt_size = pkt_dev->skb->len; 3562 pkt_dev->clone_count = 0; /* reset counter */ 3563 } 3564 3565 if (pkt_dev->delay && pkt_dev->last_ok) 3566 spin(pkt_dev, pkt_dev->next_tx); 3567 3568 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) { 3569 skb = pkt_dev->skb; 3570 skb->protocol = eth_type_trans(skb, skb->dev); 3571 if (skb_shared) 3572 refcount_add(burst, &skb->users); 3573 local_bh_disable(); 3574 do { 3575 ret = netif_receive_skb(skb); 3576 if (ret == NET_RX_DROP) 3577 pkt_dev->errors++; 3578 pkt_dev->sofar++; 3579 pkt_dev->seq_num++; 3580 if (unlikely(!skb_shared)) { 3581 pkt_dev->skb = NULL; 3582 break; 3583 } 3584 if (refcount_read(&skb->users) != burst) { 3585 /* skb was queued by rps/rfs or taps, 3586 * so cannot reuse this skb 3587 */ 3588 WARN_ON(refcount_sub_and_test(burst - 1, &skb->users)); 3589 /* get out of the loop and wait 3590 * until skb is consumed 3591 */ 3592 break; 3593 } 3594 /* skb was 'freed' by stack, so clean few 3595 * bits and reuse it 3596 */ 3597 skb_reset_redirect(skb); 3598 } while (--burst > 0); 3599 goto out; /* Skips xmit_mode M_START_XMIT */ 3600 } else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) { 3601 local_bh_disable(); 3602 if (skb_shared) 3603 refcount_inc(&pkt_dev->skb->users); 3604 3605 ret = dev_queue_xmit(pkt_dev->skb); 3606 3607 if (!skb_shared && dev_xmit_complete(ret)) 3608 pkt_dev->skb = NULL; 3609 3610 switch (ret) { 3611 case NET_XMIT_SUCCESS: 3612 pkt_dev->sofar++; 3613 pkt_dev->seq_num++; 3614 pkt_dev->tx_bytes += pkt_dev->last_pkt_size; 3615 break; 3616 case NET_XMIT_DROP: 3617 case NET_XMIT_CN: 3618 /* These are all valid return codes for a qdisc but 3619 * indicate packets are being dropped or will likely 3620 * be dropped soon. 3621 */ 3622 case NETDEV_TX_BUSY: 3623 /* qdisc may call dev_hard_start_xmit directly in cases 3624 * where no queues exist e.g. loopback device, virtual 3625 * devices, etc. In this case we need to handle 3626 * NETDEV_TX_ codes. 3627 */ 3628 default: 3629 pkt_dev->errors++; 3630 net_info_ratelimited("%s xmit error: %d\n", 3631 pkt_dev->odevname, ret); 3632 break; 3633 } 3634 goto out; 3635 } 3636 3637 txq = skb_get_tx_queue(odev, pkt_dev->skb); 3638 3639 local_bh_disable(); 3640 3641 HARD_TX_LOCK(odev, txq, smp_processor_id()); 3642 3643 if (unlikely(netif_xmit_frozen_or_drv_stopped(txq))) { 3644 pkt_dev->last_ok = 0; 3645 goto unlock; 3646 } 3647 if (skb_shared) 3648 refcount_add(burst, &pkt_dev->skb->users); 3649 3650 xmit_more: 3651 ret = netdev_start_xmit(pkt_dev->skb, odev, txq, --burst > 0); 3652 3653 if (!skb_shared && dev_xmit_complete(ret)) 3654 pkt_dev->skb = NULL; 3655 3656 switch (ret) { 3657 case NETDEV_TX_OK: 3658 pkt_dev->last_ok = 1; 3659 pkt_dev->sofar++; 3660 pkt_dev->seq_num++; 3661 pkt_dev->tx_bytes += pkt_dev->last_pkt_size; 3662 if (burst > 0 && !netif_xmit_frozen_or_drv_stopped(txq)) 3663 goto xmit_more; 3664 break; 3665 case NET_XMIT_DROP: 3666 case NET_XMIT_CN: 3667 /* skb has been consumed */ 3668 pkt_dev->errors++; 3669 break; 3670 default: /* Drivers are not supposed to return other values! */ 3671 net_info_ratelimited("%s xmit error: %d\n", 3672 pkt_dev->odevname, ret); 3673 pkt_dev->errors++; 3674 fallthrough; 3675 case NETDEV_TX_BUSY: 3676 /* Retry it next time */ 3677 if (skb_shared) 3678 refcount_dec(&pkt_dev->skb->users); 3679 pkt_dev->last_ok = 0; 3680 } 3681 if (unlikely(burst)) 3682 WARN_ON(refcount_sub_and_test(burst, &pkt_dev->skb->users)); 3683 unlock: 3684 HARD_TX_UNLOCK(odev, txq); 3685 3686 out: 3687 local_bh_enable(); 3688 3689 /* If pkt_dev->count is zero, then run forever */ 3690 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) { 3691 if (pkt_dev->skb) 3692 pktgen_wait_for_skb(pkt_dev); 3693 3694 /* Done with this */ 3695 pktgen_stop_device(pkt_dev); 3696 } 3697 } 3698 3699 /* 3700 * Main loop of the thread goes here 3701 */ 3702 3703 static int pktgen_thread_worker(void *arg) 3704 { 3705 struct pktgen_thread *t = arg; 3706 struct pktgen_dev *pkt_dev = NULL; 3707 int cpu = t->cpu; 3708 3709 WARN_ON_ONCE(smp_processor_id() != cpu); 3710 3711 init_waitqueue_head(&t->queue); 3712 complete(&t->start_done); 3713 3714 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current)); 3715 3716 set_freezable(); 3717 3718 while (!kthread_should_stop()) { 3719 pkt_dev = next_to_run(t); 3720 3721 if (unlikely(!pkt_dev && t->control == 0)) { 3722 if (t->net->pktgen_exiting) 3723 break; 3724 wait_event_freezable_timeout(t->queue, 3725 t->control != 0, HZ / 10); 3726 continue; 3727 } 3728 3729 if (likely(pkt_dev)) { 3730 pktgen_xmit(pkt_dev); 3731 3732 if (need_resched()) 3733 pktgen_resched(pkt_dev); 3734 else 3735 cpu_relax(); 3736 } 3737 3738 if (t->control & T_STOP) { 3739 pktgen_stop(t); 3740 t->control &= ~(T_STOP); 3741 } 3742 3743 if (t->control & T_RUN) { 3744 pktgen_run(t); 3745 t->control &= ~(T_RUN); 3746 } 3747 3748 if (t->control & T_REMDEVALL) { 3749 pktgen_rem_all_ifs(t); 3750 t->control &= ~(T_REMDEVALL); 3751 } 3752 3753 if (t->control & T_REMDEV) { 3754 pktgen_rem_one_if(t); 3755 t->control &= ~(T_REMDEV); 3756 } 3757 3758 try_to_freeze(); 3759 } 3760 3761 pr_debug("%s stopping all device\n", t->tsk->comm); 3762 pktgen_stop(t); 3763 3764 pr_debug("%s removing all device\n", t->tsk->comm); 3765 pktgen_rem_all_ifs(t); 3766 3767 pr_debug("%s removing thread\n", t->tsk->comm); 3768 pktgen_rem_thread(t); 3769 3770 return 0; 3771 } 3772 3773 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 3774 const char *ifname, bool exact) 3775 { 3776 struct pktgen_dev *p, *pkt_dev = NULL; 3777 size_t len = strlen(ifname); 3778 3779 rcu_read_lock(); 3780 list_for_each_entry_rcu(p, &t->if_list, list) 3781 if (strncmp(p->odevname, ifname, len) == 0) { 3782 if (p->odevname[len]) { 3783 if (exact || p->odevname[len] != '@') 3784 continue; 3785 } 3786 pkt_dev = p; 3787 break; 3788 } 3789 3790 rcu_read_unlock(); 3791 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev); 3792 return pkt_dev; 3793 } 3794 3795 /* 3796 * Adds a dev at front of if_list. 3797 */ 3798 3799 static int add_dev_to_thread(struct pktgen_thread *t, 3800 struct pktgen_dev *pkt_dev) 3801 { 3802 int rv = 0; 3803 3804 /* This function cannot be called concurrently, as its called 3805 * under pktgen_thread_lock mutex, but it can run from 3806 * userspace on another CPU than the kthread. The if_lock() 3807 * is used here to sync with concurrent instances of 3808 * _rem_dev_from_if_list() invoked via kthread, which is also 3809 * updating the if_list 3810 */ 3811 if_lock(t); 3812 3813 if (pkt_dev->pg_thread) { 3814 pr_err("ERROR: already assigned to a thread\n"); 3815 rv = -EBUSY; 3816 goto out; 3817 } 3818 3819 pkt_dev->running = 0; 3820 pkt_dev->pg_thread = t; 3821 list_add_rcu(&pkt_dev->list, &t->if_list); 3822 3823 out: 3824 if_unlock(t); 3825 return rv; 3826 } 3827 3828 /* Called under thread lock */ 3829 3830 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) 3831 { 3832 struct pktgen_dev *pkt_dev; 3833 int err; 3834 int node = cpu_to_node(t->cpu); 3835 3836 /* We don't allow a device to be on several threads */ 3837 3838 pkt_dev = __pktgen_NN_threads(t->net, ifname, FIND); 3839 if (pkt_dev) { 3840 pr_err("ERROR: interface already used\n"); 3841 return -EBUSY; 3842 } 3843 3844 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node); 3845 if (!pkt_dev) 3846 return -ENOMEM; 3847 3848 strscpy(pkt_dev->odevname, ifname); 3849 pkt_dev->flows = vzalloc_node(array_size(MAX_CFLOWS, 3850 sizeof(struct flow_state)), 3851 node); 3852 if (pkt_dev->flows == NULL) { 3853 kfree(pkt_dev); 3854 return -ENOMEM; 3855 } 3856 3857 pkt_dev->removal_mark = 0; 3858 pkt_dev->nfrags = 0; 3859 pkt_dev->delay = pg_delay_d; 3860 pkt_dev->count = pg_count_d; 3861 pkt_dev->sofar = 0; 3862 pkt_dev->udp_src_min = 9; /* sink port */ 3863 pkt_dev->udp_src_max = 9; 3864 pkt_dev->udp_dst_min = 9; 3865 pkt_dev->udp_dst_max = 9; 3866 pkt_dev->vlan_p = 0; 3867 pkt_dev->vlan_cfi = 0; 3868 pkt_dev->vlan_id = 0xffff; 3869 pkt_dev->svlan_p = 0; 3870 pkt_dev->svlan_cfi = 0; 3871 pkt_dev->svlan_id = 0xffff; 3872 pkt_dev->burst = 1; 3873 pkt_dev->node = NUMA_NO_NODE; 3874 pkt_dev->flags = F_SHARED; /* SKB shared by default */ 3875 3876 err = pktgen_setup_dev(t->net, pkt_dev, ifname); 3877 if (err) 3878 goto out1; 3879 if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING) 3880 pkt_dev->clone_skb = pg_clone_skb_d; 3881 3882 pkt_dev->entry = proc_create_data(ifname, 0600, t->net->proc_dir, 3883 &pktgen_if_proc_ops, pkt_dev); 3884 if (!pkt_dev->entry) { 3885 pr_err("cannot create %s/%s procfs entry\n", 3886 PG_PROC_DIR, ifname); 3887 err = -EINVAL; 3888 goto out2; 3889 } 3890 #ifdef CONFIG_XFRM 3891 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT; 3892 pkt_dev->ipsproto = IPPROTO_ESP; 3893 3894 /* xfrm tunnel mode needs additional dst to extract outer 3895 * ip header protocol/ttl/id field, here create a phony one. 3896 * instead of looking for a valid rt, which definitely hurting 3897 * performance under such circumstance. 3898 */ 3899 pkt_dev->dstops.family = AF_INET; 3900 pkt_dev->xdst.u.dst.dev = pkt_dev->odev; 3901 dst_init_metrics(&pkt_dev->xdst.u.dst, pktgen_dst_metrics, false); 3902 pkt_dev->xdst.child = &pkt_dev->xdst.u.dst; 3903 pkt_dev->xdst.u.dst.ops = &pkt_dev->dstops; 3904 #endif 3905 3906 return add_dev_to_thread(t, pkt_dev); 3907 out2: 3908 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); 3909 out1: 3910 #ifdef CONFIG_XFRM 3911 free_SAs(pkt_dev); 3912 #endif 3913 vfree(pkt_dev->flows); 3914 kfree(pkt_dev); 3915 return err; 3916 } 3917 3918 static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn) 3919 { 3920 struct pktgen_thread *t; 3921 struct proc_dir_entry *pe; 3922 struct task_struct *p; 3923 3924 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL, 3925 cpu_to_node(cpu)); 3926 if (!t) { 3927 pr_err("ERROR: out of memory, can't create new thread\n"); 3928 return -ENOMEM; 3929 } 3930 3931 mutex_init(&t->if_lock); 3932 t->cpu = cpu; 3933 3934 INIT_LIST_HEAD(&t->if_list); 3935 3936 list_add_tail(&t->th_list, &pn->pktgen_threads); 3937 init_completion(&t->start_done); 3938 3939 p = kthread_create_on_cpu(pktgen_thread_worker, t, cpu, "kpktgend_%d"); 3940 if (IS_ERR(p)) { 3941 pr_err("kthread_create_on_node() failed for cpu %d\n", t->cpu); 3942 list_del(&t->th_list); 3943 kfree(t); 3944 return PTR_ERR(p); 3945 } 3946 3947 t->tsk = p; 3948 3949 pe = proc_create_data(t->tsk->comm, 0600, pn->proc_dir, 3950 &pktgen_thread_proc_ops, t); 3951 if (!pe) { 3952 pr_err("cannot create %s/%s procfs entry\n", 3953 PG_PROC_DIR, t->tsk->comm); 3954 kthread_stop(p); 3955 list_del(&t->th_list); 3956 kfree(t); 3957 return -EINVAL; 3958 } 3959 3960 t->net = pn; 3961 get_task_struct(p); 3962 wake_up_process(p); 3963 wait_for_completion(&t->start_done); 3964 3965 return 0; 3966 } 3967 3968 /* 3969 * Removes a device from the thread if_list. 3970 */ 3971 static void _rem_dev_from_if_list(struct pktgen_thread *t, 3972 struct pktgen_dev *pkt_dev) 3973 { 3974 struct list_head *q, *n; 3975 struct pktgen_dev *p; 3976 3977 if_lock(t); 3978 proc_remove(pkt_dev->entry); 3979 list_for_each_safe(q, n, &t->if_list) { 3980 p = list_entry(q, struct pktgen_dev, list); 3981 if (p == pkt_dev) 3982 list_del_rcu(&p->list); 3983 } 3984 if_unlock(t); 3985 } 3986 3987 static int pktgen_remove_device(struct pktgen_thread *t, 3988 struct pktgen_dev *pkt_dev) 3989 { 3990 pr_debug("remove_device pkt_dev=%p\n", pkt_dev); 3991 3992 if (pkt_dev->running) { 3993 pr_warn("WARNING: trying to remove a running interface, stopping it now\n"); 3994 pktgen_stop_device(pkt_dev); 3995 } 3996 3997 /* Dis-associate from the interface */ 3998 3999 if (pkt_dev->odev) { 4000 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker); 4001 pkt_dev->odev = NULL; 4002 } 4003 4004 /* Remove proc before if_list entry, because add_device uses 4005 * list to determine if interface already exist, avoid race 4006 * with proc_create_data() 4007 */ 4008 _rem_dev_from_if_list(t, pkt_dev); 4009 4010 #ifdef CONFIG_XFRM 4011 free_SAs(pkt_dev); 4012 #endif 4013 vfree(pkt_dev->flows); 4014 if (pkt_dev->page) 4015 put_page(pkt_dev->page); 4016 kfree_rcu(pkt_dev, rcu); 4017 return 0; 4018 } 4019 4020 static int __net_init pg_net_init(struct net *net) 4021 { 4022 struct pktgen_net *pn = net_generic(net, pg_net_id); 4023 struct proc_dir_entry *pe; 4024 int cpu, ret = 0; 4025 4026 pn->net = net; 4027 INIT_LIST_HEAD(&pn->pktgen_threads); 4028 pn->pktgen_exiting = false; 4029 pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net); 4030 if (!pn->proc_dir) { 4031 pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR); 4032 return -ENODEV; 4033 } 4034 pe = proc_create(PGCTRL, 0600, pn->proc_dir, &pktgen_proc_ops); 4035 if (pe == NULL) { 4036 pr_err("cannot create %s procfs entry\n", PGCTRL); 4037 ret = -EINVAL; 4038 goto remove; 4039 } 4040 4041 cpus_read_lock(); 4042 for_each_online_cpu(cpu) { 4043 int err; 4044 4045 err = pktgen_create_thread(cpu, pn); 4046 if (err) 4047 pr_warn("Cannot create thread for cpu %d (%d)\n", 4048 cpu, err); 4049 } 4050 cpus_read_unlock(); 4051 4052 if (list_empty(&pn->pktgen_threads)) { 4053 pr_err("Initialization failed for all threads\n"); 4054 ret = -ENODEV; 4055 goto remove_entry; 4056 } 4057 4058 return 0; 4059 4060 remove_entry: 4061 remove_proc_entry(PGCTRL, pn->proc_dir); 4062 remove: 4063 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net); 4064 return ret; 4065 } 4066 4067 static void __net_exit pg_net_exit(struct net *net) 4068 { 4069 struct pktgen_net *pn = net_generic(net, pg_net_id); 4070 struct pktgen_thread *t; 4071 struct list_head *q, *n; 4072 LIST_HEAD(list); 4073 4074 /* Stop all interfaces & threads */ 4075 pn->pktgen_exiting = true; 4076 4077 mutex_lock(&pktgen_thread_lock); 4078 list_splice_init(&pn->pktgen_threads, &list); 4079 mutex_unlock(&pktgen_thread_lock); 4080 4081 list_for_each_safe(q, n, &list) { 4082 t = list_entry(q, struct pktgen_thread, th_list); 4083 list_del(&t->th_list); 4084 kthread_stop_put(t->tsk); 4085 kfree(t); 4086 } 4087 4088 remove_proc_entry(PGCTRL, pn->proc_dir); 4089 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net); 4090 } 4091 4092 static struct pernet_operations pg_net_ops = { 4093 .init = pg_net_init, 4094 .exit = pg_net_exit, 4095 .id = &pg_net_id, 4096 .size = sizeof(struct pktgen_net), 4097 }; 4098 4099 static int __init pg_init(void) 4100 { 4101 int ret = 0; 4102 4103 pr_info("%s", version); 4104 ret = register_pernet_subsys(&pg_net_ops); 4105 if (ret) 4106 return ret; 4107 ret = register_netdevice_notifier(&pktgen_notifier_block); 4108 if (ret) 4109 unregister_pernet_subsys(&pg_net_ops); 4110 4111 return ret; 4112 } 4113 4114 static void __exit pg_cleanup(void) 4115 { 4116 unregister_netdevice_notifier(&pktgen_notifier_block); 4117 unregister_pernet_subsys(&pg_net_ops); 4118 /* Don't need rcu_barrier() due to use of kfree_rcu() */ 4119 } 4120 4121 module_init(pg_init); 4122 module_exit(pg_cleanup); 4123 4124 MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>"); 4125 MODULE_DESCRIPTION("Packet Generator tool"); 4126 MODULE_LICENSE("GPL"); 4127 MODULE_VERSION(VERSION); 4128 module_param(pg_count_d, int, 0); 4129 MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject"); 4130 module_param(pg_delay_d, int, 0); 4131 MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)"); 4132 module_param(pg_clone_skb_d, int, 0); 4133 MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet"); 4134 module_param(debug, int, 0); 4135 MODULE_PARM_DESC(debug, "Enable debugging of pktgen module"); 4136