1#!/bin/bash 2# 3# Multiqueue: Using pktgen threads for sending on multiple CPUs 4# * adding devices to kernel threads which are in the same NUMA node 5# * bound devices queue's irq affinity to the threads, 1:1 mapping 6# * notice the naming scheme for keeping device names unique 7# * nameing scheme: dev@thread_number 8# * flow variation via random UDP source port 9# 10basedir=`dirname $0` 11source ${basedir}/functions.sh 12root_check_run_with_sudo "$@" 13# 14# Required param: -i dev in $DEV 15source ${basedir}/parameters.sh 16 17# Base Config 18[ -z "$COUNT" ] && COUNT="20000000" # Zero means indefinitely 19[ -z "$CLONE_SKB" ] && CLONE_SKB="0" 20 21# Flow variation random source port between min and max 22UDP_SRC_MIN=9 23UDP_SRC_MAX=109 24 25node=`get_iface_node $DEV` 26irq_array=(`get_iface_irqs $DEV`) 27cpu_array=(`get_node_cpus $node`) 28 29[ $THREADS -gt ${#irq_array[*]} -o $THREADS -gt ${#cpu_array[*]} ] && \ 30 err 1 "Thread number $THREADS exceeds: min (${#irq_array[*]},${#cpu_array[*]})" 31 32# (example of setting default params in your script) 33if [ -z "$DEST_IP" ]; then 34 [ -z "$IP6" ] && DEST_IP="198.18.0.42" || DEST_IP="FD00::1" 35fi 36[ -z "$DST_MAC" ] && DST_MAC="90:e2:ba:ff:ff:ff" 37if [ -n "$DEST_IP" ]; then 38 validate_addr${IP6} $DEST_IP 39 read -r DST_MIN DST_MAX <<< $(parse_addr${IP6} $DEST_IP) 40fi 41if [ -n "$DST_PORT" ]; then 42 read -r UDP_DST_MIN UDP_DST_MAX <<< $(parse_ports $DST_PORT) 43 validate_ports $UDP_DST_MIN $UDP_DST_MAX 44fi 45 46# General cleanup everything since last run 47[ -z "$APPEND" ] && pg_ctrl "reset" 48 49# Threads are specified with parameter -t value in $THREADS 50for ((i = 0; i < $THREADS; i++)); do 51 # The device name is extended with @name, using thread number to 52 # make then unique, but any name will do. 53 # Set the queue's irq affinity to this $thread (processor) 54 # if '-f' is designated, offset cpu id 55 thread=${cpu_array[$((i+F_THREAD))]} 56 dev=${DEV}@${thread} 57 echo $thread > /proc/irq/${irq_array[$i]}/smp_affinity_list 58 info "irq ${irq_array[$i]} is set affinity to `cat /proc/irq/${irq_array[$i]}/smp_affinity_list`" 59 60 # Add remove all other devices and add_device $dev to thread 61 [ -z "$APPEND" ] && pg_thread $thread "rem_device_all" 62 pg_thread $thread "add_device" $dev 63 64 # select queue and bind the queue and $dev in 1:1 relationship 65 queue_num=$i 66 info "queue number is $queue_num" 67 pg_set $dev "queue_map_min $queue_num" 68 pg_set $dev "queue_map_max $queue_num" 69 70 # Notice config queue to map to cpu (mirrors smp_processor_id()) 71 # It is beneficial to map IRQ /proc/irq/*/smp_affinity 1:1 to CPU number 72 pg_set $dev "flag QUEUE_MAP_CPU" 73 74 # Base config of dev 75 pg_set $dev "count $COUNT" 76 pg_set $dev "clone_skb $CLONE_SKB" 77 pg_set $dev "pkt_size $PKT_SIZE" 78 pg_set $dev "delay $DELAY" 79 80 # Flag example disabling timestamping 81 pg_set $dev "flag NO_TIMESTAMP" 82 83 # Destination 84 pg_set $dev "dst_mac $DST_MAC" 85 pg_set $dev "dst${IP6}_min $DST_MIN" 86 pg_set $dev "dst${IP6}_max $DST_MAX" 87 88 if [ -n "$DST_PORT" ]; then 89 # Single destination port or random port range 90 pg_set $dev "flag UDPDST_RND" 91 pg_set $dev "udp_dst_min $UDP_DST_MIN" 92 pg_set $dev "udp_dst_max $UDP_DST_MAX" 93 fi 94 95 [ ! -z "$UDP_CSUM" ] && pg_set $dev "flag UDPCSUM" 96 97 # Setup random UDP port src range 98 pg_set $dev "flag UDPSRC_RND" 99 pg_set $dev "udp_src_min $UDP_SRC_MIN" 100 pg_set $dev "udp_src_max $UDP_SRC_MAX" 101done 102 103# Run if user hits control-c 104function print_result() { 105 # Print results 106 for ((i = 0; i < $THREADS; i++)); do 107 thread=${cpu_array[$((i+F_THREAD))]} 108 dev=${DEV}@${thread} 109 echo "Device: $dev" 110 cat /proc/net/pktgen/$dev | grep -A2 "Result:" 111 done 112} 113# trap keyboard interrupt (Ctrl-C) 114trap true SIGINT 115 116# start_run 117if [ -z "$APPEND" ]; then 118 echo "Running... ctrl^C to stop" >&2 119 pg_ctrl "start" 120 echo "Done" >&2 121 122 print_result 123else 124 echo "Append mode: config done. Do more or use 'pg_ctrl start' to run" 125fi 126