1#!/bin/bash 2# 3# Multiqueue: Using pktgen threads for sending on multiple CPUs 4# * adding devices to kernel threads which are in the same NUMA node 5# * bound devices queue's irq affinity to the threads, 1:1 mapping 6# * notice the naming scheme for keeping device names unique 7# * nameing scheme: dev@thread_number 8# * flow variation via random UDP source port 9# 10basedir=`dirname $0` 11source ${basedir}/functions.sh 12root_check_run_with_sudo "$@" 13# 14# Required param: -i dev in $DEV 15source ${basedir}/parameters.sh 16 17# Base Config 18DELAY="0" # Zero means max speed 19[ -z "$COUNT" ] && COUNT="20000000" # Zero means indefinitely 20[ -z "$CLONE_SKB" ] && CLONE_SKB="0" 21 22# Flow variation random source port between min and max 23UDP_MIN=9 24UDP_MAX=109 25 26node=`get_iface_node $DEV` 27irq_array=(`get_iface_irqs $DEV`) 28cpu_array=(`get_node_cpus $node`) 29 30[ $THREADS -gt ${#irq_array[*]} -o $THREADS -gt ${#cpu_array[*]} ] && \ 31 err 1 "Thread number $THREADS exceeds: min (${#irq_array[*]},${#cpu_array[*]})" 32 33# (example of setting default params in your script) 34if [ -z "$DEST_IP" ]; then 35 [ -z "$IP6" ] && DEST_IP="198.18.0.42" || DEST_IP="FD00::1" 36fi 37[ -z "$DST_MAC" ] && DST_MAC="90:e2:ba:ff:ff:ff" 38 39# General cleanup everything since last run 40pg_ctrl "reset" 41 42# Threads are specified with parameter -t value in $THREADS 43for ((i = 0; i < $THREADS; i++)); do 44 # The device name is extended with @name, using thread number to 45 # make then unique, but any name will do. 46 # Set the queue's irq affinity to this $thread (processor) 47 # if '-f' is designated, offset cpu id 48 thread=${cpu_array[$((i+F_THREAD))]} 49 dev=${DEV}@${thread} 50 echo $thread > /proc/irq/${irq_array[$i]}/smp_affinity_list 51 info "irq ${irq_array[$i]} is set affinity to `cat /proc/irq/${irq_array[$i]}/smp_affinity_list`" 52 53 # Add remove all other devices and add_device $dev to thread 54 pg_thread $thread "rem_device_all" 55 pg_thread $thread "add_device" $dev 56 57 # select queue and bind the queue and $dev in 1:1 relationship 58 queue_num=$i 59 info "queue number is $queue_num" 60 pg_set $dev "queue_map_min $queue_num" 61 pg_set $dev "queue_map_max $queue_num" 62 63 # Notice config queue to map to cpu (mirrors smp_processor_id()) 64 # It is beneficial to map IRQ /proc/irq/*/smp_affinity 1:1 to CPU number 65 pg_set $dev "flag QUEUE_MAP_CPU" 66 67 # Base config of dev 68 pg_set $dev "count $COUNT" 69 pg_set $dev "clone_skb $CLONE_SKB" 70 pg_set $dev "pkt_size $PKT_SIZE" 71 pg_set $dev "delay $DELAY" 72 73 # Flag example disabling timestamping 74 pg_set $dev "flag NO_TIMESTAMP" 75 76 # Destination 77 pg_set $dev "dst_mac $DST_MAC" 78 pg_set $dev "dst$IP6 $DEST_IP" 79 80 # Setup random UDP port src range 81 pg_set $dev "flag UDPSRC_RND" 82 pg_set $dev "udp_src_min $UDP_MIN" 83 pg_set $dev "udp_src_max $UDP_MAX" 84done 85 86# start_run 87echo "Running... ctrl^C to stop" >&2 88pg_ctrl "start" 89echo "Done" >&2 90 91# Print results 92for ((i = 0; i < $THREADS; i++)); do 93 thread=${cpu_array[$((i+F_THREAD))]} 94 dev=${DEV}@${thread} 95 echo "Device: $dev" 96 cat /proc/net/pktgen/$dev | grep -A2 "Result:" 97done 98