1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0-or-later 3# Copyright (c) 2016 Microsemi. All Rights Reserved. 4# 5# Author: Logan Gunthorpe <logang@deltatee.com> 6 7REMOTE_HOST= 8LIST_DEVS=FALSE 9 10DEBUGFS=${DEBUGFS-/sys/kernel/debug} 11 12PERF_RUN_ORDER=32 13MAX_MW_SIZE=0 14RUN_DMA_TESTS= 15DONT_CLEANUP= 16MW_SIZE=65536 17 18function show_help() 19{ 20 echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV" 21 echo "Run tests on a pair of NTB endpoints." 22 echo 23 echo "If the NTB device loops back to the same host then," 24 echo "just specifying the two PCI ids on the command line is" 25 echo "sufficient. Otherwise, if the NTB link spans two hosts" 26 echo "use the -r option to specify the hostname for the remote" 27 echo "device. SSH will then be used to test the remote side." 28 echo "An SSH key between the root users of the host would then" 29 echo "be highly recommended." 30 echo 31 echo "Options:" 32 echo " -C don't cleanup ntb modules on exit" 33 echo " -h show this help message" 34 echo " -l list available local and remote PCI ids" 35 echo " -r REMOTE_HOST specify the remote's hostname to connect" 36 echo " to for the test (using ssh)" 37 echo " -m MW_SIZE memory window size for ntb_tool" 38 echo " (default: $MW_SIZE)" 39 echo " -d run dma tests for ntb_perf" 40 echo " -p ORDER total data order for ntb_perf" 41 echo " (default: $PERF_RUN_ORDER)" 42 echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf" 43 echo 44} 45 46function parse_args() 47{ 48 OPTIND=0 49 while getopts "b:Cdhlm:r:p:w:" opt; do 50 case "$opt" in 51 C) DONT_CLEANUP=1 ;; 52 d) RUN_DMA_TESTS=1 ;; 53 h) show_help; exit 0 ;; 54 l) LIST_DEVS=TRUE ;; 55 m) MW_SIZE=${OPTARG} ;; 56 r) REMOTE_HOST=${OPTARG} ;; 57 p) PERF_RUN_ORDER=${OPTARG} ;; 58 w) MAX_MW_SIZE=${OPTARG} ;; 59 \?) 60 echo "Invalid option: -$OPTARG" >&2 61 exit 1 62 ;; 63 esac 64 done 65} 66 67parse_args "$@" 68shift $((OPTIND-1)) 69LOCAL_DEV=$1 70shift 71parse_args "$@" 72shift $((OPTIND-1)) 73REMOTE_DEV=$1 74shift 75parse_args "$@" 76 77set -e 78 79function _modprobe() 80{ 81 modprobe "$@" || return 1 82 83 if [[ "$REMOTE_HOST" != "" ]]; then 84 ssh "$REMOTE_HOST" modprobe "$@" || return 1 85 fi 86} 87 88function split_remote() 89{ 90 VPATH=$1 91 REMOTE= 92 93 if [[ "$VPATH" == *":/"* ]]; then 94 REMOTE=${VPATH%%:*} 95 VPATH=${VPATH#*:} 96 fi 97} 98 99function read_file() 100{ 101 split_remote $1 102 if [[ "$REMOTE" != "" ]]; then 103 ssh "$REMOTE" cat "$VPATH" 104 else 105 cat "$VPATH" 106 fi 107} 108 109function write_file() 110{ 111 split_remote $2 112 VALUE=$1 113 114 if [[ "$REMOTE" != "" ]]; then 115 ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\"" 116 else 117 echo "$VALUE" > "$VPATH" 118 fi 119} 120 121function check_file() 122{ 123 split_remote $1 124 125 if [[ "$REMOTE" != "" ]]; then 126 ssh "$REMOTE" "[[ -e ${VPATH} ]]" 127 else 128 [[ -e ${VPATH} ]] 129 fi 130} 131 132function subdirname() 133{ 134 echo $(basename $(dirname $1)) 2> /dev/null 135} 136 137function find_pidx() 138{ 139 PORT=$1 140 PPATH=$2 141 142 for ((i = 0; i < 64; i++)); do 143 PEER_DIR="$PPATH/peer$i" 144 145 check_file ${PEER_DIR} || break 146 147 PEER_PORT=$(read_file "${PEER_DIR}/port") 148 if [[ ${PORT} -eq $PEER_PORT ]]; then 149 echo $i 150 return 0 151 fi 152 done 153 154 return 1 155} 156 157function port_test() 158{ 159 LOC=$1 160 REM=$2 161 162 echo "Running port tests on: $(basename $LOC) / $(basename $REM)" 163 164 LOCAL_PORT=$(read_file "$LOC/port") 165 REMOTE_PORT=$(read_file "$REM/port") 166 167 LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC") 168 REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM") 169 170 echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host" 171 echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host" 172 173 echo " Passed" 174} 175 176function link_test() 177{ 178 LOC=$1 179 REM=$2 180 EXP=0 181 182 echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)" 183 184 if ! write_file "N" "$LOC/../link" 2> /dev/null; then 185 echo " Unsupported" 186 return 187 fi 188 189 write_file "N" "$LOC/link_event" 190 191 if [[ $(read_file "$REM/link") != "N" ]]; then 192 echo "Expected link to be down in $REM/link" >&2 193 exit -1 194 fi 195 196 write_file "Y" "$LOC/../link" 197 198 echo " Passed" 199} 200 201function doorbell_test() 202{ 203 LOC=$1 204 REM=$2 205 EXP=0 206 207 echo "Running db tests on: $(basename $LOC) / $(basename $REM)" 208 209 DB_VALID_MASK=$(read_file "$LOC/db_valid_mask") 210 211 write_file "c $DB_VALID_MASK" "$REM/db" 212 213 for ((i = 0; i < 64; i++)); do 214 DB=$(read_file "$REM/db") 215 if [[ "$DB" -ne "$EXP" ]]; then 216 echo "Doorbell doesn't match expected value $EXP " \ 217 "in $REM/db" >&2 218 exit -1 219 fi 220 221 let "MASK = (1 << $i) & $DB_VALID_MASK" || true 222 let "EXP = $EXP | $MASK" || true 223 224 write_file "s $MASK" "$LOC/peer_db" 225 done 226 227 write_file "c $DB_VALID_MASK" "$REM/db_mask" 228 write_file $DB_VALID_MASK "$REM/db_event" 229 write_file "s $DB_VALID_MASK" "$REM/db_mask" 230 231 write_file "c $DB_VALID_MASK" "$REM/db" 232 233 echo " Passed" 234} 235 236function get_files_count() 237{ 238 NAME=$1 239 LOC=$2 240 241 split_remote $LOC 242 243 if [[ "$REMOTE" == "" ]]; then 244 echo $(ls -1 "$LOC"/${NAME}* 2>/dev/null | wc -l) 245 else 246 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \ 247 wc -l" 2> /dev/null) 248 fi 249} 250 251function scratchpad_test() 252{ 253 LOC=$1 254 REM=$2 255 256 echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)" 257 258 CNT=$(get_files_count "spad" "$LOC") 259 260 if [[ $CNT -eq 0 ]]; then 261 echo " Unsupported" 262 return 263 fi 264 265 for ((i = 0; i < $CNT; i++)); do 266 VAL=$RANDOM 267 write_file "$VAL" "$LOC/spad$i" 268 RVAL=$(read_file "$REM/../spad$i") 269 270 if [[ "$VAL" -ne "$RVAL" ]]; then 271 echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2 272 exit -1 273 fi 274 done 275 276 echo " Passed" 277} 278 279function message_test() 280{ 281 LOC=$1 282 REM=$2 283 284 echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)" 285 286 CNT=$(get_files_count "msg" "$LOC") 287 288 if [[ $CNT -eq 0 ]]; then 289 echo " Unsupported" 290 return 291 fi 292 293 MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits") 294 MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits") 295 296 write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts" 297 write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts" 298 299 for ((i = 0; i < $CNT; i++)); do 300 VAL=$RANDOM 301 write_file "$VAL" "$LOC/msg$i" 302 RVAL=$(read_file "$REM/../msg$i") 303 304 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then 305 echo "Message $i value $RVAL doesn't match $VAL" >&2 306 exit -1 307 fi 308 done 309 310 echo " Passed" 311} 312 313function get_number() 314{ 315 KEY=$1 316 317 sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p" 318} 319 320function mw_alloc() 321{ 322 IDX=$1 323 LOC=$2 324 REM=$3 325 326 write_file $MW_SIZE "$LOC/mw_trans$IDX" 327 328 INB_MW=$(read_file "$LOC/mw_trans$IDX") 329 MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size") 330 MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address") 331 332 write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX" 333 334 if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then 335 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE" 336 fi 337} 338 339function write_mw() 340{ 341 split_remote $2 342 343 if [[ "$REMOTE" != "" ]]; then 344 ssh "$REMOTE" \ 345 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true 346 else 347 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true 348 fi 349} 350 351function mw_check() 352{ 353 IDX=$1 354 LOC=$2 355 REM=$3 356 357 write_mw "$LOC/mw$IDX" 358 359 split_remote "$LOC/mw$IDX" 360 if [[ "$REMOTE" == "" ]]; then 361 A=$VPATH 362 else 363 A=/tmp/ntb_test.$$.A 364 ssh "$REMOTE" cat "$VPATH" > "$A" 365 fi 366 367 split_remote "$REM/peer_mw$IDX" 368 if [[ "$REMOTE" == "" ]]; then 369 B=$VPATH 370 else 371 B=/tmp/ntb_test.$$.B 372 ssh "$REMOTE" cat "$VPATH" > "$B" 373 fi 374 375 cmp -n $MW_ALIGNED_SIZE "$A" "$B" 376 if [[ $? != 0 ]]; then 377 echo "Memory window $MW did not match!" >&2 378 fi 379 380 if [[ "$A" == "/tmp/*" ]]; then 381 rm "$A" 382 fi 383 384 if [[ "$B" == "/tmp/*" ]]; then 385 rm "$B" 386 fi 387} 388 389function mw_free() 390{ 391 IDX=$1 392 LOC=$2 393 REM=$3 394 395 write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX" 396 397 write_file 0 "$LOC/mw_trans$IDX" 398} 399 400function mw_test() 401{ 402 LOC=$1 403 REM=$2 404 405 CNT=$(get_files_count "mw_trans" "$LOC") 406 407 for ((i = 0; i < $CNT; i++)); do 408 echo "Running mw$i tests on: $(subdirname $LOC) / " \ 409 "$(subdirname $REM)" 410 411 mw_alloc $i $LOC $REM 412 413 mw_check $i $LOC $REM 414 415 mw_free $i $LOC $REM 416 417 echo " Passed" 418 done 419 420} 421 422function pingpong_test() 423{ 424 LOC=$1 425 REM=$2 426 427 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)" 428 429 LOC_START=$(read_file "$LOC/count") 430 REM_START=$(read_file "$REM/count") 431 432 sleep 7 433 434 LOC_END=$(read_file "$LOC/count") 435 REM_END=$(read_file "$REM/count") 436 437 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then 438 echo "Ping pong counter not incrementing!" >&2 439 exit 1 440 fi 441 442 echo " Passed" 443} 444 445function msi_test() 446{ 447 LOC=$1 448 REM=$2 449 450 write_file 1 $LOC/ready 451 452 echo "Running MSI interrupt tests on: $(subdirname $LOC) / $(subdirname $REM)" 453 454 CNT=$(read_file "$LOC/count") 455 for ((i = 0; i < $CNT; i++)); do 456 START=$(read_file $REM/../irq${i}_occurrences) 457 write_file $i $LOC/trigger 458 END=$(read_file $REM/../irq${i}_occurrences) 459 460 if [[ $(($END - $START)) != 1 ]]; then 461 echo "MSI did not trigger the interrupt on the remote side!" >&2 462 exit 1 463 fi 464 done 465 466 echo " Passed" 467} 468 469function perf_test() 470{ 471 USE_DMA=$1 472 473 if [[ $USE_DMA == "1" ]]; then 474 WITH="with" 475 else 476 WITH="without" 477 fi 478 479 _modprobe ntb_perf total_order=$PERF_RUN_ORDER \ 480 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA 481 482 echo "Running local perf test $WITH DMA" 483 write_file "$LOCAL_PIDX" "$LOCAL_PERF/run" 484 echo -n " " 485 read_file "$LOCAL_PERF/run" 486 echo " Passed" 487 488 echo "Running remote perf test $WITH DMA" 489 write_file "$REMOTE_PIDX" "$REMOTE_PERF/run" 490 echo -n " " 491 read_file "$REMOTE_PERF/run" 492 echo " Passed" 493 494 _modprobe -r ntb_perf 495} 496 497function ntb_tool_tests() 498{ 499 LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV" 500 REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV" 501 502 echo "Starting ntb_tool tests..." 503 504 _modprobe ntb_tool 505 506 port_test "$LOCAL_TOOL" "$REMOTE_TOOL" 507 508 LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX" 509 REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX" 510 511 link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 512 link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 513 514 #Ensure the link is up on both sides before continuing 515 write_file "Y" "$LOCAL_PEER_TOOL/link_event" 516 write_file "Y" "$REMOTE_PEER_TOOL/link_event" 517 518 doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL" 519 doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL" 520 521 scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 522 scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 523 524 message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 525 message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 526 527 mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 528 mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 529 530 _modprobe -r ntb_tool 531} 532 533function ntb_pingpong_tests() 534{ 535 LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV" 536 REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV" 537 538 echo "Starting ntb_pingpong tests..." 539 540 _modprobe ntb_pingpong 541 542 pingpong_test $LOCAL_PP $REMOTE_PP 543 544 _modprobe -r ntb_pingpong 545} 546 547function ntb_msi_tests() 548{ 549 LOCAL_MSI="$DEBUGFS/ntb_msi_test/$LOCAL_DEV" 550 REMOTE_MSI="$REMOTE_HOST:$DEBUGFS/ntb_msi_test/$REMOTE_DEV" 551 552 echo "Starting ntb_msi_test tests..." 553 554 if ! _modprobe ntb_msi_test 2> /dev/null; then 555 echo " Not doing MSI tests seeing the module is not available." 556 return 557 fi 558 559 port_test $LOCAL_MSI $REMOTE_MSI 560 561 LOCAL_PEER="$LOCAL_MSI/peer$LOCAL_PIDX" 562 REMOTE_PEER="$REMOTE_MSI/peer$REMOTE_PIDX" 563 564 msi_test $LOCAL_PEER $REMOTE_PEER 565 msi_test $REMOTE_PEER $LOCAL_PEER 566 567 _modprobe -r ntb_msi_test 568} 569 570function ntb_perf_tests() 571{ 572 LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV" 573 REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV" 574 575 echo "Starting ntb_perf tests..." 576 577 perf_test 0 578 579 if [[ $RUN_DMA_TESTS ]]; then 580 perf_test 1 581 fi 582} 583 584function cleanup() 585{ 586 set +e 587 _modprobe -r ntb_tool 2> /dev/null 588 _modprobe -r ntb_perf 2> /dev/null 589 _modprobe -r ntb_pingpong 2> /dev/null 590 _modprobe -r ntb_transport 2> /dev/null 591 _modprobe -r ntb_msi_test 2> /dev/null 592 set -e 593} 594 595cleanup 596 597if ! [[ $$DONT_CLEANUP ]]; then 598 trap cleanup EXIT 599fi 600 601if [ "$(id -u)" != "0" ]; then 602 echo "This script must be run as root" 1>&2 603 exit 1 604fi 605 606if [[ "$LIST_DEVS" == TRUE ]]; then 607 echo "Local Devices:" 608 ls -1 /sys/bus/ntb/devices 609 echo 610 611 if [[ "$REMOTE_HOST" != "" ]]; then 612 echo "Remote Devices:" 613 ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices 614 fi 615 616 exit 0 617fi 618 619if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then 620 show_help 621 exit 1 622fi 623 624ntb_tool_tests 625echo 626ntb_pingpong_tests 627echo 628ntb_msi_tests 629echo 630ntb_perf_tests 631echo 632