1#!/bin/bash 2# Copyright (c) 2016 Microsemi. All Rights Reserved. 3# 4# This program is free software; you can redistribute it and/or 5# modify it under the terms of the GNU General Public License as 6# published by the Free Software Foundation; either version 2 of 7# the License, or (at your option) any later version. 8# 9# This program is distributed in the hope that it would be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# Author: Logan Gunthorpe <logang@deltatee.com> 15 16REMOTE_HOST= 17LIST_DEVS=FALSE 18 19DEBUGFS=${DEBUGFS-/sys/kernel/debug} 20 21PERF_RUN_ORDER=32 22MAX_MW_SIZE=0 23RUN_DMA_TESTS= 24DONT_CLEANUP= 25MW_SIZE=65536 26 27function show_help() 28{ 29 echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV" 30 echo "Run tests on a pair of NTB endpoints." 31 echo 32 echo "If the NTB device loops back to the same host then," 33 echo "just specifying the two PCI ids on the command line is" 34 echo "sufficient. Otherwise, if the NTB link spans two hosts" 35 echo "use the -r option to specify the hostname for the remote" 36 echo "device. SSH will then be used to test the remote side." 37 echo "An SSH key between the root users of the host would then" 38 echo "be highly recommended." 39 echo 40 echo "Options:" 41 echo " -C don't cleanup ntb modules on exit" 42 echo " -h show this help message" 43 echo " -l list available local and remote PCI ids" 44 echo " -r REMOTE_HOST specify the remote's hostname to connect" 45 echo " to for the test (using ssh)" 46 echo " -m MW_SIZE memory window size for ntb_tool" 47 echo " (default: $MW_SIZE)" 48 echo " -d run dma tests for ntb_perf" 49 echo " -p ORDER total data order for ntb_perf" 50 echo " (default: $PERF_RUN_ORDER)" 51 echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf" 52 echo 53} 54 55function parse_args() 56{ 57 OPTIND=0 58 while getopts "b:Cdhlm:r:p:w:" opt; do 59 case "$opt" in 60 C) DONT_CLEANUP=1 ;; 61 d) RUN_DMA_TESTS=1 ;; 62 h) show_help; exit 0 ;; 63 l) LIST_DEVS=TRUE ;; 64 m) MW_SIZE=${OPTARG} ;; 65 r) REMOTE_HOST=${OPTARG} ;; 66 p) PERF_RUN_ORDER=${OPTARG} ;; 67 w) MAX_MW_SIZE=${OPTARG} ;; 68 \?) 69 echo "Invalid option: -$OPTARG" >&2 70 exit 1 71 ;; 72 esac 73 done 74} 75 76parse_args "$@" 77shift $((OPTIND-1)) 78LOCAL_DEV=$1 79shift 80parse_args "$@" 81shift $((OPTIND-1)) 82REMOTE_DEV=$1 83shift 84parse_args "$@" 85 86set -e 87 88function _modprobe() 89{ 90 modprobe "$@" 91 92 if [[ "$REMOTE_HOST" != "" ]]; then 93 ssh "$REMOTE_HOST" modprobe "$@" 94 fi 95} 96 97function split_remote() 98{ 99 VPATH=$1 100 REMOTE= 101 102 if [[ "$VPATH" == *":/"* ]]; then 103 REMOTE=${VPATH%%:*} 104 VPATH=${VPATH#*:} 105 fi 106} 107 108function read_file() 109{ 110 split_remote $1 111 if [[ "$REMOTE" != "" ]]; then 112 ssh "$REMOTE" cat "$VPATH" 113 else 114 cat "$VPATH" 115 fi 116} 117 118function write_file() 119{ 120 split_remote $2 121 VALUE=$1 122 123 if [[ "$REMOTE" != "" ]]; then 124 ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\"" 125 else 126 echo "$VALUE" > "$VPATH" 127 fi 128} 129 130function check_file() 131{ 132 split_remote $1 133 134 if [[ "$REMOTE" != "" ]]; then 135 ssh "$REMOTE" "[[ -e ${VPATH} ]]" 136 else 137 [[ -e ${VPATH} ]] 138 fi 139} 140 141function subdirname() 142{ 143 echo $(basename $(dirname $1)) 2> /dev/null 144} 145 146function find_pidx() 147{ 148 PORT=$1 149 PPATH=$2 150 151 for ((i = 0; i < 64; i++)); do 152 PEER_DIR="$PPATH/peer$i" 153 154 check_file ${PEER_DIR} || break 155 156 PEER_PORT=$(read_file "${PEER_DIR}/port") 157 if [[ ${PORT} -eq $PEER_PORT ]]; then 158 echo $i 159 return 0 160 fi 161 done 162 163 return 1 164} 165 166function port_test() 167{ 168 LOC=$1 169 REM=$2 170 171 echo "Running port tests on: $(basename $LOC) / $(basename $REM)" 172 173 LOCAL_PORT=$(read_file "$LOC/port") 174 REMOTE_PORT=$(read_file "$REM/port") 175 176 LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC") 177 REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM") 178 179 echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host" 180 echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host" 181 182 echo " Passed" 183} 184 185function link_test() 186{ 187 LOC=$1 188 REM=$2 189 EXP=0 190 191 echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)" 192 193 if ! write_file "N" "$LOC/../link" 2> /dev/null; then 194 echo " Unsupported" 195 return 196 fi 197 198 write_file "N" "$LOC/link_event" 199 200 if [[ $(read_file "$REM/link") != "N" ]]; then 201 echo "Expected link to be down in $REM/link" >&2 202 exit -1 203 fi 204 205 write_file "Y" "$LOC/../link" 206 207 echo " Passed" 208} 209 210function doorbell_test() 211{ 212 LOC=$1 213 REM=$2 214 EXP=0 215 216 echo "Running db tests on: $(basename $LOC) / $(basename $REM)" 217 218 DB_VALID_MASK=$(read_file "$LOC/db_valid_mask") 219 220 write_file "c $DB_VALID_MASK" "$REM/db" 221 222 for ((i = 0; i < 64; i++)); do 223 DB=$(read_file "$REM/db") 224 if [[ "$DB" -ne "$EXP" ]]; then 225 echo "Doorbell doesn't match expected value $EXP " \ 226 "in $REM/db" >&2 227 exit -1 228 fi 229 230 let "MASK = (1 << $i) & $DB_VALID_MASK" || true 231 let "EXP = $EXP | $MASK" || true 232 233 write_file "s $MASK" "$LOC/peer_db" 234 done 235 236 write_file "c $DB_VALID_MASK" "$REM/db_mask" 237 write_file $DB_VALID_MASK "$REM/db_event" 238 write_file "s $DB_VALID_MASK" "$REM/db_mask" 239 240 write_file "c $DB_VALID_MASK" "$REM/db" 241 242 echo " Passed" 243} 244 245function get_files_count() 246{ 247 NAME=$1 248 LOC=$2 249 250 split_remote $LOC 251 252 if [[ "$REMOTE" == "" ]]; then 253 echo $(ls -1 "$LOC"/${NAME}* 2>/dev/null | wc -l) 254 else 255 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \ 256 wc -l" 2> /dev/null) 257 fi 258} 259 260function scratchpad_test() 261{ 262 LOC=$1 263 REM=$2 264 265 echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)" 266 267 CNT=$(get_files_count "spad" "$LOC") 268 269 if [[ $CNT -eq 0 ]]; then 270 echo " Unsupported" 271 return 272 fi 273 274 for ((i = 0; i < $CNT; i++)); do 275 VAL=$RANDOM 276 write_file "$VAL" "$LOC/spad$i" 277 RVAL=$(read_file "$REM/../spad$i") 278 279 if [[ "$VAL" -ne "$RVAL" ]]; then 280 echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2 281 exit -1 282 fi 283 done 284 285 echo " Passed" 286} 287 288function message_test() 289{ 290 LOC=$1 291 REM=$2 292 293 echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)" 294 295 CNT=$(get_files_count "msg" "$LOC") 296 297 if [[ $CNT -eq 0 ]]; then 298 echo " Unsupported" 299 return 300 fi 301 302 MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits") 303 MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits") 304 305 write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts" 306 write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts" 307 308 for ((i = 0; i < $CNT; i++)); do 309 VAL=$RANDOM 310 write_file "$VAL" "$LOC/msg$i" 311 RVAL=$(read_file "$REM/../msg$i") 312 313 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then 314 echo "Message $i value $RVAL doesn't match $VAL" >&2 315 exit -1 316 fi 317 done 318 319 echo " Passed" 320} 321 322function get_number() 323{ 324 KEY=$1 325 326 sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p" 327} 328 329function mw_alloc() 330{ 331 IDX=$1 332 LOC=$2 333 REM=$3 334 335 write_file $MW_SIZE "$LOC/mw_trans$IDX" 336 337 INB_MW=$(read_file "$LOC/mw_trans$IDX") 338 MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size") 339 MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address") 340 341 write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX" 342 343 if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then 344 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE" 345 fi 346} 347 348function write_mw() 349{ 350 split_remote $2 351 352 if [[ "$REMOTE" != "" ]]; then 353 ssh "$REMOTE" \ 354 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true 355 else 356 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true 357 fi 358} 359 360function mw_check() 361{ 362 IDX=$1 363 LOC=$2 364 REM=$3 365 366 write_mw "$LOC/mw$IDX" 367 368 split_remote "$LOC/mw$IDX" 369 if [[ "$REMOTE" == "" ]]; then 370 A=$VPATH 371 else 372 A=/tmp/ntb_test.$$.A 373 ssh "$REMOTE" cat "$VPATH" > "$A" 374 fi 375 376 split_remote "$REM/peer_mw$IDX" 377 if [[ "$REMOTE" == "" ]]; then 378 B=$VPATH 379 else 380 B=/tmp/ntb_test.$$.B 381 ssh "$REMOTE" cat "$VPATH" > "$B" 382 fi 383 384 cmp -n $MW_ALIGNED_SIZE "$A" "$B" 385 if [[ $? != 0 ]]; then 386 echo "Memory window $MW did not match!" >&2 387 fi 388 389 if [[ "$A" == "/tmp/*" ]]; then 390 rm "$A" 391 fi 392 393 if [[ "$B" == "/tmp/*" ]]; then 394 rm "$B" 395 fi 396} 397 398function mw_free() 399{ 400 IDX=$1 401 LOC=$2 402 REM=$3 403 404 write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX" 405 406 write_file 0 "$LOC/mw_trans$IDX" 407} 408 409function mw_test() 410{ 411 LOC=$1 412 REM=$2 413 414 CNT=$(get_files_count "mw_trans" "$LOC") 415 416 for ((i = 0; i < $CNT; i++)); do 417 echo "Running mw$i tests on: $(subdirname $LOC) / " \ 418 "$(subdirname $REM)" 419 420 mw_alloc $i $LOC $REM 421 422 mw_check $i $LOC $REM 423 424 mw_free $i $LOC $REM 425 426 echo " Passed" 427 done 428 429} 430 431function pingpong_test() 432{ 433 LOC=$1 434 REM=$2 435 436 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)" 437 438 LOC_START=$(read_file "$LOC/count") 439 REM_START=$(read_file "$REM/count") 440 441 sleep 7 442 443 LOC_END=$(read_file "$LOC/count") 444 REM_END=$(read_file "$REM/count") 445 446 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then 447 echo "Ping pong counter not incrementing!" >&2 448 exit 1 449 fi 450 451 echo " Passed" 452} 453 454function perf_test() 455{ 456 USE_DMA=$1 457 458 if [[ $USE_DMA == "1" ]]; then 459 WITH="with" 460 else 461 WITH="without" 462 fi 463 464 _modprobe ntb_perf total_order=$PERF_RUN_ORDER \ 465 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA 466 467 echo "Running local perf test $WITH DMA" 468 write_file "$LOCAL_PIDX" "$LOCAL_PERF/run" 469 echo -n " " 470 read_file "$LOCAL_PERF/run" 471 echo " Passed" 472 473 echo "Running remote perf test $WITH DMA" 474 write_file "$REMOTE_PIDX" "$REMOTE_PERF/run" 475 echo -n " " 476 read_file "$REMOTE_PERF/run" 477 echo " Passed" 478 479 _modprobe -r ntb_perf 480} 481 482function ntb_tool_tests() 483{ 484 LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV" 485 REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV" 486 487 echo "Starting ntb_tool tests..." 488 489 _modprobe ntb_tool 490 491 port_test "$LOCAL_TOOL" "$REMOTE_TOOL" 492 493 LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX" 494 REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX" 495 496 link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 497 link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 498 499 #Ensure the link is up on both sides before continuing 500 write_file "Y" "$LOCAL_PEER_TOOL/link_event" 501 write_file "Y" "$REMOTE_PEER_TOOL/link_event" 502 503 doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL" 504 doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL" 505 506 scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 507 scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 508 509 message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 510 message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 511 512 mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL" 513 mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL" 514 515 _modprobe -r ntb_tool 516} 517 518function ntb_pingpong_tests() 519{ 520 LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV" 521 REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV" 522 523 echo "Starting ntb_pingpong tests..." 524 525 _modprobe ntb_pingpong 526 527 pingpong_test $LOCAL_PP $REMOTE_PP 528 529 _modprobe -r ntb_pingpong 530} 531 532function ntb_perf_tests() 533{ 534 LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV" 535 REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV" 536 537 echo "Starting ntb_perf tests..." 538 539 perf_test 0 540 541 if [[ $RUN_DMA_TESTS ]]; then 542 perf_test 1 543 fi 544} 545 546function cleanup() 547{ 548 set +e 549 _modprobe -r ntb_tool 2> /dev/null 550 _modprobe -r ntb_perf 2> /dev/null 551 _modprobe -r ntb_pingpong 2> /dev/null 552 _modprobe -r ntb_transport 2> /dev/null 553 set -e 554} 555 556cleanup 557 558if ! [[ $$DONT_CLEANUP ]]; then 559 trap cleanup EXIT 560fi 561 562if [ "$(id -u)" != "0" ]; then 563 echo "This script must be run as root" 1>&2 564 exit 1 565fi 566 567if [[ "$LIST_DEVS" == TRUE ]]; then 568 echo "Local Devices:" 569 ls -1 /sys/bus/ntb/devices 570 echo 571 572 if [[ "$REMOTE_HOST" != "" ]]; then 573 echo "Remote Devices:" 574 ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices 575 fi 576 577 exit 0 578fi 579 580if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then 581 show_help 582 exit 1 583fi 584 585ntb_tool_tests 586echo 587ntb_pingpong_tests 588echo 589ntb_perf_tests 590echo 591