1// SPDX-License-Identifier: GPL-2.0-only 2// Copyright (C) 2015-2019 ARM Limited. 3// Original author: Dave Martin <Dave.Martin@arm.com> 4// 5// Simple Scalable Vector Extension context switch test 6// Repeatedly writes unique test patterns into each SVE register 7// and reads them back to verify integrity. 8// 9// for x in `seq 1 NR_CPUS`; do sve-test & pids=$pids\ $! ; done 10// (leave it running for as long as you want...) 11// kill $pids 12 13#include <asm/unistd.h> 14#include "assembler.h" 15#include "asm-offsets.h" 16 17#define NZR 32 18#define NPR 16 19#define MAXVL_B (2048 / 8) 20 21.arch_extension sve 22 23.macro _sve_ldr_v zt, xn 24 ldr z\zt, [x\xn] 25.endm 26 27.macro _sve_str_v zt, xn 28 str z\zt, [x\xn] 29.endm 30 31.macro _sve_ldr_p pt, xn 32 ldr p\pt, [x\xn] 33.endm 34 35.macro _sve_str_p pt, xn 36 str p\pt, [x\xn] 37.endm 38 39// Generate accessor functions to read/write programmatically selected 40// SVE registers. 41// x0 is the register index to access 42// x1 is the memory address to read from (getz,setp) or store to (setz,setp) 43// All clobber x0-x2 44define_accessor setz, NZR, _sve_ldr_v 45define_accessor getz, NZR, _sve_str_v 46define_accessor setp, NPR, _sve_ldr_p 47define_accessor getp, NPR, _sve_str_p 48 49// Declare some storate space to shadow the SVE register contents: 50.pushsection .text 51.data 52.align 4 53zref: 54 .space MAXVL_B * NZR 55pref: 56 .space MAXVL_B / 8 * NPR 57ffrref: 58 .space MAXVL_B / 8 59scratch: 60 .space MAXVL_B 61.popsection 62 63// Generate a test pattern for storage in SVE registers 64// x0: pid (16 bits) 65// x1: register number (6 bits) 66// x2: generation (4 bits) 67 68// These values are used to constuct a 32-bit pattern that is repeated in the 69// scratch buffer as many times as will fit: 70// bits 31:28 generation number (increments once per test_loop) 71// bits 27:22 32-bit lane index 72// bits 21:16 register number 73// bits 15: 0 pid 74 75function pattern 76 orr w1, w0, w1, lsl #16 77 orr w2, w1, w2, lsl #28 78 79 ldr x0, =scratch 80 mov w1, #MAXVL_B / 4 81 820: str w2, [x0], #4 83 add w2, w2, #(1 << 22) 84 subs w1, w1, #1 85 bne 0b 86 87 ret 88endfunction 89 90// Get the address of shadow data for SVE Z-register Z<xn> 91.macro _adrz xd, xn, nrtmp 92 ldr \xd, =zref 93 rdvl x\nrtmp, #1 94 madd \xd, x\nrtmp, \xn, \xd 95.endm 96 97// Get the address of shadow data for SVE P-register P<xn - NZR> 98.macro _adrp xd, xn, nrtmp 99 ldr \xd, =pref 100 rdvl x\nrtmp, #1 101 lsr x\nrtmp, x\nrtmp, #3 102 sub \xn, \xn, #NZR 103 madd \xd, x\nrtmp, \xn, \xd 104.endm 105 106// Set up test pattern in a SVE Z-register 107// x0: pid 108// x1: register number 109// x2: generation 110function setup_zreg 111 mov x4, x30 112 113 mov x6, x1 114 bl pattern 115 _adrz x0, x6, 2 116 mov x5, x0 117 ldr x1, =scratch 118 bl memcpy 119 120 mov x0, x6 121 mov x1, x5 122 bl setz 123 124 ret x4 125endfunction 126 127// Set up test pattern in a SVE P-register 128// x0: pid 129// x1: register number 130// x2: generation 131function setup_preg 132 mov x4, x30 133 134 mov x6, x1 135 bl pattern 136 _adrp x0, x6, 2 137 mov x5, x0 138 ldr x1, =scratch 139 bl memcpy 140 141 mov x0, x6 142 mov x1, x5 143 bl setp 144 145 ret x4 146endfunction 147 148// Set up test pattern in the FFR 149// x0: pid 150// x2: generation 151// 152// We need to generate a canonical FFR value, which consists of a number of 153// low "1" bits, followed by a number of zeros. This gives us 17 unique values 154// per 16 bits of FFR, so we create a 4 bit signature out of the PID and 155// generation, and use that as the initial number of ones in the pattern. 156// We fill the upper lanes of FFR with zeros. 157// Beware: corrupts P0. 158function setup_ffr 159 mov x4, x30 160 161 and w0, w0, #0x3 162 bfi w0, w2, #2, #2 163 mov w1, #1 164 lsl w1, w1, w0 165 sub w1, w1, #1 166 167 ldr x0, =ffrref 168 strh w1, [x0], 2 169 rdvl x1, #1 170 lsr x1, x1, #3 171 sub x1, x1, #2 172 bl memclr 173 174 mov x0, #0 175 ldr x1, =ffrref 176 bl setp 177 178 wrffr p0.b 179 180 ret x4 181endfunction 182 183// Trivial memory compare: compare x2 bytes starting at address x0 with 184// bytes starting at address x1. 185// Returns only if all bytes match; otherwise, the program is aborted. 186// Clobbers x0-x5. 187function memcmp 188 cbz x2, 2f 189 190 stp x0, x1, [sp, #-0x20]! 191 str x2, [sp, #0x10] 192 193 mov x5, #0 1940: ldrb w3, [x0, x5] 195 ldrb w4, [x1, x5] 196 add x5, x5, #1 197 cmp w3, w4 198 b.ne 1f 199 subs x2, x2, #1 200 b.ne 0b 201 2021: ldr x2, [sp, #0x10] 203 ldp x0, x1, [sp], #0x20 204 b.ne barf 205 2062: ret 207endfunction 208 209// Verify that a SVE Z-register matches its shadow in memory, else abort 210// x0: reg number 211// Clobbers x0-x7. 212function check_zreg 213 mov x3, x30 214 215 _adrz x5, x0, 6 216 mov x4, x0 217 ldr x7, =scratch 218 219 mov x0, x7 220 mov x1, x6 221 bl memfill_ae 222 223 mov x0, x4 224 mov x1, x7 225 bl getz 226 227 mov x0, x5 228 mov x1, x7 229 mov x2, x6 230 mov x30, x3 231 b memcmp 232endfunction 233 234// Verify that a SVE P-register matches its shadow in memory, else abort 235// x0: reg number 236// Clobbers x0-x7. 237function check_preg 238 mov x3, x30 239 240 _adrp x5, x0, 6 241 mov x4, x0 242 ldr x7, =scratch 243 244 mov x0, x7 245 mov x1, x6 246 bl memfill_ae 247 248 mov x0, x4 249 mov x1, x7 250 bl getp 251 252 mov x0, x5 253 mov x1, x7 254 mov x2, x6 255 mov x30, x3 256 b memcmp 257endfunction 258 259// Verify that the FFR matches its shadow in memory, else abort 260// Beware -- corrupts P0. 261// Clobbers x0-x5. 262function check_ffr 263 mov x3, x30 264 265 ldr x4, =scratch 266 rdvl x5, #1 267 lsr x5, x5, #3 268 269 mov x0, x4 270 mov x1, x5 271 bl memfill_ae 272 273 rdffr p0.b 274 mov x0, #0 275 mov x1, x4 276 bl getp 277 278 ldr x0, =ffrref 279 mov x1, x4 280 mov x2, x5 281 mov x30, x3 282 b memcmp 283endfunction 284 285// Any SVE register modified here can cause corruption in the main 286// thread -- but *only* the registers modified here. 287function irritator_handler 288 // Increment the irritation signal count (x23): 289 ldr x0, [x2, #ucontext_regs + 8 * 23] 290 add x0, x0, #1 291 str x0, [x2, #ucontext_regs + 8 * 23] 292 293 // Corrupt some random Z-regs 294 adr x0, .text + (irritator_handler - .text) / 16 * 16 295 movi v0.8b, #1 296 movi v9.16b, #2 297 movi v31.8b, #3 298 // And P0 299 rdffr p0.b 300 // And FFR 301 wrffr p15.b 302 303 ret 304endfunction 305 306function terminate_handler 307 mov w21, w0 308 mov x20, x2 309 310 puts "Terminated by signal " 311 mov w0, w21 312 bl putdec 313 puts ", no error, iterations=" 314 ldr x0, [x20, #ucontext_regs + 8 * 22] 315 bl putdec 316 puts ", signals=" 317 ldr x0, [x20, #ucontext_regs + 8 * 23] 318 bl putdecn 319 320 mov x0, #0 321 mov x8, #__NR_exit 322 svc #0 323endfunction 324 325// w0: signal number 326// x1: sa_action 327// w2: sa_flags 328// Clobbers x0-x6,x8 329function setsignal 330 str x30, [sp, #-((sa_sz + 15) / 16 * 16 + 16)]! 331 332 mov w4, w0 333 mov x5, x1 334 mov w6, w2 335 336 add x0, sp, #16 337 mov x1, #sa_sz 338 bl memclr 339 340 mov w0, w4 341 add x1, sp, #16 342 str w6, [x1, #sa_flags] 343 str x5, [x1, #sa_handler] 344 mov x2, #0 345 mov x3, #sa_mask_sz 346 mov x8, #__NR_rt_sigaction 347 svc #0 348 349 cbz w0, 1f 350 351 puts "sigaction failure\n" 352 b .Labort 353 3541: ldr x30, [sp], #((sa_sz + 15) / 16 * 16 + 16) 355 ret 356endfunction 357 358// Main program entry point 359.globl _start 360function _start 361_start: 362 // Sanity-check and report the vector length 363 364 rdvl x19, #8 365 cmp x19, #128 366 b.lo 1f 367 cmp x19, #2048 368 b.hi 1f 369 tst x19, #(8 - 1) 370 b.eq 2f 371 3721: puts "Bad vector length: " 373 mov x0, x19 374 bl putdecn 375 b .Labort 376 3772: puts "Vector length:\t" 378 mov x0, x19 379 bl putdec 380 puts " bits\n" 381 382 // Obtain our PID, to ensure test pattern uniqueness between processes 383 384 mov x8, #__NR_getpid 385 svc #0 386 mov x20, x0 387 388 puts "PID:\t" 389 mov x0, x20 390 bl putdecn 391 392 mov x23, #0 // Irritation signal count 393 394 mov w0, #SIGINT 395 adr x1, terminate_handler 396 mov w2, #SA_SIGINFO 397 bl setsignal 398 399 mov w0, #SIGTERM 400 adr x1, terminate_handler 401 mov w2, #SA_SIGINFO 402 bl setsignal 403 404 mov w0, #SIGUSR1 405 adr x1, irritator_handler 406 mov w2, #SA_SIGINFO 407 orr w2, w2, #SA_NODEFER 408 bl setsignal 409 410 mov x22, #0 // generation number, increments per iteration 411.Ltest_loop: 412 rdvl x0, #8 413 cmp x0, x19 414 b.ne vl_barf 415 416 mov x21, #0 // Set up Z-regs & shadow with test pattern 4170: mov x0, x20 418 mov x1, x21 419 and x2, x22, #0xf 420 bl setup_zreg 421 add x21, x21, #1 422 cmp x21, #NZR 423 b.lo 0b 424 425 mov x0, x20 // Set up FFR & shadow with test pattern 426 mov x1, #NZR + NPR 427 and x2, x22, #0xf 428 bl setup_ffr 429 4300: mov x0, x20 // Set up P-regs & shadow with test pattern 431 mov x1, x21 432 and x2, x22, #0xf 433 bl setup_preg 434 add x21, x21, #1 435 cmp x21, #NZR + NPR 436 b.lo 0b 437 438// Can't do this when SVE state is volatile across SVC: 439// mov x8, #__NR_sched_yield // Encourage preemption 440// svc #0 441 442 mov x21, #0 4430: mov x0, x21 444 bl check_zreg 445 add x21, x21, #1 446 cmp x21, #NZR 447 b.lo 0b 448 4490: mov x0, x21 450 bl check_preg 451 add x21, x21, #1 452 cmp x21, #NZR + NPR 453 b.lo 0b 454 455 bl check_ffr 456 457 add x22, x22, #1 458 b .Ltest_loop 459 460.Labort: 461 mov x0, #0 462 mov x1, #SIGABRT 463 mov x8, #__NR_kill 464 svc #0 465endfunction 466 467function barf 468// fpsimd.c acitivty log dump hack 469// ldr w0, =0xdeadc0de 470// mov w8, #__NR_exit 471// svc #0 472// end hack 473 mov x10, x0 // expected data 474 mov x11, x1 // actual data 475 mov x12, x2 // data size 476 477 puts "Mismatch: PID=" 478 mov x0, x20 479 bl putdec 480 puts ", iteration=" 481 mov x0, x22 482 bl putdec 483 puts ", reg=" 484 mov x0, x21 485 bl putdecn 486 puts "\tExpected [" 487 mov x0, x10 488 mov x1, x12 489 bl dumphex 490 puts "]\n\tGot [" 491 mov x0, x11 492 mov x1, x12 493 bl dumphex 494 puts "]\n" 495 496 mov x8, #__NR_getpid 497 svc #0 498// fpsimd.c acitivty log dump hack 499// ldr w0, =0xdeadc0de 500// mov w8, #__NR_exit 501// svc #0 502// ^ end of hack 503 mov x1, #SIGABRT 504 mov x8, #__NR_kill 505 svc #0 506// mov x8, #__NR_exit 507// mov x1, #1 508// svc #0 509endfunction 510 511function vl_barf 512 mov x10, x0 513 514 puts "Bad active VL: " 515 mov x0, x10 516 bl putdecn 517 518 mov x8, #__NR_exit 519 mov x1, #1 520 svc #0 521endfunction 522