1/* 2 * arch/xtensa/kernel/align.S 3 * 4 * Handle unalignment and load/store exceptions. 5 * 6 * This file is subject to the terms and conditions of the GNU General 7 * Public License. See the file "COPYING" in the main directory of 8 * this archive for more details. 9 * 10 * Copyright (C) 2001 - 2005 Tensilica, Inc. 11 * Copyright (C) 2014 Cadence Design Systems Inc. 12 * 13 * Rewritten by Chris Zankel <chris@zankel.net> 14 * 15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca> 17 */ 18 19#include <linux/linkage.h> 20#include <asm/current.h> 21#include <asm/asm-offsets.h> 22#include <asm/asmmacro.h> 23#include <asm/processor.h> 24#ifdef CONFIG_MMU 25#include <asm/asm-uaccess.h> 26#endif 27 28#if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE 29#define LOAD_EXCEPTION_HANDLER 30#endif 31 32#if XCHAL_UNALIGNED_STORE_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE 33#define STORE_EXCEPTION_HANDLER 34#endif 35 36#if defined LOAD_EXCEPTION_HANDLER || defined STORE_EXCEPTION_HANDLER 37#define ANY_EXCEPTION_HANDLER 38#endif 39 40#if XCHAL_HAVE_WINDOWED && defined CONFIG_MMU 41#define UNALIGNED_USER_EXCEPTION 42#endif 43 44/* Big and little endian 16-bit values are located in 45 * different halves of a register. HWORD_START helps to 46 * abstract the notion of extracting a 16-bit value from a 47 * register. 48 * We also have to define new shifting instructions because 49 * lsb and msb are on 'opposite' ends in a register for 50 * different endian machines. 51 * 52 * Assume a memory region in ascending address: 53 * 0 1 2 3|4 5 6 7 54 * 55 * When loading one word into a register, the content of that register is: 56 * LE 3 2 1 0, 7 6 5 4 57 * BE 0 1 2 3, 4 5 6 7 58 * 59 * Masking the bits of the higher/lower address means: 60 * LE X X 0 0, 0 0 X X 61 * BE 0 0 X X, X X 0 0 62 * 63 * Shifting to higher/lower addresses, means: 64 * LE shift left / shift right 65 * BE shift right / shift left 66 * 67 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means: 68 * LE mask 0 0 X X / shift left 69 * BE shift left / mask 0 0 X X 70 */ 71 72#if XCHAL_HAVE_BE 73 74#define HWORD_START 16 75#define INSN_OP0 28 76#define INSN_T 24 77#define INSN_OP1 16 78 79.macro __ssa8r r; ssa8l \r; .endm 80.macro __sh r, s; srl \r, \s; .endm 81.macro __sl r, s; sll \r, \s; .endm 82.macro __exth r, s; extui \r, \s, 0, 16; .endm 83.macro __extl r, s; slli \r, \s, 16; .endm 84 85#else 86 87#define HWORD_START 0 88#define INSN_OP0 0 89#define INSN_T 4 90#define INSN_OP1 12 91 92.macro __ssa8r r; ssa8b \r; .endm 93.macro __sh r, s; sll \r, \s; .endm 94.macro __sl r, s; srl \r, \s; .endm 95.macro __exth r, s; slli \r, \s, 16; .endm 96.macro __extl r, s; extui \r, \s, 0, 16; .endm 97 98#endif 99 100/* 101 * xxxx xxxx = imm8 field 102 * yyyy = imm4 field 103 * ssss = s field 104 * tttt = t field 105 * 106 * 16 0 107 * ------------------- 108 * L32I.N yyyy ssss tttt 1000 109 * S32I.N yyyy ssss tttt 1001 110 * 111 * 23 0 112 * ----------------------------- 113 * L8UI xxxx xxxx 0000 ssss tttt 0010 114 * L16UI xxxx xxxx 0001 ssss tttt 0010 115 * L32I xxxx xxxx 0010 ssss tttt 0010 116 * XXX 0011 ssss tttt 0010 117 * XXX 0100 ssss tttt 0010 118 * S16I xxxx xxxx 0101 ssss tttt 0010 119 * S32I xxxx xxxx 0110 ssss tttt 0010 120 * XXX 0111 ssss tttt 0010 121 * XXX 1000 ssss tttt 0010 122 * L16SI xxxx xxxx 1001 ssss tttt 0010 123 * XXX 1010 0010 124 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported 125 * XXX 1100 0010 126 * XXX 1101 0010 127 * XXX 1110 0010 128 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported 129 * ----------------------------- 130 * ^ ^ ^ 131 * sub-opcode (NIBBLE_R) -+ | | 132 * t field (NIBBLE_T) -----------+ | 133 * major opcode (NIBBLE_OP0) --------------+ 134 */ 135 136#define OP0_L32I_N 0x8 /* load immediate narrow */ 137#define OP0_S32I_N 0x9 /* store immediate narrow */ 138#define OP0_LSAI 0x2 /* load/store */ 139#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */ 140#define OP1_SI_BIT 2 /* OP1 bit number for stores */ 141 142#define OP1_L8UI 0x0 143#define OP1_L32I 0x2 144#define OP1_L16UI 0x1 145#define OP1_L16SI 0x9 146#define OP1_L32AI 0xb 147 148#define OP1_S32I 0x6 149#define OP1_S16I 0x5 150#define OP1_S32RI 0xf 151 152/* 153 * Entry condition: 154 * 155 * a0: trashed, original value saved on stack (PT_AREG0) 156 * a1: a1 157 * a2: new stack pointer, original in DEPC 158 * a3: a3 159 * depc: a2, original value saved on stack (PT_DEPC) 160 * excsave_1: dispatch table 161 * 162 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 163 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 164 */ 165 166 .literal_position 167#ifdef CONFIG_XTENSA_LOAD_STORE 168ENTRY(fast_load_store) 169 170 call0 .Lsave_and_load_instruction 171 172 /* Analyze the instruction (load or store?). */ 173 174 extui a0, a4, INSN_OP0, 4 # get insn.op0 nibble 175 176#if XCHAL_HAVE_DENSITY 177 _beqi a0, OP0_L32I_N, 1f # L32I.N, jump 178#endif 179 bnei a0, OP0_LSAI, .Linvalid_instruction 180 /* 'store indicator bit' set, jump */ 181 bbsi.l a4, OP1_SI_BIT + INSN_OP1, .Linvalid_instruction 182 1831: 184 movi a3, ~3 185 and a3, a3, a8 # align memory address 186 187 __ssa8 a8 188 189#ifdef CONFIG_MMU 190 /* l32e can't be used here even when it's available. */ 191 rsr a6, ps 192 bbci.l a6, PS_UM_BIT, 1f # kernel mode 193 movi a5, 8 194 access_ok a3, a5, a6, a2, .Linvalid_instruction 1951: 196#endif 197 l32i a5, a3, 0 198 l32i a6, a3, 4 199 __src_b a3, a5, a6 # a3 has the data word 200 201#if XCHAL_HAVE_DENSITY 202 addi a7, a7, 2 # increment PC (assume 16-bit insn) 203 _beqi a0, OP0_L32I_N, .Lload_w# l32i.n: jump 204 addi a7, a7, 1 205#else 206 addi a7, a7, 3 207#endif 208 209 extui a5, a4, INSN_OP1, 4 210 _beqi a5, OP1_L32I, .Lload_w 211 bnei a5, OP1_L8UI, .Lload16 212 extui a3, a3, 0, 8 213 j .Lload_w 214 215ENDPROC(fast_load_store) 216#endif 217 218/* 219 * Entry condition: 220 * 221 * a0: trashed, original value saved on stack (PT_AREG0) 222 * a1: a1 223 * a2: new stack pointer, original in DEPC 224 * a3: a3 225 * depc: a2, original value saved on stack (PT_DEPC) 226 * excsave_1: dispatch table 227 * 228 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 229 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 230 */ 231 232#ifdef ANY_EXCEPTION_HANDLER 233ENTRY(fast_unaligned) 234 235 call0 .Lsave_and_load_instruction 236 237 /* Analyze the instruction (load or store?). */ 238 239 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble 240 241#if XCHAL_HAVE_DENSITY 242 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump 243 addi a6, a5, -OP0_S32I_N 244 _beqz a6, .Lstore # S32I.N, do a store 245#endif 246 /* 'store indicator bit' not set, jump */ 247 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload 248 249#ifdef STORE_EXCEPTION_HANDLER 250 251 /* Store: Jump to table entry to get the value in the source register.*/ 252 253.Lstore:movi a5, .Lstore_table # table 254 extui a6, a4, INSN_T, 4 # get source register 255 addx8 a5, a6, a5 256 jx a5 # jump into table 257#endif 258#ifdef LOAD_EXCEPTION_HANDLER 259 260 /* Load: Load memory address. */ 261 262.Lload: movi a3, ~3 263 and a3, a3, a8 # align memory address 264 265 __ssa8 a8 266#ifdef UNALIGNED_USER_EXCEPTION 267 addi a3, a3, 8 268 l32e a5, a3, -8 269 l32e a6, a3, -4 270#else 271 l32i a5, a3, 0 272 l32i a6, a3, 4 273#endif 274 __src_b a3, a5, a6 # a3 has the data word 275 276#if XCHAL_HAVE_DENSITY 277 addi a7, a7, 2 # increment PC (assume 16-bit insn) 278 279 extui a5, a4, INSN_OP0, 4 280 _beqi a5, OP0_L32I_N, .Lload_w# l32i.n: jump 281 282 addi a7, a7, 1 283#else 284 addi a7, a7, 3 285#endif 286 287 extui a5, a4, INSN_OP1, 4 288 _beqi a5, OP1_L32I, .Lload_w # l32i: jump 289#endif 290#ifdef LOAD_EXCEPTION_HANDLER 291.Lload16: 292 extui a3, a3, 0, 16 # extract lower 16 bits 293 _beqi a5, OP1_L16UI, .Lload_w 294 addi a5, a5, -OP1_L16SI 295 _bnez a5, .Linvalid_instruction 296 297 /* sign extend value */ 298#if XCHAL_HAVE_SEXT 299 sext a3, a3, 15 300#else 301 slli a3, a3, 16 302 srai a3, a3, 16 303#endif 304 305 /* Set target register. */ 306 307.Lload_w: 308 extui a4, a4, INSN_T, 4 # extract target register 309 movi a5, .Lload_table 310 addx8 a4, a4, a5 311 jx a4 # jump to entry for target register 312 313 .align 8 314.Lload_table: 315 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8 316 mov a1, a3; _j .Lexit; .align 8 # fishy?? 317 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8 318 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8 319 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8 320 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8 321 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8 322 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8 323 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8 324 mov a9, a3 ; _j .Lexit; .align 8 325 mov a10, a3 ; _j .Lexit; .align 8 326 mov a11, a3 ; _j .Lexit; .align 8 327 mov a12, a3 ; _j .Lexit; .align 8 328 mov a13, a3 ; _j .Lexit; .align 8 329 mov a14, a3 ; _j .Lexit; .align 8 330 mov a15, a3 ; _j .Lexit; .align 8 331#endif 332#ifdef STORE_EXCEPTION_HANDLER 333.Lstore_table: 334 l32i a3, a2, PT_AREG0; _j .Lstore_w; .align 8 335 mov a3, a1; _j .Lstore_w; .align 8 # fishy?? 336 l32i a3, a2, PT_AREG2; _j .Lstore_w; .align 8 337 l32i a3, a2, PT_AREG3; _j .Lstore_w; .align 8 338 l32i a3, a2, PT_AREG4; _j .Lstore_w; .align 8 339 l32i a3, a2, PT_AREG5; _j .Lstore_w; .align 8 340 l32i a3, a2, PT_AREG6; _j .Lstore_w; .align 8 341 l32i a3, a2, PT_AREG7; _j .Lstore_w; .align 8 342 l32i a3, a2, PT_AREG8; _j .Lstore_w; .align 8 343 mov a3, a9 ; _j .Lstore_w; .align 8 344 mov a3, a10 ; _j .Lstore_w; .align 8 345 mov a3, a11 ; _j .Lstore_w; .align 8 346 mov a3, a12 ; _j .Lstore_w; .align 8 347 mov a3, a13 ; _j .Lstore_w; .align 8 348 mov a3, a14 ; _j .Lstore_w; .align 8 349 mov a3, a15 ; _j .Lstore_w; .align 8 350#endif 351 352 /* We cannot handle this exception. */ 353 354 .extern _kernel_exception 355.Linvalid_instruction: 356 357 movi a4, 0 358 rsr a3, excsave1 359 s32i a4, a3, EXC_TABLE_FIXUP 360 361 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ 362 363 l32i a0, a2, PT_SAR 364 l32i a8, a2, PT_AREG8 365 l32i a7, a2, PT_AREG7 366 l32i a6, a2, PT_AREG6 367 l32i a5, a2, PT_AREG5 368 l32i a4, a2, PT_AREG4 369 wsr a0, sar 370 mov a1, a2 371 372 rsr a0, ps 373 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode 374 375 movi a0, _kernel_exception 376 jx a0 377 3782: movi a0, _user_exception 379 jx a0 380 381#ifdef STORE_EXCEPTION_HANDLER 382 383 # a7: instruction pointer, a4: instruction, a3: value 384.Lstore_w: 385 movi a6, 0 # mask: ffffffff:00000000 386 387#if XCHAL_HAVE_DENSITY 388 addi a7, a7, 2 # incr. PC,assume 16-bit instruction 389 390 extui a5, a4, INSN_OP0, 4 # extract OP0 391 addi a5, a5, -OP0_S32I_N 392 _beqz a5, 1f # s32i.n: jump 393 394 addi a7, a7, 1 # increment PC, 32-bit instruction 395#else 396 addi a7, a7, 3 # increment PC, 32-bit instruction 397#endif 398 399 extui a5, a4, INSN_OP1, 4 # extract OP1 400 _beqi a5, OP1_S32I, 1f # jump if 32 bit store 401 _bnei a5, OP1_S16I, .Linvalid_instruction 402 403 movi a5, -1 404 __extl a3, a3 # get 16-bit value 405 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000 406 407 /* Get memory address */ 408 4091: 410 movi a4, ~3 411 and a4, a4, a8 # align memory address 412 413 /* Insert value into memory */ 414 415 movi a5, -1 # mask: ffffffff:XXXX0000 416#ifdef UNALIGNED_USER_EXCEPTION 417 addi a4, a4, 8 418#endif 419 420 __ssa8r a8 421 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE) 422 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE) 423#ifdef UNALIGNED_USER_EXCEPTION 424 l32e a5, a4, -8 425#else 426 l32i a5, a4, 0 # load lower address word 427#endif 428 and a5, a5, a8 # mask 429 __sh a8, a3 # shift value 430 or a5, a5, a8 # or with original value 431#ifdef UNALIGNED_USER_EXCEPTION 432 s32e a5, a4, -8 433 l32e a8, a4, -4 434#else 435 s32i a5, a4, 0 # store 436 l32i a8, a4, 4 # same for upper address word 437#endif 438 __sl a5, a3 439 and a6, a8, a6 440 or a6, a6, a5 441#ifdef UNALIGNED_USER_EXCEPTION 442 s32e a6, a4, -4 443#else 444 s32i a6, a4, 4 445#endif 446#endif 447 448.Lexit: 449#if XCHAL_HAVE_LOOPS 450 rsr a4, lend # check if we reached LEND 451 bne a7, a4, 1f 452 rsr a4, lcount # and LCOUNT != 0 453 beqz a4, 1f 454 addi a4, a4, -1 # decrement LCOUNT and set 455 rsr a7, lbeg # set PC to LBEGIN 456 wsr a4, lcount 457#endif 458 4591: wsr a7, epc1 # skip emulated instruction 460 461 /* Update icount if we're single-stepping in userspace. */ 462 rsr a4, icountlevel 463 beqz a4, 1f 464 bgeui a4, LOCKLEVEL + 1, 1f 465 rsr a4, icount 466 addi a4, a4, 1 467 wsr a4, icount 4681: 469 movi a4, 0 470 rsr a3, excsave1 471 s32i a4, a3, EXC_TABLE_FIXUP 472 473 /* Restore working register */ 474 475 l32i a0, a2, PT_SAR 476 l32i a8, a2, PT_AREG8 477 l32i a7, a2, PT_AREG7 478 l32i a6, a2, PT_AREG6 479 l32i a5, a2, PT_AREG5 480 l32i a4, a2, PT_AREG4 481 l32i a3, a2, PT_AREG3 482 483 /* restore SAR and return */ 484 485 wsr a0, sar 486 l32i a0, a2, PT_AREG0 487 l32i a2, a2, PT_AREG2 488 rfe 489 490 .align 4 491.Lsave_and_load_instruction: 492 493 /* Save some working register */ 494 495 s32i a3, a2, PT_AREG3 496 s32i a4, a2, PT_AREG4 497 s32i a5, a2, PT_AREG5 498 s32i a6, a2, PT_AREG6 499 s32i a7, a2, PT_AREG7 500 s32i a8, a2, PT_AREG8 501 502 rsr a4, depc 503 s32i a4, a2, PT_AREG2 504 505 rsr a5, sar 506 s32i a5, a2, PT_SAR 507 508 rsr a3, excsave1 509 movi a4, fast_unaligned_fixup 510 s32i a4, a3, EXC_TABLE_FIXUP 511 512 rsr a8, excvaddr # load unaligned memory address 513 514 /* Now, identify one of the following load/store instructions. 515 * 516 * The only possible danger of a double exception on the 517 * following l32i instructions is kernel code in vmalloc 518 * memory. The processor was just executing at the EPC_1 519 * address, and indeed, already fetched the instruction. That 520 * guarantees a TLB mapping, which hasn't been replaced by 521 * this unaligned exception handler that uses only static TLB 522 * mappings. However, high-level interrupt handlers might 523 * modify TLB entries, so for the generic case, we register a 524 * TABLE_FIXUP handler here, too. 525 */ 526 527 /* a3...a6 saved on stack, a2 = SP */ 528 529 /* Extract the instruction that caused the unaligned access. */ 530 531 rsr a7, epc1 # load exception address 532 movi a3, ~3 533 and a3, a3, a7 # mask lower bits 534 535 l32i a4, a3, 0 # load 2 words 536 l32i a5, a3, 4 537 538 __ssa8 a7 539 __src_b a4, a4, a5 # a4 has the instruction 540 541 ret 542 543ENDPROC(fast_unaligned) 544 545ENTRY(fast_unaligned_fixup) 546 547 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE 548 wsr a3, excsave1 549 550 l32i a8, a2, PT_AREG8 551 l32i a7, a2, PT_AREG7 552 l32i a6, a2, PT_AREG6 553 l32i a5, a2, PT_AREG5 554 l32i a4, a2, PT_SAR 555 l32i a0, a2, PT_AREG2 556 wsr a4, sar 557 wsr a0, depc # restore depc and a0 558 l32i a4, a2, PT_AREG4 559 560 rsr a0, exccause 561 s32i a0, a2, PT_DEPC # mark as a regular exception 562 563 rsr a0, ps 564 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode 565 566 rsr a0, exccause 567 addx4 a0, a0, a3 # find entry in table 568 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler 569 l32i a3, a2, PT_AREG3 570 jx a0 5711: 572 rsr a0, exccause 573 addx4 a0, a0, a3 # find entry in table 574 l32i a0, a0, EXC_TABLE_FAST_USER # load handler 575 l32i a3, a2, PT_AREG3 576 jx a0 577 578ENDPROC(fast_unaligned_fixup) 579#endif 580