1 /* align.c - handle alignment exceptions for the Power PC. 2 * 3 * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au> 4 * Copyright (c) 1998-1999 TiVo, Inc. 5 * PowerPC 403GCX modifications. 6 * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu> 7 * PowerPC 403GCX/405GP modifications. 8 * Copyright (c) 2001-2002 PPC64 team, IBM Corp 9 * 64-bit and Power4 support 10 * Copyright (c) 2005 Benjamin Herrenschmidt, IBM Corp 11 * <benh@kernel.crashing.org> 12 * Merge ppc32 and ppc64 implementations 13 * 14 * This program is free software; you can redistribute it and/or 15 * modify it under the terms of the GNU General Public License 16 * as published by the Free Software Foundation; either version 17 * 2 of the License, or (at your option) any later version. 18 */ 19 20 #include <linux/kernel.h> 21 #include <linux/mm.h> 22 #include <asm/processor.h> 23 #include <asm/uaccess.h> 24 #include <asm/system.h> 25 #include <asm/cache.h> 26 #include <asm/cputable.h> 27 28 struct aligninfo { 29 unsigned char len; 30 unsigned char flags; 31 }; 32 33 #define IS_XFORM(inst) (((inst) >> 26) == 31) 34 #define IS_DSFORM(inst) (((inst) >> 26) >= 56) 35 36 #define INVALID { 0, 0 } 37 38 /* Bits in the flags field */ 39 #define LD 0 /* load */ 40 #define ST 1 /* store */ 41 #define SE 2 /* sign-extend value */ 42 #define F 4 /* to/from fp regs */ 43 #define U 8 /* update index register */ 44 #define M 0x10 /* multiple load/store */ 45 #define SW 0x20 /* byte swap */ 46 #define S 0x40 /* single-precision fp or... */ 47 #define SX 0x40 /* ... byte count in XER */ 48 #define HARD 0x80 /* string, stwcx. */ 49 50 /* DSISR bits reported for a DCBZ instruction: */ 51 #define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */ 52 53 #define SWAP(a, b) (t = (a), (a) = (b), (b) = t) 54 55 /* 56 * The PowerPC stores certain bits of the instruction that caused the 57 * alignment exception in the DSISR register. This array maps those 58 * bits to information about the operand length and what the 59 * instruction would do. 60 */ 61 static struct aligninfo aligninfo[128] = { 62 { 4, LD }, /* 00 0 0000: lwz / lwarx */ 63 INVALID, /* 00 0 0001 */ 64 { 4, ST }, /* 00 0 0010: stw */ 65 INVALID, /* 00 0 0011 */ 66 { 2, LD }, /* 00 0 0100: lhz */ 67 { 2, LD+SE }, /* 00 0 0101: lha */ 68 { 2, ST }, /* 00 0 0110: sth */ 69 { 4, LD+M }, /* 00 0 0111: lmw */ 70 { 4, LD+F+S }, /* 00 0 1000: lfs */ 71 { 8, LD+F }, /* 00 0 1001: lfd */ 72 { 4, ST+F+S }, /* 00 0 1010: stfs */ 73 { 8, ST+F }, /* 00 0 1011: stfd */ 74 INVALID, /* 00 0 1100 */ 75 { 8, LD }, /* 00 0 1101: ld/ldu/lwa */ 76 INVALID, /* 00 0 1110 */ 77 { 8, ST }, /* 00 0 1111: std/stdu */ 78 { 4, LD+U }, /* 00 1 0000: lwzu */ 79 INVALID, /* 00 1 0001 */ 80 { 4, ST+U }, /* 00 1 0010: stwu */ 81 INVALID, /* 00 1 0011 */ 82 { 2, LD+U }, /* 00 1 0100: lhzu */ 83 { 2, LD+SE+U }, /* 00 1 0101: lhau */ 84 { 2, ST+U }, /* 00 1 0110: sthu */ 85 { 4, ST+M }, /* 00 1 0111: stmw */ 86 { 4, LD+F+S+U }, /* 00 1 1000: lfsu */ 87 { 8, LD+F+U }, /* 00 1 1001: lfdu */ 88 { 4, ST+F+S+U }, /* 00 1 1010: stfsu */ 89 { 8, ST+F+U }, /* 00 1 1011: stfdu */ 90 INVALID, /* 00 1 1100 */ 91 INVALID, /* 00 1 1101 */ 92 INVALID, /* 00 1 1110 */ 93 INVALID, /* 00 1 1111 */ 94 { 8, LD }, /* 01 0 0000: ldx */ 95 INVALID, /* 01 0 0001 */ 96 { 8, ST }, /* 01 0 0010: stdx */ 97 INVALID, /* 01 0 0011 */ 98 INVALID, /* 01 0 0100 */ 99 { 4, LD+SE }, /* 01 0 0101: lwax */ 100 INVALID, /* 01 0 0110 */ 101 INVALID, /* 01 0 0111 */ 102 { 4, LD+M+HARD+SX }, /* 01 0 1000: lswx */ 103 { 4, LD+M+HARD }, /* 01 0 1001: lswi */ 104 { 4, ST+M+HARD+SX }, /* 01 0 1010: stswx */ 105 { 4, ST+M+HARD }, /* 01 0 1011: stswi */ 106 INVALID, /* 01 0 1100 */ 107 { 8, LD+U }, /* 01 0 1101: ldu */ 108 INVALID, /* 01 0 1110 */ 109 { 8, ST+U }, /* 01 0 1111: stdu */ 110 { 8, LD+U }, /* 01 1 0000: ldux */ 111 INVALID, /* 01 1 0001 */ 112 { 8, ST+U }, /* 01 1 0010: stdux */ 113 INVALID, /* 01 1 0011 */ 114 INVALID, /* 01 1 0100 */ 115 { 4, LD+SE+U }, /* 01 1 0101: lwaux */ 116 INVALID, /* 01 1 0110 */ 117 INVALID, /* 01 1 0111 */ 118 INVALID, /* 01 1 1000 */ 119 INVALID, /* 01 1 1001 */ 120 INVALID, /* 01 1 1010 */ 121 INVALID, /* 01 1 1011 */ 122 INVALID, /* 01 1 1100 */ 123 INVALID, /* 01 1 1101 */ 124 INVALID, /* 01 1 1110 */ 125 INVALID, /* 01 1 1111 */ 126 INVALID, /* 10 0 0000 */ 127 INVALID, /* 10 0 0001 */ 128 INVALID, /* 10 0 0010: stwcx. */ 129 INVALID, /* 10 0 0011 */ 130 INVALID, /* 10 0 0100 */ 131 INVALID, /* 10 0 0101 */ 132 INVALID, /* 10 0 0110 */ 133 INVALID, /* 10 0 0111 */ 134 { 4, LD+SW }, /* 10 0 1000: lwbrx */ 135 INVALID, /* 10 0 1001 */ 136 { 4, ST+SW }, /* 10 0 1010: stwbrx */ 137 INVALID, /* 10 0 1011 */ 138 { 2, LD+SW }, /* 10 0 1100: lhbrx */ 139 { 4, LD+SE }, /* 10 0 1101 lwa */ 140 { 2, ST+SW }, /* 10 0 1110: sthbrx */ 141 INVALID, /* 10 0 1111 */ 142 INVALID, /* 10 1 0000 */ 143 INVALID, /* 10 1 0001 */ 144 INVALID, /* 10 1 0010 */ 145 INVALID, /* 10 1 0011 */ 146 INVALID, /* 10 1 0100 */ 147 INVALID, /* 10 1 0101 */ 148 INVALID, /* 10 1 0110 */ 149 INVALID, /* 10 1 0111 */ 150 INVALID, /* 10 1 1000 */ 151 INVALID, /* 10 1 1001 */ 152 INVALID, /* 10 1 1010 */ 153 INVALID, /* 10 1 1011 */ 154 INVALID, /* 10 1 1100 */ 155 INVALID, /* 10 1 1101 */ 156 INVALID, /* 10 1 1110 */ 157 { 0, ST+HARD }, /* 10 1 1111: dcbz */ 158 { 4, LD }, /* 11 0 0000: lwzx */ 159 INVALID, /* 11 0 0001 */ 160 { 4, ST }, /* 11 0 0010: stwx */ 161 INVALID, /* 11 0 0011 */ 162 { 2, LD }, /* 11 0 0100: lhzx */ 163 { 2, LD+SE }, /* 11 0 0101: lhax */ 164 { 2, ST }, /* 11 0 0110: sthx */ 165 INVALID, /* 11 0 0111 */ 166 { 4, LD+F+S }, /* 11 0 1000: lfsx */ 167 { 8, LD+F }, /* 11 0 1001: lfdx */ 168 { 4, ST+F+S }, /* 11 0 1010: stfsx */ 169 { 8, ST+F }, /* 11 0 1011: stfdx */ 170 INVALID, /* 11 0 1100 */ 171 { 8, LD+M }, /* 11 0 1101: lmd */ 172 INVALID, /* 11 0 1110 */ 173 { 8, ST+M }, /* 11 0 1111: stmd */ 174 { 4, LD+U }, /* 11 1 0000: lwzux */ 175 INVALID, /* 11 1 0001 */ 176 { 4, ST+U }, /* 11 1 0010: stwux */ 177 INVALID, /* 11 1 0011 */ 178 { 2, LD+U }, /* 11 1 0100: lhzux */ 179 { 2, LD+SE+U }, /* 11 1 0101: lhaux */ 180 { 2, ST+U }, /* 11 1 0110: sthux */ 181 INVALID, /* 11 1 0111 */ 182 { 4, LD+F+S+U }, /* 11 1 1000: lfsux */ 183 { 8, LD+F+U }, /* 11 1 1001: lfdux */ 184 { 4, ST+F+S+U }, /* 11 1 1010: stfsux */ 185 { 8, ST+F+U }, /* 11 1 1011: stfdux */ 186 INVALID, /* 11 1 1100 */ 187 INVALID, /* 11 1 1101 */ 188 INVALID, /* 11 1 1110 */ 189 INVALID, /* 11 1 1111 */ 190 }; 191 192 /* 193 * Create a DSISR value from the instruction 194 */ 195 static inline unsigned make_dsisr(unsigned instr) 196 { 197 unsigned dsisr; 198 199 200 /* bits 6:15 --> 22:31 */ 201 dsisr = (instr & 0x03ff0000) >> 16; 202 203 if (IS_XFORM(instr)) { 204 /* bits 29:30 --> 15:16 */ 205 dsisr |= (instr & 0x00000006) << 14; 206 /* bit 25 --> 17 */ 207 dsisr |= (instr & 0x00000040) << 8; 208 /* bits 21:24 --> 18:21 */ 209 dsisr |= (instr & 0x00000780) << 3; 210 } else { 211 /* bit 5 --> 17 */ 212 dsisr |= (instr & 0x04000000) >> 12; 213 /* bits 1: 4 --> 18:21 */ 214 dsisr |= (instr & 0x78000000) >> 17; 215 /* bits 30:31 --> 12:13 */ 216 if (IS_DSFORM(instr)) 217 dsisr |= (instr & 0x00000003) << 18; 218 } 219 220 return dsisr; 221 } 222 223 /* 224 * The dcbz (data cache block zero) instruction 225 * gives an alignment fault if used on non-cacheable 226 * memory. We handle the fault mainly for the 227 * case when we are running with the cache disabled 228 * for debugging. 229 */ 230 static int emulate_dcbz(struct pt_regs *regs, unsigned char __user *addr) 231 { 232 long __user *p; 233 int i, size; 234 235 #ifdef __powerpc64__ 236 size = ppc64_caches.dline_size; 237 #else 238 size = L1_CACHE_BYTES; 239 #endif 240 p = (long __user *) (regs->dar & -size); 241 if (user_mode(regs) && !access_ok(VERIFY_WRITE, p, size)) 242 return -EFAULT; 243 for (i = 0; i < size / sizeof(long); ++i) 244 if (__put_user(0, p+i)) 245 return -EFAULT; 246 return 1; 247 } 248 249 /* 250 * Emulate load & store multiple instructions 251 * On 64-bit machines, these instructions only affect/use the 252 * bottom 4 bytes of each register, and the loads clear the 253 * top 4 bytes of the affected register. 254 */ 255 #ifdef CONFIG_PPC64 256 #define REG_BYTE(rp, i) *((u8 *)((rp) + ((i) >> 2)) + ((i) & 3) + 4) 257 #else 258 #define REG_BYTE(rp, i) *((u8 *)(rp) + (i)) 259 #endif 260 261 #define SWIZ_PTR(p) ((unsigned char __user *)((p) ^ swiz)) 262 263 static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, 264 unsigned int reg, unsigned int nb, 265 unsigned int flags, unsigned int instr, 266 unsigned long swiz) 267 { 268 unsigned long *rptr; 269 unsigned int nb0, i, bswiz; 270 unsigned long p; 271 272 /* 273 * We do not try to emulate 8 bytes multiple as they aren't really 274 * available in our operating environments and we don't try to 275 * emulate multiples operations in kernel land as they should never 276 * be used/generated there at least not on unaligned boundaries 277 */ 278 if (unlikely((nb > 4) || !user_mode(regs))) 279 return 0; 280 281 /* lmw, stmw, lswi/x, stswi/x */ 282 nb0 = 0; 283 if (flags & HARD) { 284 if (flags & SX) { 285 nb = regs->xer & 127; 286 if (nb == 0) 287 return 1; 288 } else { 289 unsigned long pc = regs->nip ^ (swiz & 4); 290 291 if (__get_user(instr, (unsigned int __user *)pc)) 292 return -EFAULT; 293 if (swiz == 0 && (flags & SW)) 294 instr = cpu_to_le32(instr); 295 nb = (instr >> 11) & 0x1f; 296 if (nb == 0) 297 nb = 32; 298 } 299 if (nb + reg * 4 > 128) { 300 nb0 = nb + reg * 4 - 128; 301 nb = 128 - reg * 4; 302 } 303 } else { 304 /* lwm, stmw */ 305 nb = (32 - reg) * 4; 306 } 307 308 if (!access_ok((flags & ST ? VERIFY_WRITE: VERIFY_READ), addr, nb+nb0)) 309 return -EFAULT; /* bad address */ 310 311 rptr = ®s->gpr[reg]; 312 p = (unsigned long) addr; 313 bswiz = (flags & SW)? 3: 0; 314 315 if (!(flags & ST)) { 316 /* 317 * This zeroes the top 4 bytes of the affected registers 318 * in 64-bit mode, and also zeroes out any remaining 319 * bytes of the last register for lsw*. 320 */ 321 memset(rptr, 0, ((nb + 3) / 4) * sizeof(unsigned long)); 322 if (nb0 > 0) 323 memset(®s->gpr[0], 0, 324 ((nb0 + 3) / 4) * sizeof(unsigned long)); 325 326 for (i = 0; i < nb; ++i, ++p) 327 if (__get_user(REG_BYTE(rptr, i ^ bswiz), SWIZ_PTR(p))) 328 return -EFAULT; 329 if (nb0 > 0) { 330 rptr = ®s->gpr[0]; 331 addr += nb; 332 for (i = 0; i < nb0; ++i, ++p) 333 if (__get_user(REG_BYTE(rptr, i ^ bswiz), 334 SWIZ_PTR(p))) 335 return -EFAULT; 336 } 337 338 } else { 339 for (i = 0; i < nb; ++i, ++p) 340 if (__put_user(REG_BYTE(rptr, i ^ bswiz), SWIZ_PTR(p))) 341 return -EFAULT; 342 if (nb0 > 0) { 343 rptr = ®s->gpr[0]; 344 addr += nb; 345 for (i = 0; i < nb0; ++i, ++p) 346 if (__put_user(REG_BYTE(rptr, i ^ bswiz), 347 SWIZ_PTR(p))) 348 return -EFAULT; 349 } 350 } 351 return 1; 352 } 353 354 355 /* 356 * Called on alignment exception. Attempts to fixup 357 * 358 * Return 1 on success 359 * Return 0 if unable to handle the interrupt 360 * Return -EFAULT if data address is bad 361 */ 362 363 int fix_alignment(struct pt_regs *regs) 364 { 365 unsigned int instr, nb, flags; 366 unsigned int reg, areg; 367 unsigned int dsisr; 368 unsigned char __user *addr; 369 unsigned long p, swiz; 370 int ret, t; 371 union { 372 u64 ll; 373 double dd; 374 unsigned char v[8]; 375 struct { 376 unsigned hi32; 377 int low32; 378 } x32; 379 struct { 380 unsigned char hi48[6]; 381 short low16; 382 } x16; 383 } data; 384 385 /* 386 * We require a complete register set, if not, then our assembly 387 * is broken 388 */ 389 CHECK_FULL_REGS(regs); 390 391 dsisr = regs->dsisr; 392 393 /* Some processors don't provide us with a DSISR we can use here, 394 * let's make one up from the instruction 395 */ 396 if (cpu_has_feature(CPU_FTR_NODSISRALIGN)) { 397 unsigned long pc = regs->nip; 398 399 if (cpu_has_feature(CPU_FTR_PPC_LE) && (regs->msr & MSR_LE)) 400 pc ^= 4; 401 if (unlikely(__get_user(instr, (unsigned int __user *)pc))) 402 return -EFAULT; 403 if (cpu_has_feature(CPU_FTR_REAL_LE) && (regs->msr & MSR_LE)) 404 instr = cpu_to_le32(instr); 405 dsisr = make_dsisr(instr); 406 } 407 408 /* extract the operation and registers from the dsisr */ 409 reg = (dsisr >> 5) & 0x1f; /* source/dest register */ 410 areg = dsisr & 0x1f; /* register to update */ 411 instr = (dsisr >> 10) & 0x7f; 412 instr |= (dsisr >> 13) & 0x60; 413 414 /* Lookup the operation in our table */ 415 nb = aligninfo[instr].len; 416 flags = aligninfo[instr].flags; 417 418 /* Byteswap little endian loads and stores */ 419 swiz = 0; 420 if (regs->msr & MSR_LE) { 421 flags ^= SW; 422 /* 423 * So-called "PowerPC little endian" mode works by 424 * swizzling addresses rather than by actually doing 425 * any byte-swapping. To emulate this, we XOR each 426 * byte address with 7. We also byte-swap, because 427 * the processor's address swizzling depends on the 428 * operand size (it xors the address with 7 for bytes, 429 * 6 for halfwords, 4 for words, 0 for doublewords) but 430 * we will xor with 7 and load/store each byte separately. 431 */ 432 if (cpu_has_feature(CPU_FTR_PPC_LE)) 433 swiz = 7; 434 } 435 436 /* DAR has the operand effective address */ 437 addr = (unsigned char __user *)regs->dar; 438 439 /* A size of 0 indicates an instruction we don't support, with 440 * the exception of DCBZ which is handled as a special case here 441 */ 442 if (instr == DCBZ) 443 return emulate_dcbz(regs, addr); 444 if (unlikely(nb == 0)) 445 return 0; 446 447 /* Load/Store Multiple instructions are handled in their own 448 * function 449 */ 450 if (flags & M) 451 return emulate_multiple(regs, addr, reg, nb, 452 flags, instr, swiz); 453 454 /* Verify the address of the operand */ 455 if (unlikely(user_mode(regs) && 456 !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ), 457 addr, nb))) 458 return -EFAULT; 459 460 /* Force the fprs into the save area so we can reference them */ 461 if (flags & F) { 462 /* userland only */ 463 if (unlikely(!user_mode(regs))) 464 return 0; 465 flush_fp_to_thread(current); 466 } 467 468 /* If we are loading, get the data from user space, else 469 * get it from register values 470 */ 471 if (!(flags & ST)) { 472 data.ll = 0; 473 ret = 0; 474 p = (unsigned long) addr; 475 switch (nb) { 476 case 8: 477 ret |= __get_user(data.v[0], SWIZ_PTR(p++)); 478 ret |= __get_user(data.v[1], SWIZ_PTR(p++)); 479 ret |= __get_user(data.v[2], SWIZ_PTR(p++)); 480 ret |= __get_user(data.v[3], SWIZ_PTR(p++)); 481 case 4: 482 ret |= __get_user(data.v[4], SWIZ_PTR(p++)); 483 ret |= __get_user(data.v[5], SWIZ_PTR(p++)); 484 case 2: 485 ret |= __get_user(data.v[6], SWIZ_PTR(p++)); 486 ret |= __get_user(data.v[7], SWIZ_PTR(p++)); 487 if (unlikely(ret)) 488 return -EFAULT; 489 } 490 } else if (flags & F) { 491 data.dd = current->thread.fpr[reg]; 492 if (flags & S) { 493 /* Single-precision FP store requires conversion... */ 494 #ifdef CONFIG_PPC_FPU 495 preempt_disable(); 496 enable_kernel_fp(); 497 cvt_df(&data.dd, (float *)&data.v[4], ¤t->thread); 498 preempt_enable(); 499 #else 500 return 0; 501 #endif 502 } 503 } else 504 data.ll = regs->gpr[reg]; 505 506 if (flags & SW) { 507 switch (nb) { 508 case 8: 509 SWAP(data.v[0], data.v[7]); 510 SWAP(data.v[1], data.v[6]); 511 SWAP(data.v[2], data.v[5]); 512 SWAP(data.v[3], data.v[4]); 513 break; 514 case 4: 515 SWAP(data.v[4], data.v[7]); 516 SWAP(data.v[5], data.v[6]); 517 break; 518 case 2: 519 SWAP(data.v[6], data.v[7]); 520 break; 521 } 522 } 523 524 /* Perform other misc operations like sign extension 525 * or floating point single precision conversion 526 */ 527 switch (flags & ~(U|SW)) { 528 case LD+SE: /* sign extend */ 529 if ( nb == 2 ) 530 data.ll = data.x16.low16; 531 else /* nb must be 4 */ 532 data.ll = data.x32.low32; 533 break; 534 535 /* Single-precision FP load requires conversion... */ 536 case LD+F+S: 537 #ifdef CONFIG_PPC_FPU 538 preempt_disable(); 539 enable_kernel_fp(); 540 cvt_fd((float *)&data.v[4], &data.dd, ¤t->thread); 541 preempt_enable(); 542 #else 543 return 0; 544 #endif 545 break; 546 } 547 548 /* Store result to memory or update registers */ 549 if (flags & ST) { 550 ret = 0; 551 p = (unsigned long) addr; 552 switch (nb) { 553 case 8: 554 ret |= __put_user(data.v[0], SWIZ_PTR(p++)); 555 ret |= __put_user(data.v[1], SWIZ_PTR(p++)); 556 ret |= __put_user(data.v[2], SWIZ_PTR(p++)); 557 ret |= __put_user(data.v[3], SWIZ_PTR(p++)); 558 case 4: 559 ret |= __put_user(data.v[4], SWIZ_PTR(p++)); 560 ret |= __put_user(data.v[5], SWIZ_PTR(p++)); 561 case 2: 562 ret |= __put_user(data.v[6], SWIZ_PTR(p++)); 563 ret |= __put_user(data.v[7], SWIZ_PTR(p++)); 564 } 565 if (unlikely(ret)) 566 return -EFAULT; 567 } else if (flags & F) 568 current->thread.fpr[reg] = data.dd; 569 else 570 regs->gpr[reg] = data.ll; 571 572 /* Update RA as needed */ 573 if (flags & U) 574 regs->gpr[areg] = regs->dar; 575 576 return 1; 577 } 578