xref: /linux/arch/powerpc/lib/sstep.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Single-step support.
4  *
5  * Copyright (C) 2004 Paul Mackerras <paulus@au.ibm.com>, IBM
6  */
7 #include <linux/kernel.h>
8 #include <linux/kprobes.h>
9 #include <linux/ptrace.h>
10 #include <linux/prefetch.h>
11 #include <asm/sstep.h>
12 #include <asm/processor.h>
13 #include <linux/uaccess.h>
14 #include <asm/cpu_has_feature.h>
15 #include <asm/cputable.h>
16 #include <asm/disassemble.h>
17 
18 #ifdef CONFIG_PPC64
19 /* Bits in SRR1 that are copied from MSR */
20 #define MSR_MASK	0xffffffff87c0ffffUL
21 #else
22 #define MSR_MASK	0x87c0ffff
23 #endif
24 
25 /* Bits in XER */
26 #define XER_SO		0x80000000U
27 #define XER_OV		0x40000000U
28 #define XER_CA		0x20000000U
29 #define XER_OV32	0x00080000U
30 #define XER_CA32	0x00040000U
31 
32 #ifdef CONFIG_VSX
33 #define VSX_REGISTER_XTP(rd)   ((((rd) & 1) << 5) | ((rd) & 0xfe))
34 #endif
35 
36 #ifdef CONFIG_PPC_FPU
37 /*
38  * Functions in ldstfp.S
39  */
40 extern void get_fpr(int rn, double *p);
41 extern void put_fpr(int rn, const double *p);
42 extern void get_vr(int rn, __vector128 *p);
43 extern void put_vr(int rn, __vector128 *p);
44 extern void load_vsrn(int vsr, const void *p);
45 extern void store_vsrn(int vsr, void *p);
46 extern void conv_sp_to_dp(const float *sp, double *dp);
47 extern void conv_dp_to_sp(const double *dp, float *sp);
48 #endif
49 
50 #ifdef CONFIG_PPC64
51 /*
52  * Functions in quad.S
53  */
54 extern int do_lq(unsigned long ea, unsigned long *regs);
55 extern int do_stq(unsigned long ea, unsigned long val0, unsigned long val1);
56 extern int do_lqarx(unsigned long ea, unsigned long *regs);
57 extern int do_stqcx(unsigned long ea, unsigned long val0, unsigned long val1,
58 		    unsigned int *crp);
59 #endif
60 
61 #ifdef __LITTLE_ENDIAN__
62 #define IS_LE	1
63 #define IS_BE	0
64 #else
65 #define IS_LE	0
66 #define IS_BE	1
67 #endif
68 
69 /*
70  * Emulate the truncation of 64 bit values in 32-bit mode.
71  */
72 static nokprobe_inline unsigned long truncate_if_32bit(unsigned long msr,
73 							unsigned long val)
74 {
75 	if ((msr & MSR_64BIT) == 0)
76 		val &= 0xffffffffUL;
77 	return val;
78 }
79 
80 /*
81  * Determine whether a conditional branch instruction would branch.
82  */
83 static nokprobe_inline int branch_taken(unsigned int instr,
84 					const struct pt_regs *regs,
85 					struct instruction_op *op)
86 {
87 	unsigned int bo = (instr >> 21) & 0x1f;
88 	unsigned int bi;
89 
90 	if ((bo & 4) == 0) {
91 		/* decrement counter */
92 		op->type |= DECCTR;
93 		if (((bo >> 1) & 1) ^ (regs->ctr == 1))
94 			return 0;
95 	}
96 	if ((bo & 0x10) == 0) {
97 		/* check bit from CR */
98 		bi = (instr >> 16) & 0x1f;
99 		if (((regs->ccr >> (31 - bi)) & 1) != ((bo >> 3) & 1))
100 			return 0;
101 	}
102 	return 1;
103 }
104 
105 static nokprobe_inline long address_ok(struct pt_regs *regs,
106 				       unsigned long ea, int nb)
107 {
108 	if (!user_mode(regs))
109 		return 1;
110 	if (access_ok((void __user *)ea, nb))
111 		return 1;
112 	if (access_ok((void __user *)ea, 1))
113 		/* Access overlaps the end of the user region */
114 		regs->dar = TASK_SIZE_MAX - 1;
115 	else
116 		regs->dar = ea;
117 	return 0;
118 }
119 
120 /*
121  * Calculate effective address for a D-form instruction
122  */
123 static nokprobe_inline unsigned long dform_ea(unsigned int instr,
124 					      const struct pt_regs *regs)
125 {
126 	int ra;
127 	unsigned long ea;
128 
129 	ra = (instr >> 16) & 0x1f;
130 	ea = (signed short) instr;		/* sign-extend */
131 	if (ra)
132 		ea += regs->gpr[ra];
133 
134 	return ea;
135 }
136 
137 #ifdef CONFIG_PPC64
138 /*
139  * Calculate effective address for a DS-form instruction
140  */
141 static nokprobe_inline unsigned long dsform_ea(unsigned int instr,
142 					       const struct pt_regs *regs)
143 {
144 	int ra;
145 	unsigned long ea;
146 
147 	ra = (instr >> 16) & 0x1f;
148 	ea = (signed short) (instr & ~3);	/* sign-extend */
149 	if (ra)
150 		ea += regs->gpr[ra];
151 
152 	return ea;
153 }
154 
155 /*
156  * Calculate effective address for a DQ-form instruction
157  */
158 static nokprobe_inline unsigned long dqform_ea(unsigned int instr,
159 					       const struct pt_regs *regs)
160 {
161 	int ra;
162 	unsigned long ea;
163 
164 	ra = (instr >> 16) & 0x1f;
165 	ea = (signed short) (instr & ~0xf);	/* sign-extend */
166 	if (ra)
167 		ea += regs->gpr[ra];
168 
169 	return ea;
170 }
171 #endif /* CONFIG_PPC64 */
172 
173 /*
174  * Calculate effective address for an X-form instruction
175  */
176 static nokprobe_inline unsigned long xform_ea(unsigned int instr,
177 					      const struct pt_regs *regs)
178 {
179 	int ra, rb;
180 	unsigned long ea;
181 
182 	ra = (instr >> 16) & 0x1f;
183 	rb = (instr >> 11) & 0x1f;
184 	ea = regs->gpr[rb];
185 	if (ra)
186 		ea += regs->gpr[ra];
187 
188 	return ea;
189 }
190 
191 /*
192  * Calculate effective address for a MLS:D-form / 8LS:D-form
193  * prefixed instruction
194  */
195 static nokprobe_inline unsigned long mlsd_8lsd_ea(unsigned int instr,
196 						  unsigned int suffix,
197 						  const struct pt_regs *regs)
198 {
199 	int ra, prefix_r;
200 	unsigned int  dd;
201 	unsigned long ea, d0, d1, d;
202 
203 	prefix_r = GET_PREFIX_R(instr);
204 	ra = GET_PREFIX_RA(suffix);
205 
206 	d0 = instr & 0x3ffff;
207 	d1 = suffix & 0xffff;
208 	d = (d0 << 16) | d1;
209 
210 	/*
211 	 * sign extend a 34 bit number
212 	 */
213 	dd = (unsigned int)(d >> 2);
214 	ea = (signed int)dd;
215 	ea = (ea << 2) | (d & 0x3);
216 
217 	if (!prefix_r && ra)
218 		ea += regs->gpr[ra];
219 	else if (!prefix_r && !ra)
220 		; /* Leave ea as is */
221 	else if (prefix_r)
222 		ea += regs->nip;
223 
224 	/*
225 	 * (prefix_r && ra) is an invalid form. Should already be
226 	 * checked for by caller!
227 	 */
228 
229 	return ea;
230 }
231 
232 /*
233  * Return the largest power of 2, not greater than sizeof(unsigned long),
234  * such that x is a multiple of it.
235  */
236 static nokprobe_inline unsigned long max_align(unsigned long x)
237 {
238 	x |= sizeof(unsigned long);
239 	return x & -x;		/* isolates rightmost bit */
240 }
241 
242 static nokprobe_inline unsigned long byterev_2(unsigned long x)
243 {
244 	return ((x >> 8) & 0xff) | ((x & 0xff) << 8);
245 }
246 
247 static nokprobe_inline unsigned long byterev_4(unsigned long x)
248 {
249 	return ((x >> 24) & 0xff) | ((x >> 8) & 0xff00) |
250 		((x & 0xff00) << 8) | ((x & 0xff) << 24);
251 }
252 
253 #ifdef CONFIG_PPC64
254 static nokprobe_inline unsigned long byterev_8(unsigned long x)
255 {
256 	return (byterev_4(x) << 32) | byterev_4(x >> 32);
257 }
258 #endif
259 
260 static nokprobe_inline void do_byte_reverse(void *ptr, int nb)
261 {
262 	switch (nb) {
263 	case 2:
264 		*(u16 *)ptr = byterev_2(*(u16 *)ptr);
265 		break;
266 	case 4:
267 		*(u32 *)ptr = byterev_4(*(u32 *)ptr);
268 		break;
269 #ifdef CONFIG_PPC64
270 	case 8:
271 		*(unsigned long *)ptr = byterev_8(*(unsigned long *)ptr);
272 		break;
273 	case 16: {
274 		unsigned long *up = (unsigned long *)ptr;
275 		unsigned long tmp;
276 		tmp = byterev_8(up[0]);
277 		up[0] = byterev_8(up[1]);
278 		up[1] = tmp;
279 		break;
280 	}
281 	case 32: {
282 		unsigned long *up = (unsigned long *)ptr;
283 		unsigned long tmp;
284 
285 		tmp = byterev_8(up[0]);
286 		up[0] = byterev_8(up[3]);
287 		up[3] = tmp;
288 		tmp = byterev_8(up[2]);
289 		up[2] = byterev_8(up[1]);
290 		up[1] = tmp;
291 		break;
292 	}
293 
294 #endif
295 	default:
296 		WARN_ON_ONCE(1);
297 	}
298 }
299 
300 static __always_inline int
301 __read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs)
302 {
303 	unsigned long x = 0;
304 
305 	switch (nb) {
306 	case 1:
307 		unsafe_get_user(x, (unsigned char __user *)ea, Efault);
308 		break;
309 	case 2:
310 		unsafe_get_user(x, (unsigned short __user *)ea, Efault);
311 		break;
312 	case 4:
313 		unsafe_get_user(x, (unsigned int __user *)ea, Efault);
314 		break;
315 #ifdef CONFIG_PPC64
316 	case 8:
317 		unsafe_get_user(x, (unsigned long __user *)ea, Efault);
318 		break;
319 #endif
320 	}
321 	*dest = x;
322 	return 0;
323 
324 Efault:
325 	regs->dar = ea;
326 	return -EFAULT;
327 }
328 
329 static nokprobe_inline int
330 read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs)
331 {
332 	void __user *uea = (void __user *)ea;
333 
334 	if (is_kernel_addr(ea))
335 		return __read_mem_aligned(dest, ea, nb, regs);
336 
337 	scoped_user_read_access_size(uea, nb, efault)
338 		return __read_mem_aligned(dest, (unsigned long)uea, nb, regs);
339 
340 efault:
341 	regs->dar = ea;
342 	return -EFAULT;
343 }
344 
345 /*
346  * Copy from userspace to a buffer, using the largest possible
347  * aligned accesses, up to sizeof(long).
348  */
349 static __always_inline int __copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
350 {
351 	int c;
352 
353 	for (; nb > 0; nb -= c) {
354 		c = max_align(ea);
355 		if (c > nb)
356 			c = max_align(nb);
357 		switch (c) {
358 		case 1:
359 			unsafe_get_user(*dest, (u8 __user *)ea, Efault);
360 			break;
361 		case 2:
362 			unsafe_get_user(*(u16 *)dest, (u16 __user *)ea, Efault);
363 			break;
364 		case 4:
365 			unsafe_get_user(*(u32 *)dest, (u32 __user *)ea, Efault);
366 			break;
367 #ifdef CONFIG_PPC64
368 		case 8:
369 			unsafe_get_user(*(u64 *)dest, (u64 __user *)ea, Efault);
370 			break;
371 #endif
372 		}
373 		dest += c;
374 		ea += c;
375 	}
376 	return 0;
377 
378 Efault:
379 	regs->dar = ea;
380 	return -EFAULT;
381 }
382 
383 static nokprobe_inline int copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
384 {
385 	void __user *uea = (void __user *)ea;
386 
387 	if (is_kernel_addr(ea))
388 		return __copy_mem_in(dest, ea, nb, regs);
389 
390 	scoped_user_read_access_size(uea, nb, efault)
391 		return __copy_mem_in(dest, (unsigned long)uea, nb, regs);
392 
393 efault:
394 	regs->dar = ea;
395 	return -EFAULT;
396 }
397 
398 static nokprobe_inline int read_mem_unaligned(unsigned long *dest,
399 					      unsigned long ea, int nb,
400 					      struct pt_regs *regs)
401 {
402 	union {
403 		unsigned long ul;
404 		u8 b[sizeof(unsigned long)];
405 	} u;
406 	int i;
407 	int err;
408 
409 	u.ul = 0;
410 	i = IS_BE ? sizeof(unsigned long) - nb : 0;
411 	err = copy_mem_in(&u.b[i], ea, nb, regs);
412 	if (!err)
413 		*dest = u.ul;
414 	return err;
415 }
416 
417 /*
418  * Read memory at address ea for nb bytes, return 0 for success
419  * or -EFAULT if an error occurred.  N.B. nb must be 1, 2, 4 or 8.
420  * If nb < sizeof(long), the result is right-justified on BE systems.
421  */
422 static int read_mem(unsigned long *dest, unsigned long ea, int nb,
423 			      struct pt_regs *regs)
424 {
425 	if (!address_ok(regs, ea, nb))
426 		return -EFAULT;
427 	if ((ea & (nb - 1)) == 0)
428 		return read_mem_aligned(dest, ea, nb, regs);
429 	return read_mem_unaligned(dest, ea, nb, regs);
430 }
431 NOKPROBE_SYMBOL(read_mem);
432 
433 static __always_inline int
434 __write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs)
435 {
436 	switch (nb) {
437 	case 1:
438 		unsafe_put_user(val, (unsigned char __user *)ea, Efault);
439 		break;
440 	case 2:
441 		unsafe_put_user(val, (unsigned short __user *)ea, Efault);
442 		break;
443 	case 4:
444 		unsafe_put_user(val, (unsigned int __user *)ea, Efault);
445 		break;
446 #ifdef CONFIG_PPC64
447 	case 8:
448 		unsafe_put_user(val, (unsigned long __user *)ea, Efault);
449 		break;
450 #endif
451 	}
452 	return 0;
453 
454 Efault:
455 	regs->dar = ea;
456 	return -EFAULT;
457 }
458 
459 static nokprobe_inline int
460 write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs)
461 {
462 	void __user *uea = (void __user *)ea;
463 
464 	if (is_kernel_addr(ea))
465 		return __write_mem_aligned(val, ea, nb, regs);
466 
467 	scoped_user_write_access_size(uea, nb, efault)
468 		return __write_mem_aligned(val, (unsigned long)uea, nb, regs);
469 
470 efault:
471 	regs->dar = ea;
472 	return -EFAULT;
473 }
474 
475 /*
476  * Copy from a buffer to userspace, using the largest possible
477  * aligned accesses, up to sizeof(long).
478  */
479 static __always_inline int __copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
480 {
481 	int c;
482 
483 	for (; nb > 0; nb -= c) {
484 		c = max_align(ea);
485 		if (c > nb)
486 			c = max_align(nb);
487 		switch (c) {
488 		case 1:
489 			unsafe_put_user(*dest, (u8 __user *)ea, Efault);
490 			break;
491 		case 2:
492 			unsafe_put_user(*(u16 *)dest, (u16 __user *)ea, Efault);
493 			break;
494 		case 4:
495 			unsafe_put_user(*(u32 *)dest, (u32 __user *)ea, Efault);
496 			break;
497 #ifdef CONFIG_PPC64
498 		case 8:
499 			unsafe_put_user(*(u64 *)dest, (u64 __user *)ea, Efault);
500 			break;
501 #endif
502 		}
503 		dest += c;
504 		ea += c;
505 	}
506 	return 0;
507 
508 Efault:
509 	regs->dar = ea;
510 	return -EFAULT;
511 }
512 
513 static nokprobe_inline int copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
514 {
515 	void __user *uea = (void __user *)ea;
516 
517 	if (is_kernel_addr(ea))
518 		return __copy_mem_out(dest, ea, nb, regs);
519 
520 	scoped_user_write_access_size(uea, nb, efault)
521 		return __copy_mem_out(dest, (unsigned long)uea, nb, regs);
522 
523 efault:
524 	regs->dar = ea;
525 	return -EFAULT;
526 }
527 
528 static nokprobe_inline int write_mem_unaligned(unsigned long val,
529 					       unsigned long ea, int nb,
530 					       struct pt_regs *regs)
531 {
532 	union {
533 		unsigned long ul;
534 		u8 b[sizeof(unsigned long)];
535 	} u;
536 	int i;
537 
538 	u.ul = val;
539 	i = IS_BE ? sizeof(unsigned long) - nb : 0;
540 	return copy_mem_out(&u.b[i], ea, nb, regs);
541 }
542 
543 /*
544  * Write memory at address ea for nb bytes, return 0 for success
545  * or -EFAULT if an error occurred.  N.B. nb must be 1, 2, 4 or 8.
546  */
547 static int write_mem(unsigned long val, unsigned long ea, int nb,
548 			       struct pt_regs *regs)
549 {
550 	if (!address_ok(regs, ea, nb))
551 		return -EFAULT;
552 	if ((ea & (nb - 1)) == 0)
553 		return write_mem_aligned(val, ea, nb, regs);
554 	return write_mem_unaligned(val, ea, nb, regs);
555 }
556 NOKPROBE_SYMBOL(write_mem);
557 
558 #ifdef CONFIG_PPC_FPU
559 /*
560  * These access either the real FP register or the image in the
561  * thread_struct, depending on regs->msr & MSR_FP.
562  */
563 static int do_fp_load(struct instruction_op *op, unsigned long ea,
564 		      struct pt_regs *regs, bool cross_endian)
565 {
566 	int err, rn, nb;
567 	union {
568 		int i;
569 		unsigned int u;
570 		float f;
571 		double d[2];
572 		unsigned long l[2];
573 		u8 b[2 * sizeof(double)];
574 	} u;
575 
576 	nb = GETSIZE(op->type);
577 	if (nb > sizeof(u))
578 		return -EINVAL;
579 	if (!address_ok(regs, ea, nb))
580 		return -EFAULT;
581 	rn = op->reg;
582 	err = copy_mem_in(u.b, ea, nb, regs);
583 	if (err)
584 		return err;
585 	if (unlikely(cross_endian)) {
586 		do_byte_reverse(u.b, min(nb, 8));
587 		if (nb == 16)
588 			do_byte_reverse(&u.b[8], 8);
589 	}
590 	preempt_disable();
591 	if (nb == 4) {
592 		if (op->type & FPCONV)
593 			conv_sp_to_dp(&u.f, &u.d[0]);
594 		else if (op->type & SIGNEXT)
595 			u.l[0] = u.i;
596 		else
597 			u.l[0] = u.u;
598 	}
599 	if (regs->msr & MSR_FP)
600 		put_fpr(rn, &u.d[0]);
601 	else
602 		current->thread.TS_FPR(rn) = u.l[0];
603 	if (nb == 16) {
604 		/* lfdp */
605 		rn |= 1;
606 		if (regs->msr & MSR_FP)
607 			put_fpr(rn, &u.d[1]);
608 		else
609 			current->thread.TS_FPR(rn) = u.l[1];
610 	}
611 	preempt_enable();
612 	return 0;
613 }
614 NOKPROBE_SYMBOL(do_fp_load);
615 
616 static int do_fp_store(struct instruction_op *op, unsigned long ea,
617 		       struct pt_regs *regs, bool cross_endian)
618 {
619 	int rn, nb;
620 	union {
621 		unsigned int u;
622 		float f;
623 		double d[2];
624 		unsigned long l[2];
625 		u8 b[2 * sizeof(double)];
626 	} u;
627 
628 	nb = GETSIZE(op->type);
629 	if (nb > sizeof(u))
630 		return -EINVAL;
631 	if (!address_ok(regs, ea, nb))
632 		return -EFAULT;
633 	rn = op->reg;
634 	preempt_disable();
635 	if (regs->msr & MSR_FP)
636 		get_fpr(rn, &u.d[0]);
637 	else
638 		u.l[0] = current->thread.TS_FPR(rn);
639 	if (nb == 4) {
640 		if (op->type & FPCONV)
641 			conv_dp_to_sp(&u.d[0], &u.f);
642 		else
643 			u.u = u.l[0];
644 	}
645 	if (nb == 16) {
646 		rn |= 1;
647 		if (regs->msr & MSR_FP)
648 			get_fpr(rn, &u.d[1]);
649 		else
650 			u.l[1] = current->thread.TS_FPR(rn);
651 	}
652 	preempt_enable();
653 	if (unlikely(cross_endian)) {
654 		do_byte_reverse(u.b, min(nb, 8));
655 		if (nb == 16)
656 			do_byte_reverse(&u.b[8], 8);
657 	}
658 	return copy_mem_out(u.b, ea, nb, regs);
659 }
660 NOKPROBE_SYMBOL(do_fp_store);
661 #endif
662 
663 #ifdef CONFIG_ALTIVEC
664 /* For Altivec/VMX, no need to worry about alignment */
665 static nokprobe_inline int do_vec_load(int rn, unsigned long ea,
666 				       int size, struct pt_regs *regs,
667 				       bool cross_endian)
668 {
669 	int err;
670 	union {
671 		__vector128 v;
672 		u8 b[sizeof(__vector128)];
673 	} u = {};
674 
675 	if (size > sizeof(u))
676 		return -EINVAL;
677 
678 	if (!address_ok(regs, ea & ~0xfUL, 16))
679 		return -EFAULT;
680 	/* align to multiple of size */
681 	ea &= ~(size - 1);
682 	err = copy_mem_in(&u.b[ea & 0xf], ea, size, regs);
683 	if (err)
684 		return err;
685 	if (unlikely(cross_endian))
686 		do_byte_reverse(&u.b[ea & 0xf], min_t(size_t, size, sizeof(u)));
687 	preempt_disable();
688 	if (regs->msr & MSR_VEC)
689 		put_vr(rn, &u.v);
690 	else
691 		current->thread.vr_state.vr[rn] = u.v;
692 	preempt_enable();
693 	return 0;
694 }
695 
696 static nokprobe_inline int do_vec_store(int rn, unsigned long ea,
697 					int size, struct pt_regs *regs,
698 					bool cross_endian)
699 {
700 	union {
701 		__vector128 v;
702 		u8 b[sizeof(__vector128)];
703 	} u;
704 
705 	if (size > sizeof(u))
706 		return -EINVAL;
707 
708 	if (!address_ok(regs, ea & ~0xfUL, 16))
709 		return -EFAULT;
710 	/* align to multiple of size */
711 	ea &= ~(size - 1);
712 
713 	preempt_disable();
714 	if (regs->msr & MSR_VEC)
715 		get_vr(rn, &u.v);
716 	else
717 		u.v = current->thread.vr_state.vr[rn];
718 	preempt_enable();
719 	if (unlikely(cross_endian))
720 		do_byte_reverse(&u.b[ea & 0xf], min_t(size_t, size, sizeof(u)));
721 	return copy_mem_out(&u.b[ea & 0xf], ea, size, regs);
722 }
723 #endif /* CONFIG_ALTIVEC */
724 
725 #ifdef CONFIG_PPC64
726 static nokprobe_inline int emulate_lq(struct pt_regs *regs, unsigned long ea,
727 				      int reg, bool cross_endian)
728 {
729 	int err;
730 
731 	if (!address_ok(regs, ea, 16))
732 		return -EFAULT;
733 	/* if aligned, should be atomic */
734 	if ((ea & 0xf) == 0) {
735 		err = do_lq(ea, &regs->gpr[reg]);
736 	} else {
737 		err = read_mem(&regs->gpr[reg + IS_LE], ea, 8, regs);
738 		if (!err)
739 			err = read_mem(&regs->gpr[reg + IS_BE], ea + 8, 8, regs);
740 	}
741 	if (!err && unlikely(cross_endian))
742 		do_byte_reverse(&regs->gpr[reg], 16);
743 	return err;
744 }
745 
746 static nokprobe_inline int emulate_stq(struct pt_regs *regs, unsigned long ea,
747 				       int reg, bool cross_endian)
748 {
749 	int err;
750 	unsigned long vals[2];
751 
752 	if (!address_ok(regs, ea, 16))
753 		return -EFAULT;
754 	vals[0] = regs->gpr[reg];
755 	vals[1] = regs->gpr[reg + 1];
756 	if (unlikely(cross_endian))
757 		do_byte_reverse(vals, 16);
758 
759 	/* if aligned, should be atomic */
760 	if ((ea & 0xf) == 0)
761 		return do_stq(ea, vals[0], vals[1]);
762 
763 	err = write_mem(vals[IS_LE], ea, 8, regs);
764 	if (!err)
765 		err = write_mem(vals[IS_BE], ea + 8, 8, regs);
766 	return err;
767 }
768 #endif /* CONFIG_PPC64 */
769 
770 #ifdef CONFIG_VSX
771 static nokprobe_inline void emulate_vsx_load(struct instruction_op *op, union vsx_reg *reg,
772 					     const void *mem, bool rev)
773 {
774 	int size, read_size;
775 	int i, j;
776 	const unsigned int *wp;
777 	const unsigned short *hp;
778 	const unsigned char *bp;
779 
780 	size = GETSIZE(op->type);
781 	reg->d[0] = reg->d[1] = 0;
782 
783 	switch (op->element_size) {
784 	case 32:
785 		/* [p]lxvp[x] */
786 	case 16:
787 		/* whole vector; lxv[x] or lxvl[l] */
788 		if (size == 0)
789 			break;
790 		memcpy(reg, mem, size);
791 		if (IS_LE && (op->vsx_flags & VSX_LDLEFT))
792 			rev = !rev;
793 		if (rev)
794 			do_byte_reverse(reg, size);
795 		break;
796 	case 8:
797 		/* scalar loads, lxvd2x, lxvdsx */
798 		read_size = (size >= 8) ? 8 : size;
799 		i = IS_LE ? 8 : 8 - read_size;
800 		memcpy(&reg->b[i], mem, read_size);
801 		if (rev)
802 			do_byte_reverse(&reg->b[i], 8);
803 		if (size < 8) {
804 			if (op->type & SIGNEXT) {
805 				/* size == 4 is the only case here */
806 				reg->d[IS_LE] = (signed int) reg->d[IS_LE];
807 			} else if (op->vsx_flags & VSX_FPCONV) {
808 				preempt_disable();
809 				conv_sp_to_dp(&reg->fp[1 + IS_LE],
810 					      &reg->dp[IS_LE]);
811 				preempt_enable();
812 			}
813 		} else {
814 			if (size == 16) {
815 				unsigned long v = *(unsigned long *)(mem + 8);
816 				reg->d[IS_BE] = !rev ? v : byterev_8(v);
817 			} else if (op->vsx_flags & VSX_SPLAT)
818 				reg->d[IS_BE] = reg->d[IS_LE];
819 		}
820 		break;
821 	case 4:
822 		/* lxvw4x, lxvwsx */
823 		wp = mem;
824 		for (j = 0; j < size / 4; ++j) {
825 			i = IS_LE ? 3 - j : j;
826 			reg->w[i] = !rev ? *wp++ : byterev_4(*wp++);
827 		}
828 		if (op->vsx_flags & VSX_SPLAT) {
829 			u32 val = reg->w[IS_LE ? 3 : 0];
830 			for (; j < 4; ++j) {
831 				i = IS_LE ? 3 - j : j;
832 				reg->w[i] = val;
833 			}
834 		}
835 		break;
836 	case 2:
837 		/* lxvh8x */
838 		hp = mem;
839 		for (j = 0; j < size / 2; ++j) {
840 			i = IS_LE ? 7 - j : j;
841 			reg->h[i] = !rev ? *hp++ : byterev_2(*hp++);
842 		}
843 		break;
844 	case 1:
845 		/* lxvb16x */
846 		bp = mem;
847 		for (j = 0; j < size; ++j) {
848 			i = IS_LE ? 15 - j : j;
849 			reg->b[i] = *bp++;
850 		}
851 		break;
852 	}
853 }
854 
855 static nokprobe_inline void emulate_vsx_store(struct instruction_op *op, const union vsx_reg *reg,
856 					      void *mem, bool rev)
857 {
858 	int size, write_size;
859 	int i, j;
860 	union vsx_reg buf;
861 	unsigned int *wp;
862 	unsigned short *hp;
863 	unsigned char *bp;
864 
865 	size = GETSIZE(op->type);
866 
867 	switch (op->element_size) {
868 	case 32:
869 		/* [p]stxvp[x] */
870 		if (size == 0)
871 			break;
872 		if (rev) {
873 			/* reverse 32 bytes */
874 			union vsx_reg buf32[2];
875 			buf32[0].d[0] = byterev_8(reg[1].d[1]);
876 			buf32[0].d[1] = byterev_8(reg[1].d[0]);
877 			buf32[1].d[0] = byterev_8(reg[0].d[1]);
878 			buf32[1].d[1] = byterev_8(reg[0].d[0]);
879 			memcpy(mem, buf32, size);
880 		} else {
881 			memcpy(mem, reg, size);
882 		}
883 		break;
884 	case 16:
885 		/* stxv, stxvx, stxvl, stxvll */
886 		if (size == 0)
887 			break;
888 		if (IS_LE && (op->vsx_flags & VSX_LDLEFT))
889 			rev = !rev;
890 		if (rev) {
891 			/* reverse 16 bytes */
892 			buf.d[0] = byterev_8(reg->d[1]);
893 			buf.d[1] = byterev_8(reg->d[0]);
894 			reg = &buf;
895 		}
896 		memcpy(mem, reg, size);
897 		break;
898 	case 8:
899 		/* scalar stores, stxvd2x */
900 		write_size = (size >= 8) ? 8 : size;
901 		i = IS_LE ? 8 : 8 - write_size;
902 		if (size < 8 && op->vsx_flags & VSX_FPCONV) {
903 			buf.d[0] = buf.d[1] = 0;
904 			preempt_disable();
905 			conv_dp_to_sp(&reg->dp[IS_LE], &buf.fp[1 + IS_LE]);
906 			preempt_enable();
907 			reg = &buf;
908 		}
909 		memcpy(mem, &reg->b[i], write_size);
910 		if (size == 16)
911 			memcpy(mem + 8, &reg->d[IS_BE], 8);
912 		if (unlikely(rev)) {
913 			do_byte_reverse(mem, write_size);
914 			if (size == 16)
915 				do_byte_reverse(mem + 8, 8);
916 		}
917 		break;
918 	case 4:
919 		/* stxvw4x */
920 		wp = mem;
921 		for (j = 0; j < size / 4; ++j) {
922 			i = IS_LE ? 3 - j : j;
923 			*wp++ = !rev ? reg->w[i] : byterev_4(reg->w[i]);
924 		}
925 		break;
926 	case 2:
927 		/* stxvh8x */
928 		hp = mem;
929 		for (j = 0; j < size / 2; ++j) {
930 			i = IS_LE ? 7 - j : j;
931 			*hp++ = !rev ? reg->h[i] : byterev_2(reg->h[i]);
932 		}
933 		break;
934 	case 1:
935 		/* stvxb16x */
936 		bp = mem;
937 		for (j = 0; j < size; ++j) {
938 			i = IS_LE ? 15 - j : j;
939 			*bp++ = reg->b[i];
940 		}
941 		break;
942 	}
943 }
944 
945 static nokprobe_inline int do_vsx_load(struct instruction_op *op,
946 				       unsigned long ea, struct pt_regs *regs,
947 				       bool cross_endian)
948 {
949 	int reg = op->reg;
950 	int i, j, nr_vsx_regs;
951 	u8 mem[32];
952 	union vsx_reg buf[2];
953 	int size = GETSIZE(op->type);
954 
955 	if (!address_ok(regs, ea, size) || copy_mem_in(mem, ea, size, regs))
956 		return -EFAULT;
957 
958 	nr_vsx_regs = max(1ul, size / sizeof(__vector128));
959 	emulate_vsx_load(op, buf, mem, cross_endian);
960 	preempt_disable();
961 	if (reg < 32) {
962 		/* FP regs + extensions */
963 		if (regs->msr & MSR_FP) {
964 			for (i = 0; i < nr_vsx_regs; i++) {
965 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
966 				load_vsrn(reg + i, &buf[j].v);
967 			}
968 		} else {
969 			for (i = 0; i < nr_vsx_regs; i++) {
970 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
971 				current->thread.fp_state.fpr[reg + i][0] = buf[j].d[0];
972 				current->thread.fp_state.fpr[reg + i][1] = buf[j].d[1];
973 			}
974 		}
975 	} else {
976 		if (regs->msr & MSR_VEC) {
977 			for (i = 0; i < nr_vsx_regs; i++) {
978 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
979 				load_vsrn(reg + i, &buf[j].v);
980 			}
981 		} else {
982 			for (i = 0; i < nr_vsx_regs; i++) {
983 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
984 				current->thread.vr_state.vr[reg - 32 + i] = buf[j].v;
985 			}
986 		}
987 	}
988 	preempt_enable();
989 	return 0;
990 }
991 
992 static nokprobe_inline int do_vsx_store(struct instruction_op *op,
993 					unsigned long ea, struct pt_regs *regs,
994 					bool cross_endian)
995 {
996 	int reg = op->reg;
997 	int i, j, nr_vsx_regs;
998 	u8 mem[32];
999 	union vsx_reg buf[2];
1000 	int size = GETSIZE(op->type);
1001 
1002 	if (!address_ok(regs, ea, size))
1003 		return -EFAULT;
1004 
1005 	nr_vsx_regs = max(1ul, size / sizeof(__vector128));
1006 	preempt_disable();
1007 	if (reg < 32) {
1008 		/* FP regs + extensions */
1009 		if (regs->msr & MSR_FP) {
1010 			for (i = 0; i < nr_vsx_regs; i++) {
1011 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
1012 				store_vsrn(reg + i, &buf[j].v);
1013 			}
1014 		} else {
1015 			for (i = 0; i < nr_vsx_regs; i++) {
1016 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
1017 				buf[j].d[0] = current->thread.fp_state.fpr[reg + i][0];
1018 				buf[j].d[1] = current->thread.fp_state.fpr[reg + i][1];
1019 			}
1020 		}
1021 	} else {
1022 		if (regs->msr & MSR_VEC) {
1023 			for (i = 0; i < nr_vsx_regs; i++) {
1024 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
1025 				store_vsrn(reg + i, &buf[j].v);
1026 			}
1027 		} else {
1028 			for (i = 0; i < nr_vsx_regs; i++) {
1029 				j = IS_LE ? nr_vsx_regs - i - 1 : i;
1030 				buf[j].v = current->thread.vr_state.vr[reg - 32 + i];
1031 			}
1032 		}
1033 	}
1034 	preempt_enable();
1035 	emulate_vsx_store(op, buf, mem, cross_endian);
1036 	return  copy_mem_out(mem, ea, size, regs);
1037 }
1038 #endif /* CONFIG_VSX */
1039 
1040 static __always_inline int __emulate_dcbz(unsigned long ea)
1041 {
1042 	unsigned long i;
1043 	unsigned long size = l1_dcache_bytes();
1044 
1045 	for (i = 0; i < size; i += sizeof(long))
1046 		unsafe_put_user(0, (unsigned long __user *)(ea + i), Efault);
1047 
1048 	return 0;
1049 
1050 Efault:
1051 	return -EFAULT;
1052 }
1053 
1054 int emulate_dcbz(unsigned long ea, struct pt_regs *regs)
1055 {
1056 	void __user *uea = (void __user *)ea;
1057 	int err;
1058 	unsigned long size = l1_dcache_bytes();
1059 
1060 	ea = truncate_if_32bit(regs->msr, ea);
1061 	ea &= ~(size - 1);
1062 	if (!address_ok(regs, ea, size))
1063 		return -EFAULT;
1064 
1065 	if (is_kernel_addr(ea))
1066 		err = __emulate_dcbz(ea);
1067 	else
1068 		scoped_user_write_access_size(uea, size, efault)
1069 			err = __emulate_dcbz((unsigned long)uea);
1070 
1071 	if (err)
1072 		regs->dar = ea;
1073 
1074 	return err;
1075 
1076 efault:
1077 	regs->dar = ea;
1078 	return -EFAULT;
1079 }
1080 NOKPROBE_SYMBOL(emulate_dcbz);
1081 
1082 #define __put_user_asmx(x, addr, err, op, cr)		\
1083 	__asm__ __volatile__(				\
1084 		".machine push\n"			\
1085 		".machine power8\n"			\
1086 		"1:	" op " %2,0,%3\n"		\
1087 		".machine pop\n"			\
1088 		"	mfcr	%1\n"			\
1089 		"2:\n"					\
1090 		".section .fixup,\"ax\"\n"		\
1091 		"3:	li	%0,%4\n"		\
1092 		"	b	2b\n"			\
1093 		".previous\n"				\
1094 		EX_TABLE(1b, 3b)			\
1095 		: "=r" (err), "=r" (cr)			\
1096 		: "r" (x), "r" (addr), "i" (-EFAULT), "0" (err))
1097 
1098 #define __get_user_asmx(x, addr, err, op)		\
1099 	__asm__ __volatile__(				\
1100 		".machine push\n"			\
1101 		".machine power8\n"			\
1102 		"1:	"op" %1,0,%2\n"			\
1103 		".machine pop\n"			\
1104 		"2:\n"					\
1105 		".section .fixup,\"ax\"\n"		\
1106 		"3:	li	%0,%3\n"		\
1107 		"	b	2b\n"			\
1108 		".previous\n"				\
1109 		EX_TABLE(1b, 3b)			\
1110 		: "=r" (err), "=r" (x)			\
1111 		: "r" (addr), "i" (-EFAULT), "0" (err))
1112 
1113 #define __cacheop_user_asmx(addr, err, op)		\
1114 	__asm__ __volatile__(				\
1115 		"1:	"op" 0,%1\n"			\
1116 		"2:\n"					\
1117 		".section .fixup,\"ax\"\n"		\
1118 		"3:	li	%0,%3\n"		\
1119 		"	b	2b\n"			\
1120 		".previous\n"				\
1121 		EX_TABLE(1b, 3b)			\
1122 		: "=r" (err)				\
1123 		: "r" (addr), "i" (-EFAULT), "0" (err))
1124 
1125 static nokprobe_inline void set_cr0(const struct pt_regs *regs,
1126 				    struct instruction_op *op)
1127 {
1128 	long val = op->val;
1129 
1130 	op->type |= SETCC;
1131 	op->ccval = (regs->ccr & 0x0fffffff) | ((regs->xer >> 3) & 0x10000000);
1132 	if (!(regs->msr & MSR_64BIT))
1133 		val = (int) val;
1134 	if (val < 0)
1135 		op->ccval |= 0x80000000;
1136 	else if (val > 0)
1137 		op->ccval |= 0x40000000;
1138 	else
1139 		op->ccval |= 0x20000000;
1140 }
1141 
1142 static nokprobe_inline void set_ca32(struct instruction_op *op, bool val)
1143 {
1144 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1145 		if (val)
1146 			op->xerval |= XER_CA32;
1147 		else
1148 			op->xerval &= ~XER_CA32;
1149 	}
1150 }
1151 
1152 static nokprobe_inline void add_with_carry(const struct pt_regs *regs,
1153 				     struct instruction_op *op, int rd,
1154 				     unsigned long val1, unsigned long val2,
1155 				     unsigned long carry_in)
1156 {
1157 	unsigned long val = val1 + val2;
1158 
1159 	if (carry_in)
1160 		++val;
1161 	op->type = COMPUTE | SETREG | SETXER;
1162 	op->reg = rd;
1163 	op->val = val;
1164 	val = truncate_if_32bit(regs->msr, val);
1165 	val1 = truncate_if_32bit(regs->msr, val1);
1166 	op->xerval = regs->xer;
1167 	if (val < val1 || (carry_in && val == val1))
1168 		op->xerval |= XER_CA;
1169 	else
1170 		op->xerval &= ~XER_CA;
1171 
1172 	set_ca32(op, (unsigned int)val < (unsigned int)val1 ||
1173 			(carry_in && (unsigned int)val == (unsigned int)val1));
1174 }
1175 
1176 static nokprobe_inline void do_cmp_signed(const struct pt_regs *regs,
1177 					  struct instruction_op *op,
1178 					  long v1, long v2, int crfld)
1179 {
1180 	unsigned int crval, shift;
1181 
1182 	op->type = COMPUTE | SETCC;
1183 	crval = (regs->xer >> 31) & 1;		/* get SO bit */
1184 	if (v1 < v2)
1185 		crval |= 8;
1186 	else if (v1 > v2)
1187 		crval |= 4;
1188 	else
1189 		crval |= 2;
1190 	shift = (7 - crfld) * 4;
1191 	op->ccval = (regs->ccr & ~(0xf << shift)) | (crval << shift);
1192 }
1193 
1194 static nokprobe_inline void do_cmp_unsigned(const struct pt_regs *regs,
1195 					    struct instruction_op *op,
1196 					    unsigned long v1,
1197 					    unsigned long v2, int crfld)
1198 {
1199 	unsigned int crval, shift;
1200 
1201 	op->type = COMPUTE | SETCC;
1202 	crval = (regs->xer >> 31) & 1;		/* get SO bit */
1203 	if (v1 < v2)
1204 		crval |= 8;
1205 	else if (v1 > v2)
1206 		crval |= 4;
1207 	else
1208 		crval |= 2;
1209 	shift = (7 - crfld) * 4;
1210 	op->ccval = (regs->ccr & ~(0xf << shift)) | (crval << shift);
1211 }
1212 
1213 static nokprobe_inline void do_cmpb(const struct pt_regs *regs,
1214 				    struct instruction_op *op,
1215 				    unsigned long v1, unsigned long v2)
1216 {
1217 	unsigned long long out_val, mask;
1218 	int i;
1219 
1220 	out_val = 0;
1221 	for (i = 0; i < 8; i++) {
1222 		mask = 0xffUL << (i * 8);
1223 		if ((v1 & mask) == (v2 & mask))
1224 			out_val |= mask;
1225 	}
1226 	op->val = out_val;
1227 }
1228 
1229 /*
1230  * The size parameter is used to adjust the equivalent popcnt instruction.
1231  * popcntb = 8, popcntw = 32, popcntd = 64
1232  */
1233 static nokprobe_inline void do_popcnt(const struct pt_regs *regs,
1234 				      struct instruction_op *op,
1235 				      unsigned long v1, int size)
1236 {
1237 	unsigned long long out = v1;
1238 
1239 	out -= (out >> 1) & 0x5555555555555555ULL;
1240 	out = (0x3333333333333333ULL & out) +
1241 	      (0x3333333333333333ULL & (out >> 2));
1242 	out = (out + (out >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
1243 
1244 	if (size == 8) {	/* popcntb */
1245 		op->val = out;
1246 		return;
1247 	}
1248 	out += out >> 8;
1249 	out += out >> 16;
1250 	if (size == 32) {	/* popcntw */
1251 		op->val = out & 0x0000003f0000003fULL;
1252 		return;
1253 	}
1254 
1255 	out = (out + (out >> 32)) & 0x7f;
1256 	op->val = out;	/* popcntd */
1257 }
1258 
1259 #ifdef CONFIG_PPC64
1260 static nokprobe_inline void do_bpermd(const struct pt_regs *regs,
1261 				      struct instruction_op *op,
1262 				      unsigned long v1, unsigned long v2)
1263 {
1264 	unsigned char perm, idx;
1265 	unsigned int i;
1266 
1267 	perm = 0;
1268 	for (i = 0; i < 8; i++) {
1269 		idx = (v1 >> (i * 8)) & 0xff;
1270 		if (idx < 64)
1271 			if (v2 & PPC_BIT(idx))
1272 				perm |= 1 << i;
1273 	}
1274 	op->val = perm;
1275 }
1276 #endif /* CONFIG_PPC64 */
1277 /*
1278  * The size parameter adjusts the equivalent prty instruction.
1279  * prtyw = 32, prtyd = 64
1280  */
1281 static nokprobe_inline void do_prty(const struct pt_regs *regs,
1282 				    struct instruction_op *op,
1283 				    unsigned long v, int size)
1284 {
1285 	unsigned long long res = v ^ (v >> 8);
1286 
1287 	res ^= res >> 16;
1288 	if (size == 32) {		/* prtyw */
1289 		op->val = res & 0x0000000100000001ULL;
1290 		return;
1291 	}
1292 
1293 	res ^= res >> 32;
1294 	op->val = res & 1;	/*prtyd */
1295 }
1296 
1297 static nokprobe_inline int trap_compare(long v1, long v2)
1298 {
1299 	int ret = 0;
1300 
1301 	if (v1 < v2)
1302 		ret |= 0x10;
1303 	else if (v1 > v2)
1304 		ret |= 0x08;
1305 	else
1306 		ret |= 0x04;
1307 	if ((unsigned long)v1 < (unsigned long)v2)
1308 		ret |= 0x02;
1309 	else if ((unsigned long)v1 > (unsigned long)v2)
1310 		ret |= 0x01;
1311 	return ret;
1312 }
1313 
1314 /*
1315  * Elements of 32-bit rotate and mask instructions.
1316  */
1317 #define MASK32(mb, me)	((0xffffffffUL >> (mb)) + \
1318 			 ((signed long)-0x80000000L >> (me)) + ((me) >= (mb)))
1319 #ifdef CONFIG_PPC64
1320 #define MASK64_L(mb)	(~0UL >> (mb))
1321 #define MASK64_R(me)	((signed long)-0x8000000000000000L >> (me))
1322 #define MASK64(mb, me)	(MASK64_L(mb) + MASK64_R(me) + ((me) >= (mb)))
1323 #define DATA32(x)	(((x) & 0xffffffffUL) | (((x) & 0xffffffffUL) << 32))
1324 #else
1325 #define DATA32(x)	(x)
1326 #endif
1327 #define ROTATE(x, n)	((n) ? (((x) << (n)) | ((x) >> (8 * sizeof(long) - (n)))) : (x))
1328 
1329 /*
1330  * Decode an instruction, and return information about it in *op
1331  * without changing *regs.
1332  * Integer arithmetic and logical instructions, branches, and barrier
1333  * instructions can be emulated just using the information in *op.
1334  *
1335  * Return value is 1 if the instruction can be emulated just by
1336  * updating *regs with the information in *op, -1 if we need the
1337  * GPRs but *regs doesn't contain the full register set, or 0
1338  * otherwise.
1339  */
1340 int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
1341 		  ppc_inst_t instr)
1342 {
1343 #ifdef CONFIG_PPC64
1344 	unsigned int suffixopcode, prefixtype, prefix_r;
1345 	unsigned int rc, suffix;
1346 #endif
1347 	unsigned int opcode, ra, rb, rd, spr, u;
1348 	unsigned long int imm;
1349 	unsigned long int val, val2;
1350 	unsigned int mb, me, sh;
1351 	unsigned int word;
1352 	long ival;
1353 
1354 	word = ppc_inst_val(instr);
1355 #ifdef CONFIG_PPC64
1356 	suffix = ppc_inst_suffix(instr);
1357 #endif
1358 
1359 	op->type = COMPUTE;
1360 
1361 	opcode = ppc_inst_primary_opcode(instr);
1362 	switch (opcode) {
1363 	case 16:	/* bc */
1364 		op->type = BRANCH;
1365 		imm = (signed short)(word & 0xfffc);
1366 		if ((word & 2) == 0)
1367 			imm += regs->nip;
1368 		op->val = truncate_if_32bit(regs->msr, imm);
1369 		if (word & 1)
1370 			op->type |= SETLK;
1371 		if (branch_taken(word, regs, op))
1372 			op->type |= BRTAKEN;
1373 		return 1;
1374 	case 17:	/* sc */
1375 		if ((word & 0xfe2) == 2)
1376 			op->type = SYSCALL;
1377 		else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
1378 				(word & 0xfe3) == 1) {	/* scv */
1379 			op->type = SYSCALL_VECTORED_0;
1380 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
1381 				goto unknown_opcode;
1382 		} else
1383 			op->type = UNKNOWN;
1384 		return 0;
1385 	case 18:	/* b */
1386 		op->type = BRANCH | BRTAKEN;
1387 		imm = word & 0x03fffffc;
1388 		if (imm & 0x02000000)
1389 			imm -= 0x04000000;
1390 		if ((word & 2) == 0)
1391 			imm += regs->nip;
1392 		op->val = truncate_if_32bit(regs->msr, imm);
1393 		if (word & 1)
1394 			op->type |= SETLK;
1395 		return 1;
1396 	case 19:
1397 		switch ((word >> 1) & 0x3ff) {
1398 		case 0:		/* mcrf */
1399 			op->type = COMPUTE + SETCC;
1400 			rd = 7 - ((word >> 23) & 0x7);
1401 			ra = 7 - ((word >> 18) & 0x7);
1402 			rd *= 4;
1403 			ra *= 4;
1404 			val = (regs->ccr >> ra) & 0xf;
1405 			op->ccval = (regs->ccr & ~(0xfUL << rd)) | (val << rd);
1406 			return 1;
1407 
1408 		case 16:	/* bclr */
1409 		case 528:	/* bcctr */
1410 			op->type = BRANCH;
1411 			imm = (word & 0x400)? regs->ctr: regs->link;
1412 			op->val = truncate_if_32bit(regs->msr, imm);
1413 			if (word & 1)
1414 				op->type |= SETLK;
1415 			if (branch_taken(word, regs, op))
1416 				op->type |= BRTAKEN;
1417 			return 1;
1418 
1419 		case 18:	/* rfid, scary */
1420 			if (user_mode(regs))
1421 				goto priv;
1422 			op->type = RFI;
1423 			return 0;
1424 
1425 		case 150:	/* isync */
1426 			op->type = BARRIER | BARRIER_ISYNC;
1427 			return 1;
1428 
1429 		case 33:	/* crnor */
1430 		case 129:	/* crandc */
1431 		case 193:	/* crxor */
1432 		case 225:	/* crnand */
1433 		case 257:	/* crand */
1434 		case 289:	/* creqv */
1435 		case 417:	/* crorc */
1436 		case 449:	/* cror */
1437 			op->type = COMPUTE + SETCC;
1438 			ra = (word >> 16) & 0x1f;
1439 			rb = (word >> 11) & 0x1f;
1440 			rd = (word >> 21) & 0x1f;
1441 			ra = (regs->ccr >> (31 - ra)) & 1;
1442 			rb = (regs->ccr >> (31 - rb)) & 1;
1443 			val = (word >> (6 + ra * 2 + rb)) & 1;
1444 			op->ccval = (regs->ccr & ~(1UL << (31 - rd))) |
1445 				(val << (31 - rd));
1446 			return 1;
1447 		}
1448 		break;
1449 	case 31:
1450 		switch ((word >> 1) & 0x3ff) {
1451 		case 598:	/* sync */
1452 			op->type = BARRIER + BARRIER_SYNC;
1453 #ifdef CONFIG_PPC64
1454 			switch ((word >> 21) & 3) {
1455 			case 1:		/* lwsync */
1456 				op->type = BARRIER + BARRIER_LWSYNC;
1457 				break;
1458 			case 2:		/* ptesync */
1459 				op->type = BARRIER + BARRIER_PTESYNC;
1460 				break;
1461 			}
1462 #endif
1463 			return 1;
1464 
1465 		case 854:	/* eieio */
1466 			op->type = BARRIER + BARRIER_EIEIO;
1467 			return 1;
1468 		}
1469 		break;
1470 	}
1471 
1472 	rd = (word >> 21) & 0x1f;
1473 	ra = (word >> 16) & 0x1f;
1474 	rb = (word >> 11) & 0x1f;
1475 #ifdef CONFIG_PPC64
1476 	rc = (word >> 6) & 0x1f;
1477 #endif
1478 
1479 	switch (opcode) {
1480 #ifdef CONFIG_PPC64
1481 	case 1:
1482 		if (!cpu_has_feature(CPU_FTR_ARCH_31))
1483 			goto unknown_opcode;
1484 
1485 		prefix_r = GET_PREFIX_R(word);
1486 		ra = GET_PREFIX_RA(suffix);
1487 		rd = (suffix >> 21) & 0x1f;
1488 		op->reg = rd;
1489 		op->val = regs->gpr[rd];
1490 		suffixopcode = get_op(suffix);
1491 		prefixtype = (word >> 24) & 0x3;
1492 		switch (prefixtype) {
1493 		case 2:
1494 			if (prefix_r && ra)
1495 				return 0;
1496 			switch (suffixopcode) {
1497 			case 14:	/* paddi */
1498 				op->type = COMPUTE | PREFIXED;
1499 				op->val = mlsd_8lsd_ea(word, suffix, regs);
1500 				goto compute_done;
1501 			}
1502 		}
1503 		break;
1504 	case 2:		/* tdi */
1505 		if (rd & trap_compare(regs->gpr[ra], (short) word))
1506 			goto trap;
1507 		return 1;
1508 #endif
1509 	case 3:		/* twi */
1510 		if (rd & trap_compare((int)regs->gpr[ra], (short) word))
1511 			goto trap;
1512 		return 1;
1513 
1514 #ifdef CONFIG_PPC64
1515 	case 4:
1516 		/*
1517 		 * There are very many instructions with this primary opcode
1518 		 * introduced in the ISA as early as v2.03. However, the ones
1519 		 * we currently emulate were all introduced with ISA 3.0
1520 		 */
1521 		if (!cpu_has_feature(CPU_FTR_ARCH_300))
1522 			goto unknown_opcode;
1523 
1524 		switch (word & 0x3f) {
1525 		case 48:	/* maddhd */
1526 			asm volatile(PPC_MADDHD(%0, %1, %2, %3) :
1527 				     "=r" (op->val) : "r" (regs->gpr[ra]),
1528 				     "r" (regs->gpr[rb]), "r" (regs->gpr[rc]));
1529 			goto compute_done;
1530 
1531 		case 49:	/* maddhdu */
1532 			asm volatile(PPC_MADDHDU(%0, %1, %2, %3) :
1533 				     "=r" (op->val) : "r" (regs->gpr[ra]),
1534 				     "r" (regs->gpr[rb]), "r" (regs->gpr[rc]));
1535 			goto compute_done;
1536 
1537 		case 51:	/* maddld */
1538 			asm volatile(PPC_MADDLD(%0, %1, %2, %3) :
1539 				     "=r" (op->val) : "r" (regs->gpr[ra]),
1540 				     "r" (regs->gpr[rb]), "r" (regs->gpr[rc]));
1541 			goto compute_done;
1542 		}
1543 
1544 		/*
1545 		 * There are other instructions from ISA 3.0 with the same
1546 		 * primary opcode which do not have emulation support yet.
1547 		 */
1548 		goto unknown_opcode;
1549 #endif
1550 
1551 	case 7:		/* mulli */
1552 		op->val = regs->gpr[ra] * (short) word;
1553 		goto compute_done;
1554 
1555 	case 8:		/* subfic */
1556 		imm = (short) word;
1557 		add_with_carry(regs, op, rd, ~regs->gpr[ra], imm, 1);
1558 		return 1;
1559 
1560 	case 10:	/* cmpli */
1561 		imm = (unsigned short) word;
1562 		val = regs->gpr[ra];
1563 #ifdef CONFIG_PPC64
1564 		if ((rd & 1) == 0)
1565 			val = (unsigned int) val;
1566 #endif
1567 		do_cmp_unsigned(regs, op, val, imm, rd >> 2);
1568 		return 1;
1569 
1570 	case 11:	/* cmpi */
1571 		imm = (short) word;
1572 		val = regs->gpr[ra];
1573 #ifdef CONFIG_PPC64
1574 		if ((rd & 1) == 0)
1575 			val = (int) val;
1576 #endif
1577 		do_cmp_signed(regs, op, val, imm, rd >> 2);
1578 		return 1;
1579 
1580 	case 12:	/* addic */
1581 		imm = (short) word;
1582 		add_with_carry(regs, op, rd, regs->gpr[ra], imm, 0);
1583 		return 1;
1584 
1585 	case 13:	/* addic. */
1586 		imm = (short) word;
1587 		add_with_carry(regs, op, rd, regs->gpr[ra], imm, 0);
1588 		set_cr0(regs, op);
1589 		return 1;
1590 
1591 	case 14:	/* addi */
1592 		imm = (short) word;
1593 		if (ra)
1594 			imm += regs->gpr[ra];
1595 		op->val = imm;
1596 		goto compute_done;
1597 
1598 	case 15:	/* addis */
1599 		imm = ((short) word) << 16;
1600 		if (ra)
1601 			imm += regs->gpr[ra];
1602 		op->val = imm;
1603 		goto compute_done;
1604 
1605 	case 19:
1606 		if (((word >> 1) & 0x1f) == 2) {
1607 			/* addpcis */
1608 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
1609 				goto unknown_opcode;
1610 			imm = (short) (word & 0xffc1);	/* d0 + d2 fields */
1611 			imm |= (word >> 15) & 0x3e;	/* d1 field */
1612 			op->val = regs->nip + (imm << 16) + 4;
1613 			goto compute_done;
1614 		}
1615 		op->type = UNKNOWN;
1616 		return 0;
1617 
1618 	case 20:	/* rlwimi */
1619 		mb = (word >> 6) & 0x1f;
1620 		me = (word >> 1) & 0x1f;
1621 		val = DATA32(regs->gpr[rd]);
1622 		imm = MASK32(mb, me);
1623 		op->val = (regs->gpr[ra] & ~imm) | (ROTATE(val, rb) & imm);
1624 		goto logical_done;
1625 
1626 	case 21:	/* rlwinm */
1627 		mb = (word >> 6) & 0x1f;
1628 		me = (word >> 1) & 0x1f;
1629 		val = DATA32(regs->gpr[rd]);
1630 		op->val = ROTATE(val, rb) & MASK32(mb, me);
1631 		goto logical_done;
1632 
1633 	case 23:	/* rlwnm */
1634 		mb = (word >> 6) & 0x1f;
1635 		me = (word >> 1) & 0x1f;
1636 		rb = regs->gpr[rb] & 0x1f;
1637 		val = DATA32(regs->gpr[rd]);
1638 		op->val = ROTATE(val, rb) & MASK32(mb, me);
1639 		goto logical_done;
1640 
1641 	case 24:	/* ori */
1642 		op->val = regs->gpr[rd] | (unsigned short) word;
1643 		goto logical_done_nocc;
1644 
1645 	case 25:	/* oris */
1646 		imm = (unsigned short) word;
1647 		op->val = regs->gpr[rd] | (imm << 16);
1648 		goto logical_done_nocc;
1649 
1650 	case 26:	/* xori */
1651 		op->val = regs->gpr[rd] ^ (unsigned short) word;
1652 		goto logical_done_nocc;
1653 
1654 	case 27:	/* xoris */
1655 		imm = (unsigned short) word;
1656 		op->val = regs->gpr[rd] ^ (imm << 16);
1657 		goto logical_done_nocc;
1658 
1659 	case 28:	/* andi. */
1660 		op->val = regs->gpr[rd] & (unsigned short) word;
1661 		set_cr0(regs, op);
1662 		goto logical_done_nocc;
1663 
1664 	case 29:	/* andis. */
1665 		imm = (unsigned short) word;
1666 		op->val = regs->gpr[rd] & (imm << 16);
1667 		set_cr0(regs, op);
1668 		goto logical_done_nocc;
1669 
1670 #ifdef CONFIG_PPC64
1671 	case 30:	/* rld* */
1672 		mb = ((word >> 6) & 0x1f) | (word & 0x20);
1673 		val = regs->gpr[rd];
1674 		if ((word & 0x10) == 0) {
1675 			sh = rb | ((word & 2) << 4);
1676 			val = ROTATE(val, sh);
1677 			switch ((word >> 2) & 3) {
1678 			case 0:		/* rldicl */
1679 				val &= MASK64_L(mb);
1680 				break;
1681 			case 1:		/* rldicr */
1682 				val &= MASK64_R(mb);
1683 				break;
1684 			case 2:		/* rldic */
1685 				val &= MASK64(mb, 63 - sh);
1686 				break;
1687 			case 3:		/* rldimi */
1688 				imm = MASK64(mb, 63 - sh);
1689 				val = (regs->gpr[ra] & ~imm) |
1690 					(val & imm);
1691 			}
1692 			op->val = val;
1693 			goto logical_done;
1694 		} else {
1695 			sh = regs->gpr[rb] & 0x3f;
1696 			val = ROTATE(val, sh);
1697 			switch ((word >> 1) & 7) {
1698 			case 0:		/* rldcl */
1699 				op->val = val & MASK64_L(mb);
1700 				goto logical_done;
1701 			case 1:		/* rldcr */
1702 				op->val = val & MASK64_R(mb);
1703 				goto logical_done;
1704 			}
1705 		}
1706 #endif
1707 		op->type = UNKNOWN;	/* illegal instruction */
1708 		return 0;
1709 
1710 	case 31:
1711 		/* isel occupies 32 minor opcodes */
1712 		if (((word >> 1) & 0x1f) == 15) {
1713 			mb = (word >> 6) & 0x1f; /* bc field */
1714 			val = (regs->ccr >> (31 - mb)) & 1;
1715 			val2 = (ra) ? regs->gpr[ra] : 0;
1716 
1717 			op->val = (val) ? val2 : regs->gpr[rb];
1718 			goto compute_done;
1719 		}
1720 
1721 		switch ((word >> 1) & 0x3ff) {
1722 		case 4:		/* tw */
1723 			if (rd == 0x1f ||
1724 			    (rd & trap_compare((int)regs->gpr[ra],
1725 					       (int)regs->gpr[rb])))
1726 				goto trap;
1727 			return 1;
1728 #ifdef CONFIG_PPC64
1729 		case 68:	/* td */
1730 			if (rd & trap_compare(regs->gpr[ra], regs->gpr[rb]))
1731 				goto trap;
1732 			return 1;
1733 #endif
1734 		case 83:	/* mfmsr */
1735 			if (user_mode(regs))
1736 				goto priv;
1737 			op->type = MFMSR;
1738 			op->reg = rd;
1739 			return 0;
1740 		case 146:	/* mtmsr */
1741 			if (user_mode(regs))
1742 				goto priv;
1743 			op->type = MTMSR;
1744 			op->reg = rd;
1745 			op->val = 0xffffffff & ~(MSR_ME | MSR_LE);
1746 			return 0;
1747 #ifdef CONFIG_PPC64
1748 		case 178:	/* mtmsrd */
1749 			if (user_mode(regs))
1750 				goto priv;
1751 			op->type = MTMSR;
1752 			op->reg = rd;
1753 			/* only MSR_EE and MSR_RI get changed if bit 15 set */
1754 			/* mtmsrd doesn't change MSR_HV, MSR_ME or MSR_LE */
1755 			imm = (word & 0x10000)? 0x8002: 0xefffffffffffeffeUL;
1756 			op->val = imm;
1757 			return 0;
1758 #endif
1759 
1760 		case 19:	/* mfcr */
1761 			imm = 0xffffffffUL;
1762 			if ((word >> 20) & 1) {
1763 				imm = 0xf0000000UL;
1764 				for (sh = 0; sh < 8; ++sh) {
1765 					if (word & (0x80000 >> sh))
1766 						break;
1767 					imm >>= 4;
1768 				}
1769 			}
1770 			op->val = regs->ccr & imm;
1771 			goto compute_done;
1772 
1773 		case 128:	/* setb */
1774 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
1775 				goto unknown_opcode;
1776 			/*
1777 			 * 'ra' encodes the CR field number (bfa) in the top 3 bits.
1778 			 * Since each CR field is 4 bits,
1779 			 * we can simply mask off the bottom two bits (bfa * 4)
1780 			 * to yield the first bit in the CR field.
1781 			 */
1782 			ra = ra & ~0x3;
1783 			/* 'val' stores bits of the CR field (bfa) */
1784 			val = regs->ccr >> (CR0_SHIFT - ra);
1785 			/* checks if the LT bit of CR field (bfa) is set */
1786 			if (val & 8)
1787 				op->val = -1;
1788 			/* checks if the GT bit of CR field (bfa) is set */
1789 			else if (val & 4)
1790 				op->val = 1;
1791 			else
1792 				op->val = 0;
1793 			goto compute_done;
1794 
1795 		case 144:	/* mtcrf */
1796 			op->type = COMPUTE + SETCC;
1797 			imm = 0xf0000000UL;
1798 			val = regs->gpr[rd];
1799 			op->ccval = regs->ccr;
1800 			for (sh = 0; sh < 8; ++sh) {
1801 				if (word & (0x80000 >> sh))
1802 					op->ccval = (op->ccval & ~imm) |
1803 						(val & imm);
1804 				imm >>= 4;
1805 			}
1806 			return 1;
1807 
1808 		case 339:	/* mfspr */
1809 			spr = ((word >> 16) & 0x1f) | ((word >> 6) & 0x3e0);
1810 			op->type = MFSPR;
1811 			op->reg = rd;
1812 			op->spr = spr;
1813 			if (spr == SPRN_XER || spr == SPRN_LR ||
1814 			    spr == SPRN_CTR)
1815 				return 1;
1816 			return 0;
1817 
1818 		case 467:	/* mtspr */
1819 			spr = ((word >> 16) & 0x1f) | ((word >> 6) & 0x3e0);
1820 			op->type = MTSPR;
1821 			op->val = regs->gpr[rd];
1822 			op->spr = spr;
1823 			if (spr == SPRN_XER || spr == SPRN_LR ||
1824 			    spr == SPRN_CTR)
1825 				return 1;
1826 			return 0;
1827 
1828 /*
1829  * Compare instructions
1830  */
1831 		case 0:	/* cmp */
1832 			val = regs->gpr[ra];
1833 			val2 = regs->gpr[rb];
1834 #ifdef CONFIG_PPC64
1835 			if ((rd & 1) == 0) {
1836 				/* word (32-bit) compare */
1837 				val = (int) val;
1838 				val2 = (int) val2;
1839 			}
1840 #endif
1841 			do_cmp_signed(regs, op, val, val2, rd >> 2);
1842 			return 1;
1843 
1844 		case 32:	/* cmpl */
1845 			val = regs->gpr[ra];
1846 			val2 = regs->gpr[rb];
1847 #ifdef CONFIG_PPC64
1848 			if ((rd & 1) == 0) {
1849 				/* word (32-bit) compare */
1850 				val = (unsigned int) val;
1851 				val2 = (unsigned int) val2;
1852 			}
1853 #endif
1854 			do_cmp_unsigned(regs, op, val, val2, rd >> 2);
1855 			return 1;
1856 
1857 		case 508: /* cmpb */
1858 			do_cmpb(regs, op, regs->gpr[rd], regs->gpr[rb]);
1859 			goto logical_done_nocc;
1860 
1861 /*
1862  * Arithmetic instructions
1863  */
1864 		case 8:	/* subfc */
1865 			add_with_carry(regs, op, rd, ~regs->gpr[ra],
1866 				       regs->gpr[rb], 1);
1867 			goto arith_done;
1868 #ifdef CONFIG_PPC64
1869 		case 9:	/* mulhdu */
1870 			asm("mulhdu %0,%1,%2" : "=r" (op->val) :
1871 			    "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
1872 			goto arith_done;
1873 #endif
1874 		case 10:	/* addc */
1875 			add_with_carry(regs, op, rd, regs->gpr[ra],
1876 				       regs->gpr[rb], 0);
1877 			goto arith_done;
1878 
1879 		case 11:	/* mulhwu */
1880 			asm("mulhwu %0,%1,%2" : "=r" (op->val) :
1881 			    "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
1882 			goto arith_done;
1883 
1884 		case 40:	/* subf */
1885 			op->val = regs->gpr[rb] - regs->gpr[ra];
1886 			goto arith_done;
1887 #ifdef CONFIG_PPC64
1888 		case 73:	/* mulhd */
1889 			asm("mulhd %0,%1,%2" : "=r" (op->val) :
1890 			    "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
1891 			goto arith_done;
1892 #endif
1893 		case 75:	/* mulhw */
1894 			asm("mulhw %0,%1,%2" : "=r" (op->val) :
1895 			    "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
1896 			goto arith_done;
1897 
1898 		case 104:	/* neg */
1899 			op->val = -regs->gpr[ra];
1900 			goto arith_done;
1901 
1902 		case 136:	/* subfe */
1903 			add_with_carry(regs, op, rd, ~regs->gpr[ra],
1904 				       regs->gpr[rb], regs->xer & XER_CA);
1905 			goto arith_done;
1906 
1907 		case 138:	/* adde */
1908 			add_with_carry(regs, op, rd, regs->gpr[ra],
1909 				       regs->gpr[rb], regs->xer & XER_CA);
1910 			goto arith_done;
1911 
1912 		case 200:	/* subfze */
1913 			add_with_carry(regs, op, rd, ~regs->gpr[ra], 0L,
1914 				       regs->xer & XER_CA);
1915 			goto arith_done;
1916 
1917 		case 202:	/* addze */
1918 			add_with_carry(regs, op, rd, regs->gpr[ra], 0L,
1919 				       regs->xer & XER_CA);
1920 			goto arith_done;
1921 
1922 		case 232:	/* subfme */
1923 			add_with_carry(regs, op, rd, ~regs->gpr[ra], -1L,
1924 				       regs->xer & XER_CA);
1925 			goto arith_done;
1926 #ifdef CONFIG_PPC64
1927 		case 233:	/* mulld */
1928 			op->val = regs->gpr[ra] * regs->gpr[rb];
1929 			goto arith_done;
1930 #endif
1931 		case 234:	/* addme */
1932 			add_with_carry(regs, op, rd, regs->gpr[ra], -1L,
1933 				       regs->xer & XER_CA);
1934 			goto arith_done;
1935 
1936 		case 235:	/* mullw */
1937 			op->val = (long)(int) regs->gpr[ra] *
1938 				(int) regs->gpr[rb];
1939 
1940 			goto arith_done;
1941 #ifdef CONFIG_PPC64
1942 		case 265:	/* modud */
1943 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
1944 				goto unknown_opcode;
1945 			op->val = regs->gpr[ra] % regs->gpr[rb];
1946 			goto compute_done;
1947 #endif
1948 		case 266:	/* add */
1949 			op->val = regs->gpr[ra] + regs->gpr[rb];
1950 			goto arith_done;
1951 
1952 		case 267:	/* moduw */
1953 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
1954 				goto unknown_opcode;
1955 			op->val = (unsigned int) regs->gpr[ra] %
1956 				(unsigned int) regs->gpr[rb];
1957 			goto compute_done;
1958 #ifdef CONFIG_PPC64
1959 		case 457:	/* divdu */
1960 			op->val = regs->gpr[ra] / regs->gpr[rb];
1961 			goto arith_done;
1962 #endif
1963 		case 459:	/* divwu */
1964 			op->val = (unsigned int) regs->gpr[ra] /
1965 				(unsigned int) regs->gpr[rb];
1966 			goto arith_done;
1967 #ifdef CONFIG_PPC64
1968 		case 489:	/* divd */
1969 			op->val = (long int) regs->gpr[ra] /
1970 				(long int) regs->gpr[rb];
1971 			goto arith_done;
1972 #endif
1973 		case 491:	/* divw */
1974 			op->val = (int) regs->gpr[ra] /
1975 				(int) regs->gpr[rb];
1976 			goto arith_done;
1977 #ifdef CONFIG_PPC64
1978 		case 425:	/* divde[.] */
1979 			asm volatile(PPC_DIVDE(%0, %1, %2) :
1980 				"=r" (op->val) : "r" (regs->gpr[ra]),
1981 				"r" (regs->gpr[rb]));
1982 			goto arith_done;
1983 		case 393:	/* divdeu[.] */
1984 			asm volatile(PPC_DIVDEU(%0, %1, %2) :
1985 				"=r" (op->val) : "r" (regs->gpr[ra]),
1986 				"r" (regs->gpr[rb]));
1987 			goto arith_done;
1988 #endif
1989 		case 755:	/* darn */
1990 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
1991 				goto unknown_opcode;
1992 			switch (ra & 0x3) {
1993 			case 0:
1994 				/* 32-bit conditioned */
1995 				asm volatile(PPC_DARN(%0, 0) : "=r" (op->val));
1996 				goto compute_done;
1997 
1998 			case 1:
1999 				/* 64-bit conditioned */
2000 				asm volatile(PPC_DARN(%0, 1) : "=r" (op->val));
2001 				goto compute_done;
2002 
2003 			case 2:
2004 				/* 64-bit raw */
2005 				asm volatile(PPC_DARN(%0, 2) : "=r" (op->val));
2006 				goto compute_done;
2007 			}
2008 
2009 			goto unknown_opcode;
2010 #ifdef CONFIG_PPC64
2011 		case 777:	/* modsd */
2012 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2013 				goto unknown_opcode;
2014 			op->val = (long int) regs->gpr[ra] %
2015 				(long int) regs->gpr[rb];
2016 			goto compute_done;
2017 #endif
2018 		case 779:	/* modsw */
2019 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2020 				goto unknown_opcode;
2021 			op->val = (int) regs->gpr[ra] %
2022 				(int) regs->gpr[rb];
2023 			goto compute_done;
2024 
2025 
2026 /*
2027  * Logical instructions
2028  */
2029 		case 26:	/* cntlzw */
2030 			val = (unsigned int) regs->gpr[rd];
2031 			op->val = ( val ? __builtin_clz(val) : 32 );
2032 			goto logical_done;
2033 #ifdef CONFIG_PPC64
2034 		case 58:	/* cntlzd */
2035 			val = regs->gpr[rd];
2036 			op->val = ( val ? __builtin_clzl(val) : 64 );
2037 			goto logical_done;
2038 #endif
2039 		case 28:	/* and */
2040 			op->val = regs->gpr[rd] & regs->gpr[rb];
2041 			goto logical_done;
2042 
2043 		case 60:	/* andc */
2044 			op->val = regs->gpr[rd] & ~regs->gpr[rb];
2045 			goto logical_done;
2046 
2047 		case 122:	/* popcntb */
2048 			do_popcnt(regs, op, regs->gpr[rd], 8);
2049 			goto logical_done_nocc;
2050 
2051 		case 124:	/* nor */
2052 			op->val = ~(regs->gpr[rd] | regs->gpr[rb]);
2053 			goto logical_done;
2054 
2055 		case 154:	/* prtyw */
2056 			do_prty(regs, op, regs->gpr[rd], 32);
2057 			goto logical_done_nocc;
2058 
2059 		case 186:	/* prtyd */
2060 			do_prty(regs, op, regs->gpr[rd], 64);
2061 			goto logical_done_nocc;
2062 #ifdef CONFIG_PPC64
2063 		case 252:	/* bpermd */
2064 			do_bpermd(regs, op, regs->gpr[rd], regs->gpr[rb]);
2065 			goto logical_done_nocc;
2066 #endif
2067 		case 284:	/* xor */
2068 			op->val = ~(regs->gpr[rd] ^ regs->gpr[rb]);
2069 			goto logical_done;
2070 
2071 		case 316:	/* xor */
2072 			op->val = regs->gpr[rd] ^ regs->gpr[rb];
2073 			goto logical_done;
2074 
2075 		case 378:	/* popcntw */
2076 			do_popcnt(regs, op, regs->gpr[rd], 32);
2077 			goto logical_done_nocc;
2078 
2079 		case 412:	/* orc */
2080 			op->val = regs->gpr[rd] | ~regs->gpr[rb];
2081 			goto logical_done;
2082 
2083 		case 444:	/* or */
2084 			op->val = regs->gpr[rd] | regs->gpr[rb];
2085 			goto logical_done;
2086 
2087 		case 476:	/* nand */
2088 			op->val = ~(regs->gpr[rd] & regs->gpr[rb]);
2089 			goto logical_done;
2090 #ifdef CONFIG_PPC64
2091 		case 506:	/* popcntd */
2092 			do_popcnt(regs, op, regs->gpr[rd], 64);
2093 			goto logical_done_nocc;
2094 #endif
2095 		case 538:	/* cnttzw */
2096 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2097 				goto unknown_opcode;
2098 			val = (unsigned int) regs->gpr[rd];
2099 			op->val = (val ? __builtin_ctz(val) : 32);
2100 			goto logical_done;
2101 #ifdef CONFIG_PPC64
2102 		case 570:	/* cnttzd */
2103 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2104 				goto unknown_opcode;
2105 			val = regs->gpr[rd];
2106 			op->val = (val ? __builtin_ctzl(val) : 64);
2107 			goto logical_done;
2108 #endif
2109 		case 922:	/* extsh */
2110 			op->val = (signed short) regs->gpr[rd];
2111 			goto logical_done;
2112 
2113 		case 954:	/* extsb */
2114 			op->val = (signed char) regs->gpr[rd];
2115 			goto logical_done;
2116 #ifdef CONFIG_PPC64
2117 		case 986:	/* extsw */
2118 			op->val = (signed int) regs->gpr[rd];
2119 			goto logical_done;
2120 #endif
2121 
2122 /*
2123  * Shift instructions
2124  */
2125 		case 24:	/* slw */
2126 			sh = regs->gpr[rb] & 0x3f;
2127 			if (sh < 32)
2128 				op->val = (regs->gpr[rd] << sh) & 0xffffffffUL;
2129 			else
2130 				op->val = 0;
2131 			goto logical_done;
2132 
2133 		case 536:	/* srw */
2134 			sh = regs->gpr[rb] & 0x3f;
2135 			if (sh < 32)
2136 				op->val = (regs->gpr[rd] & 0xffffffffUL) >> sh;
2137 			else
2138 				op->val = 0;
2139 			goto logical_done;
2140 
2141 		case 792:	/* sraw */
2142 			op->type = COMPUTE + SETREG + SETXER;
2143 			sh = regs->gpr[rb] & 0x3f;
2144 			ival = (signed int) regs->gpr[rd];
2145 			op->val = ival >> (sh < 32 ? sh : 31);
2146 			op->xerval = regs->xer;
2147 			if (ival < 0 && (sh >= 32 || (ival & ((1ul << sh) - 1)) != 0))
2148 				op->xerval |= XER_CA;
2149 			else
2150 				op->xerval &= ~XER_CA;
2151 			set_ca32(op, op->xerval & XER_CA);
2152 			goto logical_done;
2153 
2154 		case 824:	/* srawi */
2155 			op->type = COMPUTE + SETREG + SETXER;
2156 			sh = rb;
2157 			ival = (signed int) regs->gpr[rd];
2158 			op->val = ival >> sh;
2159 			op->xerval = regs->xer;
2160 			if (ival < 0 && (ival & ((1ul << sh) - 1)) != 0)
2161 				op->xerval |= XER_CA;
2162 			else
2163 				op->xerval &= ~XER_CA;
2164 			set_ca32(op, op->xerval & XER_CA);
2165 			goto logical_done;
2166 
2167 #ifdef CONFIG_PPC64
2168 		case 27:	/* sld */
2169 			sh = regs->gpr[rb] & 0x7f;
2170 			if (sh < 64)
2171 				op->val = regs->gpr[rd] << sh;
2172 			else
2173 				op->val = 0;
2174 			goto logical_done;
2175 
2176 		case 539:	/* srd */
2177 			sh = regs->gpr[rb] & 0x7f;
2178 			if (sh < 64)
2179 				op->val = regs->gpr[rd] >> sh;
2180 			else
2181 				op->val = 0;
2182 			goto logical_done;
2183 
2184 		case 794:	/* srad */
2185 			op->type = COMPUTE + SETREG + SETXER;
2186 			sh = regs->gpr[rb] & 0x7f;
2187 			ival = (signed long int) regs->gpr[rd];
2188 			op->val = ival >> (sh < 64 ? sh : 63);
2189 			op->xerval = regs->xer;
2190 			if (ival < 0 && (sh >= 64 || (ival & ((1ul << sh) - 1)) != 0))
2191 				op->xerval |= XER_CA;
2192 			else
2193 				op->xerval &= ~XER_CA;
2194 			set_ca32(op, op->xerval & XER_CA);
2195 			goto logical_done;
2196 
2197 		case 826:	/* sradi with sh_5 = 0 */
2198 		case 827:	/* sradi with sh_5 = 1 */
2199 			op->type = COMPUTE + SETREG + SETXER;
2200 			sh = rb | ((word & 2) << 4);
2201 			ival = (signed long int) regs->gpr[rd];
2202 			op->val = ival >> sh;
2203 			op->xerval = regs->xer;
2204 			if (ival < 0 && (ival & ((1ul << sh) - 1)) != 0)
2205 				op->xerval |= XER_CA;
2206 			else
2207 				op->xerval &= ~XER_CA;
2208 			set_ca32(op, op->xerval & XER_CA);
2209 			goto logical_done;
2210 
2211 		case 890:	/* extswsli with sh_5 = 0 */
2212 		case 891:	/* extswsli with sh_5 = 1 */
2213 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2214 				goto unknown_opcode;
2215 			op->type = COMPUTE + SETREG;
2216 			sh = rb | ((word & 2) << 4);
2217 			val = (signed int) regs->gpr[rd];
2218 			if (sh)
2219 				op->val = ROTATE(val, sh) & MASK64(0, 63 - sh);
2220 			else
2221 				op->val = val;
2222 			goto logical_done;
2223 
2224 #endif /* CONFIG_PPC64 */
2225 
2226 /*
2227  * Cache instructions
2228  */
2229 		case 54:	/* dcbst */
2230 			op->type = MKOP(CACHEOP, DCBST, 0);
2231 			op->ea = xform_ea(word, regs);
2232 			return 0;
2233 
2234 		case 86:	/* dcbf */
2235 			op->type = MKOP(CACHEOP, DCBF, 0);
2236 			op->ea = xform_ea(word, regs);
2237 			return 0;
2238 
2239 		case 246:	/* dcbtst */
2240 			op->type = MKOP(CACHEOP, DCBTST, 0);
2241 			op->ea = xform_ea(word, regs);
2242 			op->reg = rd;
2243 			return 0;
2244 
2245 		case 278:	/* dcbt */
2246 			op->type = MKOP(CACHEOP, DCBTST, 0);
2247 			op->ea = xform_ea(word, regs);
2248 			op->reg = rd;
2249 			return 0;
2250 
2251 		case 982:	/* icbi */
2252 			op->type = MKOP(CACHEOP, ICBI, 0);
2253 			op->ea = xform_ea(word, regs);
2254 			return 0;
2255 
2256 		case 1014:	/* dcbz */
2257 			op->type = MKOP(CACHEOP, DCBZ, 0);
2258 			op->ea = xform_ea(word, regs);
2259 			return 0;
2260 		}
2261 		break;
2262 	}
2263 
2264 /*
2265  * Loads and stores.
2266  */
2267 	op->type = UNKNOWN;
2268 	op->update_reg = ra;
2269 	op->reg = rd;
2270 	op->val = regs->gpr[rd];
2271 	u = (word >> 20) & UPDATE;
2272 	op->vsx_flags = 0;
2273 
2274 	switch (opcode) {
2275 	case 31:
2276 		u = word & UPDATE;
2277 		op->ea = xform_ea(word, regs);
2278 		switch ((word >> 1) & 0x3ff) {
2279 		case 20:	/* lwarx */
2280 			op->type = MKOP(LARX, 0, 4);
2281 			break;
2282 
2283 		case 150:	/* stwcx. */
2284 			op->type = MKOP(STCX, 0, 4);
2285 			break;
2286 
2287 #ifdef CONFIG_PPC_HAS_LBARX_LHARX
2288 		case 52:	/* lbarx */
2289 			op->type = MKOP(LARX, 0, 1);
2290 			break;
2291 
2292 		case 694:	/* stbcx. */
2293 			op->type = MKOP(STCX, 0, 1);
2294 			break;
2295 
2296 		case 116:	/* lharx */
2297 			op->type = MKOP(LARX, 0, 2);
2298 			break;
2299 
2300 		case 726:	/* sthcx. */
2301 			op->type = MKOP(STCX, 0, 2);
2302 			break;
2303 #endif
2304 #ifdef CONFIG_PPC64
2305 		case 84:	/* ldarx */
2306 			op->type = MKOP(LARX, 0, 8);
2307 			break;
2308 
2309 		case 214:	/* stdcx. */
2310 			op->type = MKOP(STCX, 0, 8);
2311 			break;
2312 
2313 		case 276:	/* lqarx */
2314 			if (!((rd & 1) || rd == ra || rd == rb))
2315 				op->type = MKOP(LARX, 0, 16);
2316 			break;
2317 
2318 		case 182:	/* stqcx. */
2319 			if (!(rd & 1))
2320 				op->type = MKOP(STCX, 0, 16);
2321 			break;
2322 #endif
2323 
2324 		case 23:	/* lwzx */
2325 		case 55:	/* lwzux */
2326 			op->type = MKOP(LOAD, u, 4);
2327 			break;
2328 
2329 		case 87:	/* lbzx */
2330 		case 119:	/* lbzux */
2331 			op->type = MKOP(LOAD, u, 1);
2332 			break;
2333 
2334 #ifdef CONFIG_ALTIVEC
2335 		/*
2336 		 * Note: for the load/store vector element instructions,
2337 		 * bits of the EA say which field of the VMX register to use.
2338 		 */
2339 		case 7:		/* lvebx */
2340 			op->type = MKOP(LOAD_VMX, 0, 1);
2341 			op->element_size = 1;
2342 			break;
2343 
2344 		case 39:	/* lvehx */
2345 			op->type = MKOP(LOAD_VMX, 0, 2);
2346 			op->element_size = 2;
2347 			break;
2348 
2349 		case 71:	/* lvewx */
2350 			op->type = MKOP(LOAD_VMX, 0, 4);
2351 			op->element_size = 4;
2352 			break;
2353 
2354 		case 103:	/* lvx */
2355 		case 359:	/* lvxl */
2356 			op->type = MKOP(LOAD_VMX, 0, 16);
2357 			op->element_size = 16;
2358 			break;
2359 
2360 		case 135:	/* stvebx */
2361 			op->type = MKOP(STORE_VMX, 0, 1);
2362 			op->element_size = 1;
2363 			break;
2364 
2365 		case 167:	/* stvehx */
2366 			op->type = MKOP(STORE_VMX, 0, 2);
2367 			op->element_size = 2;
2368 			break;
2369 
2370 		case 199:	/* stvewx */
2371 			op->type = MKOP(STORE_VMX, 0, 4);
2372 			op->element_size = 4;
2373 			break;
2374 
2375 		case 231:	/* stvx */
2376 		case 487:	/* stvxl */
2377 			op->type = MKOP(STORE_VMX, 0, 16);
2378 			break;
2379 #endif /* CONFIG_ALTIVEC */
2380 
2381 #ifdef CONFIG_PPC64
2382 		case 21:	/* ldx */
2383 		case 53:	/* ldux */
2384 			op->type = MKOP(LOAD, u, 8);
2385 			break;
2386 
2387 		case 149:	/* stdx */
2388 		case 181:	/* stdux */
2389 			op->type = MKOP(STORE, u, 8);
2390 			break;
2391 #endif
2392 
2393 		case 151:	/* stwx */
2394 		case 183:	/* stwux */
2395 			op->type = MKOP(STORE, u, 4);
2396 			break;
2397 
2398 		case 215:	/* stbx */
2399 		case 247:	/* stbux */
2400 			op->type = MKOP(STORE, u, 1);
2401 			break;
2402 
2403 		case 279:	/* lhzx */
2404 		case 311:	/* lhzux */
2405 			op->type = MKOP(LOAD, u, 2);
2406 			break;
2407 
2408 #ifdef CONFIG_PPC64
2409 		case 341:	/* lwax */
2410 		case 373:	/* lwaux */
2411 			op->type = MKOP(LOAD, SIGNEXT | u, 4);
2412 			break;
2413 #endif
2414 
2415 		case 343:	/* lhax */
2416 		case 375:	/* lhaux */
2417 			op->type = MKOP(LOAD, SIGNEXT | u, 2);
2418 			break;
2419 
2420 		case 407:	/* sthx */
2421 		case 439:	/* sthux */
2422 			op->type = MKOP(STORE, u, 2);
2423 			break;
2424 
2425 #ifdef CONFIG_PPC64
2426 		case 532:	/* ldbrx */
2427 			op->type = MKOP(LOAD, BYTEREV, 8);
2428 			break;
2429 
2430 #endif
2431 		case 533:	/* lswx */
2432 			op->type = MKOP(LOAD_MULTI, 0, regs->xer & 0x7f);
2433 			break;
2434 
2435 		case 534:	/* lwbrx */
2436 			op->type = MKOP(LOAD, BYTEREV, 4);
2437 			break;
2438 
2439 		case 597:	/* lswi */
2440 			if (rb == 0)
2441 				rb = 32;	/* # bytes to load */
2442 			op->type = MKOP(LOAD_MULTI, 0, rb);
2443 			op->ea = ra ? regs->gpr[ra] : 0;
2444 			break;
2445 
2446 #ifdef CONFIG_PPC_FPU
2447 		case 535:	/* lfsx */
2448 		case 567:	/* lfsux */
2449 			op->type = MKOP(LOAD_FP, u | FPCONV, 4);
2450 			break;
2451 
2452 		case 599:	/* lfdx */
2453 		case 631:	/* lfdux */
2454 			op->type = MKOP(LOAD_FP, u, 8);
2455 			break;
2456 
2457 		case 663:	/* stfsx */
2458 		case 695:	/* stfsux */
2459 			op->type = MKOP(STORE_FP, u | FPCONV, 4);
2460 			break;
2461 
2462 		case 727:	/* stfdx */
2463 		case 759:	/* stfdux */
2464 			op->type = MKOP(STORE_FP, u, 8);
2465 			break;
2466 
2467 #ifdef CONFIG_PPC64
2468 		case 791:	/* lfdpx */
2469 			op->type = MKOP(LOAD_FP, 0, 16);
2470 			break;
2471 
2472 		case 855:	/* lfiwax */
2473 			op->type = MKOP(LOAD_FP, SIGNEXT, 4);
2474 			break;
2475 
2476 		case 887:	/* lfiwzx */
2477 			op->type = MKOP(LOAD_FP, 0, 4);
2478 			break;
2479 
2480 		case 919:	/* stfdpx */
2481 			op->type = MKOP(STORE_FP, 0, 16);
2482 			break;
2483 
2484 		case 983:	/* stfiwx */
2485 			op->type = MKOP(STORE_FP, 0, 4);
2486 			break;
2487 #endif /* CONFIG_PPC64 */
2488 #endif /* CONFIG_PPC_FPU */
2489 
2490 #ifdef CONFIG_PPC64
2491 		case 660:	/* stdbrx */
2492 			op->type = MKOP(STORE, BYTEREV, 8);
2493 			op->val = byterev_8(regs->gpr[rd]);
2494 			break;
2495 
2496 #endif
2497 		case 661:	/* stswx */
2498 			op->type = MKOP(STORE_MULTI, 0, regs->xer & 0x7f);
2499 			break;
2500 
2501 		case 662:	/* stwbrx */
2502 			op->type = MKOP(STORE, BYTEREV, 4);
2503 			op->val = byterev_4(regs->gpr[rd]);
2504 			break;
2505 
2506 		case 725:	/* stswi */
2507 			if (rb == 0)
2508 				rb = 32;	/* # bytes to store */
2509 			op->type = MKOP(STORE_MULTI, 0, rb);
2510 			op->ea = ra ? regs->gpr[ra] : 0;
2511 			break;
2512 
2513 		case 790:	/* lhbrx */
2514 			op->type = MKOP(LOAD, BYTEREV, 2);
2515 			break;
2516 
2517 		case 918:	/* sthbrx */
2518 			op->type = MKOP(STORE, BYTEREV, 2);
2519 			op->val = byterev_2(regs->gpr[rd]);
2520 			break;
2521 
2522 #ifdef CONFIG_VSX
2523 		case 12:	/* lxsiwzx */
2524 			op->reg = rd | ((word & 1) << 5);
2525 			op->type = MKOP(LOAD_VSX, 0, 4);
2526 			op->element_size = 8;
2527 			break;
2528 
2529 		case 76:	/* lxsiwax */
2530 			op->reg = rd | ((word & 1) << 5);
2531 			op->type = MKOP(LOAD_VSX, SIGNEXT, 4);
2532 			op->element_size = 8;
2533 			break;
2534 
2535 		case 140:	/* stxsiwx */
2536 			op->reg = rd | ((word & 1) << 5);
2537 			op->type = MKOP(STORE_VSX, 0, 4);
2538 			op->element_size = 8;
2539 			break;
2540 
2541 		case 268:	/* lxvx */
2542 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2543 				goto unknown_opcode;
2544 			op->reg = rd | ((word & 1) << 5);
2545 			op->type = MKOP(LOAD_VSX, 0, 16);
2546 			op->element_size = 16;
2547 			op->vsx_flags = VSX_CHECK_VEC;
2548 			break;
2549 
2550 		case 269:	/* lxvl */
2551 		case 301: {	/* lxvll */
2552 			int nb;
2553 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2554 				goto unknown_opcode;
2555 			op->reg = rd | ((word & 1) << 5);
2556 			op->ea = ra ? regs->gpr[ra] : 0;
2557 			nb = regs->gpr[rb] & 0xff;
2558 			if (nb > 16)
2559 				nb = 16;
2560 			op->type = MKOP(LOAD_VSX, 0, nb);
2561 			op->element_size = 16;
2562 			op->vsx_flags = ((word & 0x20) ? VSX_LDLEFT : 0) |
2563 				VSX_CHECK_VEC;
2564 			break;
2565 		}
2566 		case 332:	/* lxvdsx */
2567 			op->reg = rd | ((word & 1) << 5);
2568 			op->type = MKOP(LOAD_VSX, 0, 8);
2569 			op->element_size = 8;
2570 			op->vsx_flags = VSX_SPLAT;
2571 			break;
2572 
2573 		case 333:       /* lxvpx */
2574 			if (!cpu_has_feature(CPU_FTR_ARCH_31))
2575 				goto unknown_opcode;
2576 			op->reg = VSX_REGISTER_XTP(rd);
2577 			op->type = MKOP(LOAD_VSX, 0, 32);
2578 			op->element_size = 32;
2579 			break;
2580 
2581 		case 364:	/* lxvwsx */
2582 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2583 				goto unknown_opcode;
2584 			op->reg = rd | ((word & 1) << 5);
2585 			op->type = MKOP(LOAD_VSX, 0, 4);
2586 			op->element_size = 4;
2587 			op->vsx_flags = VSX_SPLAT | VSX_CHECK_VEC;
2588 			break;
2589 
2590 		case 396:	/* stxvx */
2591 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2592 				goto unknown_opcode;
2593 			op->reg = rd | ((word & 1) << 5);
2594 			op->type = MKOP(STORE_VSX, 0, 16);
2595 			op->element_size = 16;
2596 			op->vsx_flags = VSX_CHECK_VEC;
2597 			break;
2598 
2599 		case 397:	/* stxvl */
2600 		case 429: {	/* stxvll */
2601 			int nb;
2602 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2603 				goto unknown_opcode;
2604 			op->reg = rd | ((word & 1) << 5);
2605 			op->ea = ra ? regs->gpr[ra] : 0;
2606 			nb = regs->gpr[rb] & 0xff;
2607 			if (nb > 16)
2608 				nb = 16;
2609 			op->type = MKOP(STORE_VSX, 0, nb);
2610 			op->element_size = 16;
2611 			op->vsx_flags = ((word & 0x20) ? VSX_LDLEFT : 0) |
2612 				VSX_CHECK_VEC;
2613 			break;
2614 		}
2615 		case 461:       /* stxvpx */
2616 			if (!cpu_has_feature(CPU_FTR_ARCH_31))
2617 				goto unknown_opcode;
2618 			op->reg = VSX_REGISTER_XTP(rd);
2619 			op->type = MKOP(STORE_VSX, 0, 32);
2620 			op->element_size = 32;
2621 			break;
2622 		case 524:	/* lxsspx */
2623 			op->reg = rd | ((word & 1) << 5);
2624 			op->type = MKOP(LOAD_VSX, 0, 4);
2625 			op->element_size = 8;
2626 			op->vsx_flags = VSX_FPCONV;
2627 			break;
2628 
2629 		case 588:	/* lxsdx */
2630 			op->reg = rd | ((word & 1) << 5);
2631 			op->type = MKOP(LOAD_VSX, 0, 8);
2632 			op->element_size = 8;
2633 			break;
2634 
2635 		case 652:	/* stxsspx */
2636 			op->reg = rd | ((word & 1) << 5);
2637 			op->type = MKOP(STORE_VSX, 0, 4);
2638 			op->element_size = 8;
2639 			op->vsx_flags = VSX_FPCONV;
2640 			break;
2641 
2642 		case 716:	/* stxsdx */
2643 			op->reg = rd | ((word & 1) << 5);
2644 			op->type = MKOP(STORE_VSX, 0, 8);
2645 			op->element_size = 8;
2646 			break;
2647 
2648 		case 780:	/* lxvw4x */
2649 			op->reg = rd | ((word & 1) << 5);
2650 			op->type = MKOP(LOAD_VSX, 0, 16);
2651 			op->element_size = 4;
2652 			break;
2653 
2654 		case 781:	/* lxsibzx */
2655 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2656 				goto unknown_opcode;
2657 			op->reg = rd | ((word & 1) << 5);
2658 			op->type = MKOP(LOAD_VSX, 0, 1);
2659 			op->element_size = 8;
2660 			op->vsx_flags = VSX_CHECK_VEC;
2661 			break;
2662 
2663 		case 812:	/* lxvh8x */
2664 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2665 				goto unknown_opcode;
2666 			op->reg = rd | ((word & 1) << 5);
2667 			op->type = MKOP(LOAD_VSX, 0, 16);
2668 			op->element_size = 2;
2669 			op->vsx_flags = VSX_CHECK_VEC;
2670 			break;
2671 
2672 		case 813:	/* lxsihzx */
2673 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2674 				goto unknown_opcode;
2675 			op->reg = rd | ((word & 1) << 5);
2676 			op->type = MKOP(LOAD_VSX, 0, 2);
2677 			op->element_size = 8;
2678 			op->vsx_flags = VSX_CHECK_VEC;
2679 			break;
2680 
2681 		case 844:	/* lxvd2x */
2682 			op->reg = rd | ((word & 1) << 5);
2683 			op->type = MKOP(LOAD_VSX, 0, 16);
2684 			op->element_size = 8;
2685 			break;
2686 
2687 		case 876:	/* lxvb16x */
2688 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2689 				goto unknown_opcode;
2690 			op->reg = rd | ((word & 1) << 5);
2691 			op->type = MKOP(LOAD_VSX, 0, 16);
2692 			op->element_size = 1;
2693 			op->vsx_flags = VSX_CHECK_VEC;
2694 			break;
2695 
2696 		case 908:	/* stxvw4x */
2697 			op->reg = rd | ((word & 1) << 5);
2698 			op->type = MKOP(STORE_VSX, 0, 16);
2699 			op->element_size = 4;
2700 			break;
2701 
2702 		case 909:	/* stxsibx */
2703 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2704 				goto unknown_opcode;
2705 			op->reg = rd | ((word & 1) << 5);
2706 			op->type = MKOP(STORE_VSX, 0, 1);
2707 			op->element_size = 8;
2708 			op->vsx_flags = VSX_CHECK_VEC;
2709 			break;
2710 
2711 		case 940:	/* stxvh8x */
2712 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2713 				goto unknown_opcode;
2714 			op->reg = rd | ((word & 1) << 5);
2715 			op->type = MKOP(STORE_VSX, 0, 16);
2716 			op->element_size = 2;
2717 			op->vsx_flags = VSX_CHECK_VEC;
2718 			break;
2719 
2720 		case 941:	/* stxsihx */
2721 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2722 				goto unknown_opcode;
2723 			op->reg = rd | ((word & 1) << 5);
2724 			op->type = MKOP(STORE_VSX, 0, 2);
2725 			op->element_size = 8;
2726 			op->vsx_flags = VSX_CHECK_VEC;
2727 			break;
2728 
2729 		case 972:	/* stxvd2x */
2730 			op->reg = rd | ((word & 1) << 5);
2731 			op->type = MKOP(STORE_VSX, 0, 16);
2732 			op->element_size = 8;
2733 			break;
2734 
2735 		case 1004:	/* stxvb16x */
2736 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2737 				goto unknown_opcode;
2738 			op->reg = rd | ((word & 1) << 5);
2739 			op->type = MKOP(STORE_VSX, 0, 16);
2740 			op->element_size = 1;
2741 			op->vsx_flags = VSX_CHECK_VEC;
2742 			break;
2743 
2744 #endif /* CONFIG_VSX */
2745 		}
2746 		break;
2747 
2748 	case 32:	/* lwz */
2749 	case 33:	/* lwzu */
2750 		op->type = MKOP(LOAD, u, 4);
2751 		op->ea = dform_ea(word, regs);
2752 		break;
2753 
2754 	case 34:	/* lbz */
2755 	case 35:	/* lbzu */
2756 		op->type = MKOP(LOAD, u, 1);
2757 		op->ea = dform_ea(word, regs);
2758 		break;
2759 
2760 	case 36:	/* stw */
2761 	case 37:	/* stwu */
2762 		op->type = MKOP(STORE, u, 4);
2763 		op->ea = dform_ea(word, regs);
2764 		break;
2765 
2766 	case 38:	/* stb */
2767 	case 39:	/* stbu */
2768 		op->type = MKOP(STORE, u, 1);
2769 		op->ea = dform_ea(word, regs);
2770 		break;
2771 
2772 	case 40:	/* lhz */
2773 	case 41:	/* lhzu */
2774 		op->type = MKOP(LOAD, u, 2);
2775 		op->ea = dform_ea(word, regs);
2776 		break;
2777 
2778 	case 42:	/* lha */
2779 	case 43:	/* lhau */
2780 		op->type = MKOP(LOAD, SIGNEXT | u, 2);
2781 		op->ea = dform_ea(word, regs);
2782 		break;
2783 
2784 	case 44:	/* sth */
2785 	case 45:	/* sthu */
2786 		op->type = MKOP(STORE, u, 2);
2787 		op->ea = dform_ea(word, regs);
2788 		break;
2789 
2790 	case 46:	/* lmw */
2791 		if (ra >= rd)
2792 			break;		/* invalid form, ra in range to load */
2793 		op->type = MKOP(LOAD_MULTI, 0, 4 * (32 - rd));
2794 		op->ea = dform_ea(word, regs);
2795 		break;
2796 
2797 	case 47:	/* stmw */
2798 		op->type = MKOP(STORE_MULTI, 0, 4 * (32 - rd));
2799 		op->ea = dform_ea(word, regs);
2800 		break;
2801 
2802 #ifdef CONFIG_PPC_FPU
2803 	case 48:	/* lfs */
2804 	case 49:	/* lfsu */
2805 		op->type = MKOP(LOAD_FP, u | FPCONV, 4);
2806 		op->ea = dform_ea(word, regs);
2807 		break;
2808 
2809 	case 50:	/* lfd */
2810 	case 51:	/* lfdu */
2811 		op->type = MKOP(LOAD_FP, u, 8);
2812 		op->ea = dform_ea(word, regs);
2813 		break;
2814 
2815 	case 52:	/* stfs */
2816 	case 53:	/* stfsu */
2817 		op->type = MKOP(STORE_FP, u | FPCONV, 4);
2818 		op->ea = dform_ea(word, regs);
2819 		break;
2820 
2821 	case 54:	/* stfd */
2822 	case 55:	/* stfdu */
2823 		op->type = MKOP(STORE_FP, u, 8);
2824 		op->ea = dform_ea(word, regs);
2825 		break;
2826 #endif
2827 
2828 #ifdef CONFIG_PPC64
2829 	case 56:	/* lq */
2830 		if (!((rd & 1) || (rd == ra)))
2831 			op->type = MKOP(LOAD, 0, 16);
2832 		op->ea = dqform_ea(word, regs);
2833 		break;
2834 #endif
2835 
2836 #ifdef CONFIG_VSX
2837 	case 57:	/* lfdp, lxsd, lxssp */
2838 		op->ea = dsform_ea(word, regs);
2839 		switch (word & 3) {
2840 		case 0:		/* lfdp */
2841 			if (rd & 1)
2842 				break;		/* reg must be even */
2843 			op->type = MKOP(LOAD_FP, 0, 16);
2844 			break;
2845 		case 2:		/* lxsd */
2846 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2847 				goto unknown_opcode;
2848 			op->reg = rd + 32;
2849 			op->type = MKOP(LOAD_VSX, 0, 8);
2850 			op->element_size = 8;
2851 			op->vsx_flags = VSX_CHECK_VEC;
2852 			break;
2853 		case 3:		/* lxssp */
2854 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2855 				goto unknown_opcode;
2856 			op->reg = rd + 32;
2857 			op->type = MKOP(LOAD_VSX, 0, 4);
2858 			op->element_size = 8;
2859 			op->vsx_flags = VSX_FPCONV | VSX_CHECK_VEC;
2860 			break;
2861 		}
2862 		break;
2863 #endif /* CONFIG_VSX */
2864 
2865 #ifdef CONFIG_PPC64
2866 	case 58:	/* ld[u], lwa */
2867 		op->ea = dsform_ea(word, regs);
2868 		switch (word & 3) {
2869 		case 0:		/* ld */
2870 			op->type = MKOP(LOAD, 0, 8);
2871 			break;
2872 		case 1:		/* ldu */
2873 			op->type = MKOP(LOAD, UPDATE, 8);
2874 			break;
2875 		case 2:		/* lwa */
2876 			op->type = MKOP(LOAD, SIGNEXT, 4);
2877 			break;
2878 		}
2879 		break;
2880 #endif
2881 
2882 #ifdef CONFIG_VSX
2883 	case 6:
2884 		if (!cpu_has_feature(CPU_FTR_ARCH_31))
2885 			goto unknown_opcode;
2886 		op->ea = dqform_ea(word, regs);
2887 		op->reg = VSX_REGISTER_XTP(rd);
2888 		op->element_size = 32;
2889 		switch (word & 0xf) {
2890 		case 0:         /* lxvp */
2891 			op->type = MKOP(LOAD_VSX, 0, 32);
2892 			break;
2893 		case 1:         /* stxvp */
2894 			op->type = MKOP(STORE_VSX, 0, 32);
2895 			break;
2896 		}
2897 		break;
2898 
2899 	case 61:	/* stfdp, lxv, stxsd, stxssp, stxv */
2900 		switch (word & 7) {
2901 		case 0:		/* stfdp with LSB of DS field = 0 */
2902 		case 4:		/* stfdp with LSB of DS field = 1 */
2903 			op->ea = dsform_ea(word, regs);
2904 			op->type = MKOP(STORE_FP, 0, 16);
2905 			break;
2906 
2907 		case 1:		/* lxv */
2908 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2909 				goto unknown_opcode;
2910 			op->ea = dqform_ea(word, regs);
2911 			if (word & 8)
2912 				op->reg = rd + 32;
2913 			op->type = MKOP(LOAD_VSX, 0, 16);
2914 			op->element_size = 16;
2915 			op->vsx_flags = VSX_CHECK_VEC;
2916 			break;
2917 
2918 		case 2:		/* stxsd with LSB of DS field = 0 */
2919 		case 6:		/* stxsd with LSB of DS field = 1 */
2920 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2921 				goto unknown_opcode;
2922 			op->ea = dsform_ea(word, regs);
2923 			op->reg = rd + 32;
2924 			op->type = MKOP(STORE_VSX, 0, 8);
2925 			op->element_size = 8;
2926 			op->vsx_flags = VSX_CHECK_VEC;
2927 			break;
2928 
2929 		case 3:		/* stxssp with LSB of DS field = 0 */
2930 		case 7:		/* stxssp with LSB of DS field = 1 */
2931 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2932 				goto unknown_opcode;
2933 			op->ea = dsform_ea(word, regs);
2934 			op->reg = rd + 32;
2935 			op->type = MKOP(STORE_VSX, 0, 4);
2936 			op->element_size = 8;
2937 			op->vsx_flags = VSX_FPCONV | VSX_CHECK_VEC;
2938 			break;
2939 
2940 		case 5:		/* stxv */
2941 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
2942 				goto unknown_opcode;
2943 			op->ea = dqform_ea(word, regs);
2944 			if (word & 8)
2945 				op->reg = rd + 32;
2946 			op->type = MKOP(STORE_VSX, 0, 16);
2947 			op->element_size = 16;
2948 			op->vsx_flags = VSX_CHECK_VEC;
2949 			break;
2950 		}
2951 		break;
2952 #endif /* CONFIG_VSX */
2953 
2954 #ifdef CONFIG_PPC64
2955 	case 62:	/* std[u] */
2956 		op->ea = dsform_ea(word, regs);
2957 		switch (word & 3) {
2958 		case 0:		/* std */
2959 			op->type = MKOP(STORE, 0, 8);
2960 			break;
2961 		case 1:		/* stdu */
2962 			op->type = MKOP(STORE, UPDATE, 8);
2963 			break;
2964 		case 2:		/* stq */
2965 			if (!(rd & 1))
2966 				op->type = MKOP(STORE, 0, 16);
2967 			break;
2968 		}
2969 		break;
2970 	case 1: /* Prefixed instructions */
2971 		if (!cpu_has_feature(CPU_FTR_ARCH_31))
2972 			goto unknown_opcode;
2973 
2974 		prefix_r = GET_PREFIX_R(word);
2975 		ra = GET_PREFIX_RA(suffix);
2976 		op->update_reg = ra;
2977 		rd = (suffix >> 21) & 0x1f;
2978 		op->reg = rd;
2979 		op->val = regs->gpr[rd];
2980 
2981 		suffixopcode = get_op(suffix);
2982 		prefixtype = (word >> 24) & 0x3;
2983 		switch (prefixtype) {
2984 		case 0: /* Type 00  Eight-Byte Load/Store */
2985 			if (prefix_r && ra)
2986 				break;
2987 			op->ea = mlsd_8lsd_ea(word, suffix, regs);
2988 			switch (suffixopcode) {
2989 			case 41:	/* plwa */
2990 				op->type = MKOP(LOAD, PREFIXED | SIGNEXT, 4);
2991 				break;
2992 #ifdef CONFIG_VSX
2993 			case 42:        /* plxsd */
2994 				op->reg = rd + 32;
2995 				op->type = MKOP(LOAD_VSX, PREFIXED, 8);
2996 				op->element_size = 8;
2997 				op->vsx_flags = VSX_CHECK_VEC;
2998 				break;
2999 			case 43:	/* plxssp */
3000 				op->reg = rd + 32;
3001 				op->type = MKOP(LOAD_VSX, PREFIXED, 4);
3002 				op->element_size = 8;
3003 				op->vsx_flags = VSX_FPCONV | VSX_CHECK_VEC;
3004 				break;
3005 			case 46:	/* pstxsd */
3006 				op->reg = rd + 32;
3007 				op->type = MKOP(STORE_VSX, PREFIXED, 8);
3008 				op->element_size = 8;
3009 				op->vsx_flags = VSX_CHECK_VEC;
3010 				break;
3011 			case 47:	/* pstxssp */
3012 				op->reg = rd + 32;
3013 				op->type = MKOP(STORE_VSX, PREFIXED, 4);
3014 				op->element_size = 8;
3015 				op->vsx_flags = VSX_FPCONV | VSX_CHECK_VEC;
3016 				break;
3017 			case 51:	/* plxv1 */
3018 				op->reg += 32;
3019 				fallthrough;
3020 			case 50:	/* plxv0 */
3021 				op->type = MKOP(LOAD_VSX, PREFIXED, 16);
3022 				op->element_size = 16;
3023 				op->vsx_flags = VSX_CHECK_VEC;
3024 				break;
3025 			case 55:	/* pstxv1 */
3026 				op->reg = rd + 32;
3027 				fallthrough;
3028 			case 54:	/* pstxv0 */
3029 				op->type = MKOP(STORE_VSX, PREFIXED, 16);
3030 				op->element_size = 16;
3031 				op->vsx_flags = VSX_CHECK_VEC;
3032 				break;
3033 #endif /* CONFIG_VSX */
3034 			case 56:        /* plq */
3035 				op->type = MKOP(LOAD, PREFIXED, 16);
3036 				break;
3037 			case 57:	/* pld */
3038 				op->type = MKOP(LOAD, PREFIXED, 8);
3039 				break;
3040 #ifdef CONFIG_VSX
3041 			case 58:        /* plxvp */
3042 				op->reg = VSX_REGISTER_XTP(rd);
3043 				op->type = MKOP(LOAD_VSX, PREFIXED, 32);
3044 				op->element_size = 32;
3045 				break;
3046 #endif /* CONFIG_VSX */
3047 			case 60:        /* pstq */
3048 				op->type = MKOP(STORE, PREFIXED, 16);
3049 				break;
3050 			case 61:	/* pstd */
3051 				op->type = MKOP(STORE, PREFIXED, 8);
3052 				break;
3053 #ifdef CONFIG_VSX
3054 			case 62:        /* pstxvp */
3055 				op->reg = VSX_REGISTER_XTP(rd);
3056 				op->type = MKOP(STORE_VSX, PREFIXED, 32);
3057 				op->element_size = 32;
3058 				break;
3059 #endif /* CONFIG_VSX */
3060 			}
3061 			break;
3062 		case 1: /* Type 01 Eight-Byte Register-to-Register */
3063 			break;
3064 		case 2: /* Type 10 Modified Load/Store */
3065 			if (prefix_r && ra)
3066 				break;
3067 			op->ea = mlsd_8lsd_ea(word, suffix, regs);
3068 			switch (suffixopcode) {
3069 			case 32:	/* plwz */
3070 				op->type = MKOP(LOAD, PREFIXED, 4);
3071 				break;
3072 			case 34:	/* plbz */
3073 				op->type = MKOP(LOAD, PREFIXED, 1);
3074 				break;
3075 			case 36:	/* pstw */
3076 				op->type = MKOP(STORE, PREFIXED, 4);
3077 				break;
3078 			case 38:	/* pstb */
3079 				op->type = MKOP(STORE, PREFIXED, 1);
3080 				break;
3081 			case 40:	/* plhz */
3082 				op->type = MKOP(LOAD, PREFIXED, 2);
3083 				break;
3084 			case 42:	/* plha */
3085 				op->type = MKOP(LOAD, PREFIXED | SIGNEXT, 2);
3086 				break;
3087 			case 44:	/* psth */
3088 				op->type = MKOP(STORE, PREFIXED, 2);
3089 				break;
3090 			case 48:        /* plfs */
3091 				op->type = MKOP(LOAD_FP, PREFIXED | FPCONV, 4);
3092 				break;
3093 			case 50:        /* plfd */
3094 				op->type = MKOP(LOAD_FP, PREFIXED, 8);
3095 				break;
3096 			case 52:        /* pstfs */
3097 				op->type = MKOP(STORE_FP, PREFIXED | FPCONV, 4);
3098 				break;
3099 			case 54:        /* pstfd */
3100 				op->type = MKOP(STORE_FP, PREFIXED, 8);
3101 				break;
3102 			}
3103 			break;
3104 		case 3: /* Type 11 Modified Register-to-Register */
3105 			break;
3106 		}
3107 #endif /* CONFIG_PPC64 */
3108 
3109 	}
3110 
3111 	if (OP_IS_LOAD_STORE(op->type) && (op->type & UPDATE)) {
3112 		switch (GETTYPE(op->type)) {
3113 		case LOAD:
3114 			if (ra == rd)
3115 				goto unknown_opcode;
3116 			fallthrough;
3117 		case STORE:
3118 		case LOAD_FP:
3119 		case STORE_FP:
3120 			if (ra == 0)
3121 				goto unknown_opcode;
3122 		}
3123 	}
3124 
3125 #ifdef CONFIG_VSX
3126 	if ((GETTYPE(op->type) == LOAD_VSX ||
3127 	     GETTYPE(op->type) == STORE_VSX) &&
3128 	    !cpu_has_feature(CPU_FTR_VSX)) {
3129 		return -1;
3130 	}
3131 #endif /* CONFIG_VSX */
3132 
3133 	return 0;
3134 
3135  unknown_opcode:
3136 	op->type = UNKNOWN;
3137 	return 0;
3138 
3139  logical_done:
3140 	if (word & 1)
3141 		set_cr0(regs, op);
3142  logical_done_nocc:
3143 	op->reg = ra;
3144 	op->type |= SETREG;
3145 	return 1;
3146 
3147  arith_done:
3148 	if (word & 1)
3149 		set_cr0(regs, op);
3150  compute_done:
3151 	op->reg = rd;
3152 	op->type |= SETREG;
3153 	return 1;
3154 
3155  priv:
3156 	op->type = INTERRUPT | 0x700;
3157 	op->val = SRR1_PROGPRIV;
3158 	return 0;
3159 
3160  trap:
3161 	op->type = INTERRUPT | 0x700;
3162 	op->val = SRR1_PROGTRAP;
3163 	return 0;
3164 }
3165 EXPORT_SYMBOL_GPL(analyse_instr);
3166 NOKPROBE_SYMBOL(analyse_instr);
3167 
3168 /*
3169  * For PPC32 we always use stwu with r1 to change the stack pointer.
3170  * So this emulated store may corrupt the exception frame, now we
3171  * have to provide the exception frame trampoline, which is pushed
3172  * below the kprobed function stack. So we only update gpr[1] but
3173  * don't emulate the real store operation. We will do real store
3174  * operation safely in exception return code by checking this flag.
3175  */
3176 static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs)
3177 {
3178 	/*
3179 	 * Check if we already set since that means we'll
3180 	 * lose the previous value.
3181 	 */
3182 	WARN_ON(test_thread_flag(TIF_EMULATE_STACK_STORE));
3183 	set_thread_flag(TIF_EMULATE_STACK_STORE);
3184 	return 0;
3185 }
3186 
3187 static nokprobe_inline void do_signext(unsigned long *valp, int size)
3188 {
3189 	switch (size) {
3190 	case 2:
3191 		*valp = (signed short) *valp;
3192 		break;
3193 	case 4:
3194 		*valp = (signed int) *valp;
3195 		break;
3196 	}
3197 }
3198 
3199 static nokprobe_inline void do_byterev(unsigned long *valp, int size)
3200 {
3201 	switch (size) {
3202 	case 2:
3203 		*valp = byterev_2(*valp);
3204 		break;
3205 	case 4:
3206 		*valp = byterev_4(*valp);
3207 		break;
3208 #ifdef CONFIG_PPC64
3209 	case 8:
3210 		*valp = byterev_8(*valp);
3211 		break;
3212 #endif
3213 	}
3214 }
3215 
3216 /*
3217  * Emulate an instruction that can be executed just by updating
3218  * fields in *regs.
3219  */
3220 void emulate_update_regs(struct pt_regs *regs, struct instruction_op *op)
3221 {
3222 	unsigned long next_pc;
3223 
3224 	next_pc = truncate_if_32bit(regs->msr, regs->nip + GETLENGTH(op->type));
3225 	switch (GETTYPE(op->type)) {
3226 	case COMPUTE:
3227 		if (op->type & SETREG)
3228 			regs->gpr[op->reg] = op->val;
3229 		if (op->type & SETCC)
3230 			regs->ccr = op->ccval;
3231 		if (op->type & SETXER)
3232 			regs->xer = op->xerval;
3233 		break;
3234 
3235 	case BRANCH:
3236 		if (op->type & SETLK)
3237 			regs->link = next_pc;
3238 		if (op->type & BRTAKEN)
3239 			next_pc = op->val;
3240 		if (op->type & DECCTR)
3241 			--regs->ctr;
3242 		break;
3243 
3244 	case BARRIER:
3245 		switch (op->type & BARRIER_MASK) {
3246 		case BARRIER_SYNC:
3247 			mb();
3248 			break;
3249 		case BARRIER_ISYNC:
3250 			isync();
3251 			break;
3252 		case BARRIER_EIEIO:
3253 			eieio();
3254 			break;
3255 #ifdef CONFIG_PPC64
3256 		case BARRIER_LWSYNC:
3257 			asm volatile("lwsync" : : : "memory");
3258 			break;
3259 		case BARRIER_PTESYNC:
3260 			asm volatile("ptesync" : : : "memory");
3261 			break;
3262 #endif
3263 		}
3264 		break;
3265 
3266 	case MFSPR:
3267 		switch (op->spr) {
3268 		case SPRN_XER:
3269 			regs->gpr[op->reg] = regs->xer & 0xffffffffUL;
3270 			break;
3271 		case SPRN_LR:
3272 			regs->gpr[op->reg] = regs->link;
3273 			break;
3274 		case SPRN_CTR:
3275 			regs->gpr[op->reg] = regs->ctr;
3276 			break;
3277 		default:
3278 			WARN_ON_ONCE(1);
3279 		}
3280 		break;
3281 
3282 	case MTSPR:
3283 		switch (op->spr) {
3284 		case SPRN_XER:
3285 			regs->xer = op->val & 0xffffffffUL;
3286 			break;
3287 		case SPRN_LR:
3288 			regs->link = op->val;
3289 			break;
3290 		case SPRN_CTR:
3291 			regs->ctr = op->val;
3292 			break;
3293 		default:
3294 			WARN_ON_ONCE(1);
3295 		}
3296 		break;
3297 
3298 	default:
3299 		WARN_ON_ONCE(1);
3300 	}
3301 	regs_set_return_ip(regs, next_pc);
3302 }
3303 NOKPROBE_SYMBOL(emulate_update_regs);
3304 
3305 /*
3306  * Emulate a previously-analysed load or store instruction.
3307  * Return values are:
3308  * 0 = instruction emulated successfully
3309  * -EFAULT = address out of range or access faulted (regs->dar
3310  *	     contains the faulting address)
3311  * -EACCES = misaligned access, instruction requires alignment
3312  * -EINVAL = unknown operation in *op
3313  */
3314 int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
3315 {
3316 	int err, size, type;
3317 	int i, rd, nb;
3318 	unsigned int cr;
3319 	unsigned long val;
3320 	unsigned long ea;
3321 	bool cross_endian;
3322 
3323 	err = 0;
3324 	size = GETSIZE(op->type);
3325 	type = GETTYPE(op->type);
3326 	cross_endian = (regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE);
3327 	ea = truncate_if_32bit(regs->msr, op->ea);
3328 
3329 	switch (type) {
3330 	case LARX:
3331 		if (ea & (size - 1))
3332 			return -EACCES;		/* can't handle misaligned */
3333 		if (!address_ok(regs, ea, size))
3334 			return -EFAULT;
3335 		err = 0;
3336 		val = 0;
3337 		switch (size) {
3338 #ifdef CONFIG_PPC_HAS_LBARX_LHARX
3339 		case 1:
3340 			__get_user_asmx(val, ea, err, "lbarx");
3341 			break;
3342 		case 2:
3343 			__get_user_asmx(val, ea, err, "lharx");
3344 			break;
3345 #endif
3346 		case 4:
3347 			__get_user_asmx(val, ea, err, "lwarx");
3348 			break;
3349 #ifdef CONFIG_PPC64
3350 		case 8:
3351 			__get_user_asmx(val, ea, err, "ldarx");
3352 			break;
3353 		case 16:
3354 			err = do_lqarx(ea, &regs->gpr[op->reg]);
3355 			break;
3356 #endif
3357 		default:
3358 			return -EINVAL;
3359 		}
3360 		if (err) {
3361 			regs->dar = ea;
3362 			break;
3363 		}
3364 		if (size < 16)
3365 			regs->gpr[op->reg] = val;
3366 		break;
3367 
3368 	case STCX:
3369 		if (ea & (size - 1))
3370 			return -EACCES;		/* can't handle misaligned */
3371 		if (!address_ok(regs, ea, size))
3372 			return -EFAULT;
3373 		err = 0;
3374 		switch (size) {
3375 #ifdef CONFIG_PPC64
3376 		case 1:
3377 			__put_user_asmx(op->val, ea, err, "stbcx.", cr);
3378 			break;
3379 		case 2:
3380 			__put_user_asmx(op->val, ea, err, "sthcx.", cr);
3381 			break;
3382 #endif
3383 		case 4:
3384 			__put_user_asmx(op->val, ea, err, "stwcx.", cr);
3385 			break;
3386 #ifdef CONFIG_PPC64
3387 		case 8:
3388 			__put_user_asmx(op->val, ea, err, "stdcx.", cr);
3389 			break;
3390 		case 16:
3391 			err = do_stqcx(ea, regs->gpr[op->reg],
3392 				       regs->gpr[op->reg + 1], &cr);
3393 			break;
3394 #endif
3395 		default:
3396 			return -EINVAL;
3397 		}
3398 		if (!err)
3399 			regs->ccr = (regs->ccr & 0x0fffffff) |
3400 				(cr & 0xe0000000) |
3401 				((regs->xer >> 3) & 0x10000000);
3402 		else
3403 			regs->dar = ea;
3404 		break;
3405 
3406 	case LOAD:
3407 #ifdef CONFIG_PPC64
3408 		if (size == 16) {
3409 			err = emulate_lq(regs, ea, op->reg, cross_endian);
3410 			break;
3411 		}
3412 #endif
3413 		err = read_mem(&regs->gpr[op->reg], ea, size, regs);
3414 		if (!err) {
3415 			if (op->type & SIGNEXT)
3416 				do_signext(&regs->gpr[op->reg], size);
3417 			if ((op->type & BYTEREV) == (cross_endian ? 0 : BYTEREV))
3418 				do_byterev(&regs->gpr[op->reg], size);
3419 		}
3420 		break;
3421 
3422 #ifdef CONFIG_PPC_FPU
3423 	case LOAD_FP:
3424 		/*
3425 		 * If the instruction is in userspace, we can emulate it even
3426 		 * if the VMX state is not live, because we have the state
3427 		 * stored in the thread_struct.  If the instruction is in
3428 		 * the kernel, we must not touch the state in the thread_struct.
3429 		 */
3430 		if (!user_mode(regs) && !(regs->msr & MSR_FP))
3431 			return 0;
3432 		err = do_fp_load(op, ea, regs, cross_endian);
3433 		break;
3434 #endif
3435 #ifdef CONFIG_ALTIVEC
3436 	case LOAD_VMX:
3437 		if (!user_mode(regs) && !(regs->msr & MSR_VEC))
3438 			return 0;
3439 		err = do_vec_load(op->reg, ea, size, regs, cross_endian);
3440 		break;
3441 #endif
3442 #ifdef CONFIG_VSX
3443 	case LOAD_VSX: {
3444 		unsigned long msrbit = MSR_VSX;
3445 
3446 		/*
3447 		 * Some VSX instructions check the MSR_VEC bit rather than MSR_VSX
3448 		 * when the target of the instruction is a vector register.
3449 		 */
3450 		if (op->reg >= 32 && (op->vsx_flags & VSX_CHECK_VEC))
3451 			msrbit = MSR_VEC;
3452 		if (!user_mode(regs) && !(regs->msr & msrbit))
3453 			return 0;
3454 		err = do_vsx_load(op, ea, regs, cross_endian);
3455 		break;
3456 	}
3457 #endif
3458 	case LOAD_MULTI:
3459 		if (!address_ok(regs, ea, size))
3460 			return -EFAULT;
3461 		rd = op->reg;
3462 		for (i = 0; i < size; i += 4) {
3463 			unsigned int v32 = 0;
3464 
3465 			nb = size - i;
3466 			if (nb > 4)
3467 				nb = 4;
3468 			err = copy_mem_in((u8 *) &v32, ea, nb, regs);
3469 			if (err)
3470 				break;
3471 			if (unlikely(cross_endian))
3472 				v32 = byterev_4(v32);
3473 			regs->gpr[rd] = v32;
3474 			ea += 4;
3475 			/* reg number wraps from 31 to 0 for lsw[ix] */
3476 			rd = (rd + 1) & 0x1f;
3477 		}
3478 		break;
3479 
3480 	case STORE:
3481 #ifdef CONFIG_PPC64
3482 		if (size == 16) {
3483 			err = emulate_stq(regs, ea, op->reg, cross_endian);
3484 			break;
3485 		}
3486 #endif
3487 		if ((op->type & UPDATE) && size == sizeof(long) &&
3488 		    op->reg == 1 && op->update_reg == 1 && !user_mode(regs) &&
3489 		    ea >= regs->gpr[1] - STACK_INT_FRAME_SIZE) {
3490 			err = handle_stack_update(ea, regs);
3491 			break;
3492 		}
3493 		if (unlikely(cross_endian))
3494 			do_byterev(&op->val, size);
3495 		err = write_mem(op->val, ea, size, regs);
3496 		break;
3497 
3498 #ifdef CONFIG_PPC_FPU
3499 	case STORE_FP:
3500 		if (!user_mode(regs) && !(regs->msr & MSR_FP))
3501 			return 0;
3502 		err = do_fp_store(op, ea, regs, cross_endian);
3503 		break;
3504 #endif
3505 #ifdef CONFIG_ALTIVEC
3506 	case STORE_VMX:
3507 		if (!user_mode(regs) && !(regs->msr & MSR_VEC))
3508 			return 0;
3509 		err = do_vec_store(op->reg, ea, size, regs, cross_endian);
3510 		break;
3511 #endif
3512 #ifdef CONFIG_VSX
3513 	case STORE_VSX: {
3514 		unsigned long msrbit = MSR_VSX;
3515 
3516 		/*
3517 		 * Some VSX instructions check the MSR_VEC bit rather than MSR_VSX
3518 		 * when the target of the instruction is a vector register.
3519 		 */
3520 		if (op->reg >= 32 && (op->vsx_flags & VSX_CHECK_VEC))
3521 			msrbit = MSR_VEC;
3522 		if (!user_mode(regs) && !(regs->msr & msrbit))
3523 			return 0;
3524 		err = do_vsx_store(op, ea, regs, cross_endian);
3525 		break;
3526 	}
3527 #endif
3528 	case STORE_MULTI:
3529 		if (!address_ok(regs, ea, size))
3530 			return -EFAULT;
3531 		rd = op->reg;
3532 		for (i = 0; i < size; i += 4) {
3533 			unsigned int v32 = regs->gpr[rd];
3534 
3535 			nb = size - i;
3536 			if (nb > 4)
3537 				nb = 4;
3538 			if (unlikely(cross_endian))
3539 				v32 = byterev_4(v32);
3540 			err = copy_mem_out((u8 *) &v32, ea, nb, regs);
3541 			if (err)
3542 				break;
3543 			ea += 4;
3544 			/* reg number wraps from 31 to 0 for stsw[ix] */
3545 			rd = (rd + 1) & 0x1f;
3546 		}
3547 		break;
3548 
3549 	default:
3550 		return -EINVAL;
3551 	}
3552 
3553 	if (err)
3554 		return err;
3555 
3556 	if (op->type & UPDATE)
3557 		regs->gpr[op->update_reg] = op->ea;
3558 
3559 	return 0;
3560 }
3561 NOKPROBE_SYMBOL(emulate_loadstore);
3562 
3563 /*
3564  * Emulate instructions that cause a transfer of control,
3565  * loads and stores, and a few other instructions.
3566  * Returns 1 if the step was emulated, 0 if not,
3567  * or -1 if the instruction is one that should not be stepped,
3568  * such as an rfid, or a mtmsrd that would clear MSR_RI.
3569  */
3570 int emulate_step(struct pt_regs *regs, ppc_inst_t instr)
3571 {
3572 	struct instruction_op op;
3573 	int r, err, type;
3574 	unsigned long val;
3575 	unsigned long ea;
3576 
3577 	r = analyse_instr(&op, regs, instr);
3578 	if (r < 0)
3579 		return r;
3580 	if (r > 0) {
3581 		emulate_update_regs(regs, &op);
3582 		return 1;
3583 	}
3584 
3585 	err = 0;
3586 	type = GETTYPE(op.type);
3587 
3588 	if (OP_IS_LOAD_STORE(type)) {
3589 		err = emulate_loadstore(regs, &op);
3590 		if (err)
3591 			return 0;
3592 		goto instr_done;
3593 	}
3594 
3595 	switch (type) {
3596 	case CACHEOP:
3597 		ea = truncate_if_32bit(regs->msr, op.ea);
3598 		if (!address_ok(regs, ea, 8))
3599 			return 0;
3600 		switch (op.type & CACHEOP_MASK) {
3601 		case DCBST:
3602 			__cacheop_user_asmx(ea, err, "dcbst");
3603 			break;
3604 		case DCBF:
3605 			__cacheop_user_asmx(ea, err, "dcbf");
3606 			break;
3607 		case DCBTST:
3608 			if (op.reg == 0)
3609 				prefetchw((void *) ea);
3610 			break;
3611 		case DCBT:
3612 			if (op.reg == 0)
3613 				prefetch((void *) ea);
3614 			break;
3615 		case ICBI:
3616 			__cacheop_user_asmx(ea, err, "icbi");
3617 			break;
3618 		case DCBZ:
3619 			err = emulate_dcbz(ea, regs);
3620 			break;
3621 		}
3622 		if (err) {
3623 			regs->dar = ea;
3624 			return 0;
3625 		}
3626 		goto instr_done;
3627 
3628 	case MFMSR:
3629 		regs->gpr[op.reg] = regs->msr & MSR_MASK;
3630 		goto instr_done;
3631 
3632 	case MTMSR:
3633 		val = regs->gpr[op.reg];
3634 		if ((val & MSR_RI) == 0)
3635 			/* can't step mtmsr[d] that would clear MSR_RI */
3636 			return -1;
3637 		/* here op.val is the mask of bits to change */
3638 		regs_set_return_msr(regs, (regs->msr & ~op.val) | (val & op.val));
3639 		goto instr_done;
3640 
3641 	case SYSCALL:	/* sc */
3642 		/*
3643 		 * Per ISA v3.1, section 7.5.15 'Trace Interrupt', we can't
3644 		 * single step a system call instruction:
3645 		 *
3646 		 *   Successful completion for an instruction means that the
3647 		 *   instruction caused no other interrupt. Thus a Trace
3648 		 *   interrupt never occurs for a System Call or System Call
3649 		 *   Vectored instruction, or for a Trap instruction that
3650 		 *   traps.
3651 		 */
3652 		return -1;
3653 	case SYSCALL_VECTORED_0:	/* scv 0 */
3654 		return -1;
3655 	case RFI:
3656 		return -1;
3657 	}
3658 	return 0;
3659 
3660  instr_done:
3661 	regs_set_return_ip(regs,
3662 		truncate_if_32bit(regs->msr, regs->nip + GETLENGTH(op.type)));
3663 	return 1;
3664 }
3665 NOKPROBE_SYMBOL(emulate_step);
3666