1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ASM_ARM_DIV64 3 #define __ASM_ARM_DIV64 4 5 #include <linux/types.h> 6 #include <asm/compiler.h> 7 8 /* 9 * The semantics of __div64_32() are: 10 * 11 * uint32_t __div64_32(uint64_t *n, uint32_t base) 12 * { 13 * uint32_t remainder = *n % base; 14 * *n = *n / base; 15 * return remainder; 16 * } 17 * 18 * In other words, a 64-bit dividend with a 32-bit divisor producing 19 * a 64-bit result and a 32-bit remainder. To accomplish this optimally 20 * we override the generic version in lib/div64.c to call our __do_div64 21 * assembly implementation with completely non standard calling convention 22 * for arguments and results (beware). 23 */ 24 static inline uint32_t __div64_32(uint64_t *n, uint32_t base) 25 { 26 register unsigned int __base asm("r4") = base; 27 register unsigned long long __n asm("r0") = *n; 28 register unsigned long long __res asm("r2"); 29 unsigned int __rem; 30 asm( __asmeq("%0", "r0") 31 __asmeq("%1", "r2") 32 __asmeq("%2", "r4") 33 "bl __do_div64" 34 : "+r" (__n), "=r" (__res) 35 : "r" (__base) 36 : "ip", "lr", "cc"); 37 __rem = __n >> 32; 38 *n = __res; 39 return __rem; 40 } 41 #define __div64_32 __div64_32 42 43 #if !defined(CONFIG_AEABI) 44 45 /* 46 * In OABI configurations, some uses of the do_div function 47 * cause gcc to run out of registers. To work around that, 48 * we can force the use of the out-of-line version for 49 * configurations that build a OABI kernel. 50 */ 51 #define do_div(n, base) __div64_32(&(n), base) 52 53 #else 54 55 #ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE 56 static __always_inline 57 #else 58 static inline 59 #endif 60 uint64_t __arch_xprod_64(uint64_t m, uint64_t n, bool bias) 61 { 62 unsigned long long res; 63 register unsigned int tmp asm("ip") = 0; 64 bool no_ovf = __builtin_constant_p(m) && 65 ((m >> 32) + (m & 0xffffffff) < 0x100000000); 66 67 if (!bias) { 68 asm ( "umull %Q0, %R0, %Q1, %Q2\n\t" 69 "mov %Q0, #0" 70 : "=&r" (res) 71 : "r" (m), "r" (n) 72 : "cc"); 73 } else if (no_ovf) { 74 res = m; 75 asm ( "umlal %Q0, %R0, %Q1, %Q2\n\t" 76 "mov %Q0, #0" 77 : "+&r" (res) 78 : "r" (m), "r" (n) 79 : "cc"); 80 } else { 81 asm ( "umull %Q0, %R0, %Q2, %Q3\n\t" 82 "cmn %Q0, %Q2\n\t" 83 "adcs %R0, %R0, %R2\n\t" 84 "adc %Q0, %1, #0" 85 : "=&r" (res), "+&r" (tmp) 86 : "r" (m), "r" (n) 87 : "cc"); 88 } 89 90 if (no_ovf) { 91 asm ( "umlal %R0, %Q0, %R1, %Q2\n\t" 92 "umlal %R0, %Q0, %Q1, %R2\n\t" 93 "mov %R0, #0\n\t" 94 "umlal %Q0, %R0, %R1, %R2" 95 : "+&r" (res) 96 : "r" (m), "r" (n) 97 : "cc"); 98 } else { 99 asm ( "umlal %R0, %Q0, %R2, %Q3\n\t" 100 "umlal %R0, %1, %Q2, %R3\n\t" 101 "mov %R0, #0\n\t" 102 "adds %Q0, %1, %Q0\n\t" 103 "adc %R0, %R0, #0\n\t" 104 "umlal %Q0, %R0, %R2, %R3" 105 : "+&r" (res), "+&r" (tmp) 106 : "r" (m), "r" (n) 107 : "cc"); 108 } 109 110 return res; 111 } 112 #define __arch_xprod_64 __arch_xprod_64 113 114 #include <asm-generic/div64.h> 115 116 #endif 117 118 #endif 119