1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * String handling functions. 4 * 5 * Copyright IBM Corp. 2012 6 */ 7 8#include <linux/export.h> 9#include <linux/linkage.h> 10#include <asm/nospec-insn.h> 11 12 GEN_BR_THUNK %r14 13 14/* 15 * void *memmove(void *dest, const void *src, size_t n) 16 */ 17SYM_FUNC_START(__memmove) 18 ltgr %r4,%r4 19 lgr %r1,%r2 20 jz .Lmemmove_exit 21 aghi %r4,-1 22 clgr %r2,%r3 23 jnh .Lmemmove_forward 24 la %r5,1(%r4,%r3) 25 clgr %r2,%r5 26 jl .Lmemmove_reverse 27.Lmemmove_forward: 28 srlg %r0,%r4,8 29 ltgr %r0,%r0 30 jz .Lmemmove_forward_remainder 31.Lmemmove_forward_loop: 32 mvc 0(256,%r1),0(%r3) 33 la %r1,256(%r1) 34 la %r3,256(%r3) 35 brctg %r0,.Lmemmove_forward_loop 36.Lmemmove_forward_remainder: 37 larl %r5,.Lmemmove_mvc 38 ex %r4,0(%r5) 39.Lmemmove_exit: 40 BR_EX %r14 41.Lmemmove_reverse: 42 ic %r0,0(%r4,%r3) 43 stc %r0,0(%r4,%r1) 44 brctg %r4,.Lmemmove_reverse 45 ic %r0,0(%r4,%r3) 46 stc %r0,0(%r4,%r1) 47 BR_EX %r14 48.Lmemmove_mvc: 49 mvc 0(1,%r1),0(%r3) 50SYM_FUNC_END(__memmove) 51EXPORT_SYMBOL(__memmove) 52 53SYM_FUNC_ALIAS(memmove, __memmove) 54EXPORT_SYMBOL(memmove) 55 56/* 57 * memset implementation 58 * 59 * This code corresponds to the C construct below. We do distinguish 60 * between clearing (c == 0) and setting a memory array (c != 0) simply 61 * because nearly all memset invocations in the kernel clear memory and 62 * the xc instruction is preferred in such cases. 63 * 64 * void *memset(void *s, int c, size_t n) 65 * { 66 * if (likely(c == 0)) 67 * return __builtin_memset(s, 0, n); 68 * return __builtin_memset(s, c, n); 69 * } 70 */ 71SYM_FUNC_START(__memset) 72 ltgr %r4,%r4 73 jz .Lmemset_exit 74 ltgr %r3,%r3 75 jnz .Lmemset_fill 76 aghi %r4,-1 77 srlg %r3,%r4,8 78 ltgr %r3,%r3 79 lgr %r1,%r2 80 jz .Lmemset_clear_remainder 81.Lmemset_clear_loop: 82 xc 0(256,%r1),0(%r1) 83 la %r1,256(%r1) 84 brctg %r3,.Lmemset_clear_loop 85.Lmemset_clear_remainder: 86 larl %r3,.Lmemset_xc 87 ex %r4,0(%r3) 88.Lmemset_exit: 89 BR_EX %r14 90.Lmemset_fill: 91 cghi %r4,1 92 lgr %r1,%r2 93 je .Lmemset_fill_exit 94 aghi %r4,-2 95 srlg %r5,%r4,8 96 ltgr %r5,%r5 97 jz .Lmemset_fill_remainder 98.Lmemset_fill_loop: 99 stc %r3,0(%r1) 100 mvc 1(255,%r1),0(%r1) 101 la %r1,256(%r1) 102 brctg %r5,.Lmemset_fill_loop 103.Lmemset_fill_remainder: 104 stc %r3,0(%r1) 105 larl %r5,.Lmemset_mvc 106 ex %r4,0(%r5) 107 BR_EX %r14 108.Lmemset_fill_exit: 109 stc %r3,0(%r1) 110 BR_EX %r14 111.Lmemset_xc: 112 xc 0(1,%r1),0(%r1) 113.Lmemset_mvc: 114 mvc 1(1,%r1),0(%r1) 115SYM_FUNC_END(__memset) 116EXPORT_SYMBOL(__memset) 117 118SYM_FUNC_ALIAS(memset, __memset) 119EXPORT_SYMBOL(memset) 120 121/* 122 * memcpy implementation 123 * 124 * void *memcpy(void *dest, const void *src, size_t n) 125 */ 126SYM_FUNC_START(__memcpy) 127 ltgr %r4,%r4 128 jz .Lmemcpy_exit 129 aghi %r4,-1 130 srlg %r5,%r4,8 131 ltgr %r5,%r5 132 lgr %r1,%r2 133 jnz .Lmemcpy_loop 134.Lmemcpy_remainder: 135 larl %r5,.Lmemcpy_mvc 136 ex %r4,0(%r5) 137.Lmemcpy_exit: 138 BR_EX %r14 139.Lmemcpy_loop: 140 mvc 0(256,%r1),0(%r3) 141 la %r1,256(%r1) 142 la %r3,256(%r3) 143 brctg %r5,.Lmemcpy_loop 144 j .Lmemcpy_remainder 145.Lmemcpy_mvc: 146 mvc 0(1,%r1),0(%r3) 147SYM_FUNC_END(__memcpy) 148EXPORT_SYMBOL(__memcpy) 149 150SYM_FUNC_ALIAS(memcpy, __memcpy) 151EXPORT_SYMBOL(memcpy) 152 153/* 154 * __memset16/32/64 155 * 156 * void *__memset16(uint16_t *s, uint16_t v, size_t count) 157 * void *__memset32(uint32_t *s, uint32_t v, size_t count) 158 * void *__memset64(uint64_t *s, uint64_t v, size_t count) 159 */ 160.macro __MEMSET bits,bytes,insn 161SYM_FUNC_START(__memset\bits) 162 ltgr %r4,%r4 163 jz .L__memset_exit\bits 164 cghi %r4,\bytes 165 je .L__memset_store\bits 166 aghi %r4,-(\bytes+1) 167 srlg %r5,%r4,8 168 ltgr %r5,%r5 169 lgr %r1,%r2 170 jz .L__memset_remainder\bits 171.L__memset_loop\bits: 172 \insn %r3,0(%r1) 173 mvc \bytes(256-\bytes,%r1),0(%r1) 174 la %r1,256(%r1) 175 brctg %r5,.L__memset_loop\bits 176.L__memset_remainder\bits: 177 \insn %r3,0(%r1) 178 larl %r5,.L__memset_mvc\bits 179 ex %r4,0(%r5) 180 BR_EX %r14 181.L__memset_store\bits: 182 \insn %r3,0(%r2) 183.L__memset_exit\bits: 184 BR_EX %r14 185.L__memset_mvc\bits: 186 mvc \bytes(1,%r1),0(%r1) 187SYM_FUNC_END(__memset\bits) 188.endm 189 190__MEMSET 16,2,sth 191EXPORT_SYMBOL(__memset16) 192 193__MEMSET 32,4,st 194EXPORT_SYMBOL(__memset32) 195 196__MEMSET 64,8,stg 197EXPORT_SYMBOL(__memset64) 198