1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Optimized string functions 4 * 5 * S390 version 6 * Copyright IBM Corp. 2004 7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 8 */ 9 10 #define IN_ARCH_STRING_C 1 11 #ifndef __NO_FORTIFY 12 # define __NO_FORTIFY 13 #endif 14 15 #include <linux/types.h> 16 #include <linux/string.h> 17 #include <linux/export.h> 18 #include <asm/facility.h> 19 #include <asm/asm.h> 20 21 #define SYMBOL_FUNCTION_ALIAS(alias, name) \ 22 asm(".globl " __stringify(alias) "\n\t" \ 23 ".set " __stringify(alias) "," __stringify(name)) 24 25 #ifdef __HAVE_ARCH_MEMMOVE 26 noinstr void *__memmove(void *dest, const void *src, size_t n) 27 { 28 const char *s = src; 29 char *d = dest; 30 31 if (!n) 32 return dest; 33 if ((d <= s || d >= s + n)) { 34 /* Forward copy */ 35 while (n >= 256) { 36 asm volatile( 37 " mvc 0(256,%[d]),0(%[s])\n" 38 : 39 : [d] "a" (d), [s] "a" (s) 40 : "memory"); 41 d += 256; 42 s += 256; 43 n -= 256; 44 } 45 if (n) { 46 asm volatile( 47 " exrl %[n],0f\n" 48 " j 1f\n" 49 "0: mvc 0(1,%[d]),0(%[s])\n" 50 "1:" 51 : 52 : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1) 53 : "memory"); 54 } 55 return dest; 56 } 57 /* Backward copy */ 58 if (test_facility(61)) { 59 /* Use mvcrl instruction if available */ 60 while (n >= 256) { 61 asm volatile( 62 " lghi %%r0,255\n" 63 " .insn sse,0xe50a00000000,%[d],%[s]\n" 64 : [d] "=Q" (*(d + n - 256)) 65 : [s] "Q" (*(s + n - 256)) 66 : "0", "memory"); 67 n -= 256; 68 } 69 if (n) { 70 asm volatile( 71 " lgr %%r0,%[n]\n" 72 " .insn sse,0xe50a00000000,%[d],%[s]\n" 73 : [d] "=Q" (*d) 74 : [s] "Q" (*s), [n] "d" (n - 1) 75 : "0", "memory"); 76 } 77 } else { 78 while (n--) 79 d[n] = s[n]; 80 } 81 return dest; 82 } 83 SYMBOL_FUNCTION_ALIAS(memmove, __memmove); 84 EXPORT_SYMBOL(__memmove); 85 EXPORT_SYMBOL(memmove); 86 #endif 87 88 #ifdef __HAVE_ARCH_MEMSET 89 noinstr void *__memset(void *s, int c, size_t n) 90 { 91 char *xs = s; 92 93 if (!n) 94 return s; 95 if (!c) { 96 /* Clear memory */ 97 while (n >= 256) { 98 asm volatile( 99 " xc 0(256,%[xs]),0(%[xs])" 100 : 101 : [xs] "a" (xs) 102 : "cc", "memory"); 103 xs += 256; 104 n -= 256; 105 } 106 if (!n) 107 return s; 108 asm volatile( 109 " exrl %[n],0f\n" 110 " j 1f\n" 111 "0: xc 0(1,%[xs]),0(%[xs])\n" 112 "1:" 113 : 114 : [xs] "a" (xs), [n] "a" (n - 1) 115 : "cc", "memory"); 116 } else { 117 /* Fill memory */ 118 while (n >= 256) { 119 *xs = c; 120 asm volatile( 121 " mvc 1(255,%[xs]),0(%[xs])" 122 : 123 : [xs] "a" (xs) 124 : "memory"); 125 xs += 256; 126 n -= 256; 127 } 128 if (!n) 129 return s; 130 *xs = c; 131 if (n == 1) 132 return s; 133 asm volatile( 134 " exrl %[n],0f\n" 135 " j 1f\n" 136 "0: mvc 1(1,%[xs]),0(%[xs])\n" 137 "1:" 138 : 139 : [xs] "a" (xs), [n] "a" (n - 2) 140 : "memory"); 141 } 142 return s; 143 } 144 SYMBOL_FUNCTION_ALIAS(memset, __memset); 145 EXPORT_SYMBOL(__memset); 146 EXPORT_SYMBOL(memset); 147 #endif 148 149 #ifdef __HAVE_ARCH_MEMCPY 150 noinstr void *__memcpy(void *dest, const void *src, size_t n) 151 { 152 void *d = dest; 153 154 if (!n) 155 return d; 156 while (n >= 256) { 157 asm volatile( 158 " mvc 0(256,%[dest]),0(%[src])" 159 : 160 : [dest] "a" (dest), [src] "a" (src) 161 : "memory"); 162 dest += 256; 163 src += 256; 164 n -= 256; 165 } 166 if (!n) 167 return d; 168 asm volatile( 169 " exrl %[n],1f\n" 170 " j 2f\n" 171 "1: mvc 0(1,%[dest]),0(%[src])\n" 172 "2:" 173 : 174 : [dest] "a" (dest), [src] "a" (src), [n] "a" (n - 1) 175 : "memory"); 176 return d; 177 } 178 SYMBOL_FUNCTION_ALIAS(memcpy, __memcpy); 179 EXPORT_SYMBOL(__memcpy); 180 EXPORT_SYMBOL(memcpy); 181 #endif 182 183 #define DEFINE_MEMSET(_bits, _bytes, _type) \ 184 void *__memset##_bits(_type *s, _type v, size_t n) \ 185 { \ 186 _type *xs = s; \ 187 \ 188 if (!n) \ 189 return s; \ 190 while (n >= 256) { \ 191 *xs = v; \ 192 asm volatile( \ 193 " mvc %[_b](256-%[_b],%[xs]),0(%[xs])\n" \ 194 : \ 195 : [xs] "a" (xs), [_b] "i" (_bytes) \ 196 : "memory"); \ 197 xs = (_type *)((char *)xs + 256); \ 198 n -= 256; \ 199 } \ 200 if (!n) \ 201 return s; \ 202 *xs = v; \ 203 if (n == _bytes) \ 204 return s; \ 205 n -= _bytes + 1; \ 206 asm volatile( \ 207 " exrl %[n],1f\n" \ 208 " j 2f\n" \ 209 "1: mvc %[_b](1,%[xs]),0(%[xs])\n" \ 210 "2:" \ 211 : \ 212 : [n] "a" (n), [xs] "a" (xs), [_b] "i" (_bytes) \ 213 : "memory"); \ 214 return s; \ 215 } \ 216 EXPORT_SYMBOL(__memset##_bits) 217 218 #ifdef __HAVE_ARCH_MEMSET16 219 DEFINE_MEMSET(16, 2, uint16_t); 220 #endif 221 222 #ifdef __HAVE_ARCH_MEMSET32 223 DEFINE_MEMSET(32, 4, uint32_t); 224 #endif 225 226 #ifdef __HAVE_ARCH_MEMSET64 227 DEFINE_MEMSET(64, 8, uint64_t); 228 #endif 229 230 /* 231 * Helper functions to find the end of a string 232 */ 233 static inline char *__strend(const char *s) 234 { 235 unsigned long e = 0; 236 237 asm volatile( 238 " lghi 0,0\n" 239 "0: srst %[e],%[s]\n" 240 " jo 0b" 241 : [e] "+&a" (e), [s] "+&a" (s) 242 : 243 : "cc", "memory", "0"); 244 return (char *)e; 245 } 246 247 static inline char *__strnend(const char *s, size_t n) 248 { 249 const char *p = s + n; 250 251 asm volatile( 252 " lghi 0,0\n" 253 "0: srst %[p],%[s]\n" 254 " jo 0b" 255 : [p] "+&d" (p), [s] "+&a" (s) 256 : 257 : "cc", "memory", "0"); 258 return (char *)p; 259 } 260 261 /** 262 * strlen - Find the length of a string 263 * @s: The string to be sized 264 * 265 * returns the length of @s 266 */ 267 #ifdef __HAVE_ARCH_STRLEN 268 size_t strlen(const char *s) 269 { 270 return __strend(s) - s; 271 } 272 EXPORT_SYMBOL(strlen); 273 #endif 274 275 /** 276 * strnlen - Find the length of a length-limited string 277 * @s: The string to be sized 278 * @n: The maximum number of bytes to search 279 * 280 * returns the minimum of the length of @s and @n 281 */ 282 #ifdef __HAVE_ARCH_STRNLEN 283 size_t strnlen(const char *s, size_t n) 284 { 285 return __strnend(s, n) - s; 286 } 287 EXPORT_SYMBOL(strnlen); 288 #endif 289 290 /** 291 * strcat - Append one %NUL-terminated string to another 292 * @dest: The string to be appended to 293 * @src: The string to append to it 294 * 295 * returns a pointer to @dest 296 */ 297 #ifdef __HAVE_ARCH_STRCAT 298 char *strcat(char *dest, const char *src) 299 { 300 unsigned long dummy = 0; 301 char *ret = dest; 302 303 asm volatile( 304 " lghi 0,0\n" 305 "0: srst %[dummy],%[dest]\n" 306 " jo 0b\n" 307 "1: mvst %[dummy],%[src]\n" 308 " jo 1b" 309 : [dummy] "+&a" (dummy), [dest] "+&a" (dest), [src] "+&a" (src) 310 : 311 : "cc", "memory", "0"); 312 return ret; 313 } 314 EXPORT_SYMBOL(strcat); 315 #endif 316 317 /** 318 * strncat - Append a length-limited, %NUL-terminated string to another 319 * @dest: The string to be appended to 320 * @src: The string to append to it 321 * @n: The maximum numbers of bytes to copy 322 * 323 * returns a pointer to @dest 324 */ 325 #ifdef __HAVE_ARCH_STRNCAT 326 char *strncat(char *dest, const char *src, size_t n) 327 { 328 size_t len = __strnend(src, n) - src; 329 char *p = __strend(dest); 330 331 p[len] = '\0'; 332 memcpy(p, src, len); 333 return dest; 334 } 335 EXPORT_SYMBOL(strncat); 336 #endif 337 338 /** 339 * strcmp - Compare two strings 340 * @s1: One string 341 * @s2: Another string 342 * 343 * returns 0 if @s1 and @s2 are equal, 344 * < 0 if @s1 is less than @s2 345 * > 0 if @s1 is greater than @s2 346 */ 347 #ifdef __HAVE_ARCH_STRCMP 348 int strcmp(const char *s1, const char *s2) 349 { 350 int ret = 0; 351 352 asm volatile( 353 " lghi 0,0\n" 354 "0: clst %[s1],%[s2]\n" 355 " jo 0b\n" 356 " je 1f\n" 357 " ic %[ret],0(%[s1])\n" 358 " ic 0,0(%[s2])\n" 359 " sr %[ret],0\n" 360 "1:" 361 : [ret] "+&d" (ret), [s1] "+&a" (s1), [s2] "+&a" (s2) 362 : 363 : "cc", "memory", "0"); 364 return ret; 365 } 366 EXPORT_SYMBOL(strcmp); 367 #endif 368 369 static inline int clcle(const char *s1, unsigned long l1, 370 const char *s2, unsigned long l2) 371 { 372 union register_pair r1 = { .even = (unsigned long)s1, .odd = l1, }; 373 union register_pair r3 = { .even = (unsigned long)s2, .odd = l2, }; 374 int cc; 375 376 asm volatile( 377 "0: clcle %[r1],%[r3],0\n" 378 " jo 0b\n" 379 CC_IPM(cc) 380 : CC_OUT(cc, cc), [r1] "+d" (r1.pair), [r3] "+d" (r3.pair) 381 : 382 : CC_CLOBBER_LIST("memory")); 383 return CC_TRANSFORM(cc); 384 } 385 386 /** 387 * strstr - Find the first substring in a %NUL terminated string 388 * @s1: The string to be searched 389 * @s2: The string to search for 390 */ 391 #ifdef __HAVE_ARCH_STRSTR 392 char *strstr(const char *s1, const char *s2) 393 { 394 int l1, l2; 395 396 l2 = __strend(s2) - s2; 397 if (!l2) 398 return (char *) s1; 399 l1 = __strend(s1) - s1; 400 while (l1-- >= l2) { 401 int cc; 402 403 cc = clcle(s1, l2, s2, l2); 404 if (!cc) 405 return (char *) s1; 406 s1++; 407 } 408 return NULL; 409 } 410 EXPORT_SYMBOL(strstr); 411 #endif 412 413 /** 414 * memchr - Find a character in an area of memory. 415 * @s: The memory area 416 * @c: The byte to search for 417 * @n: The size of the area. 418 * 419 * returns the address of the first occurrence of @c, or %NULL 420 * if @c is not found 421 */ 422 #ifdef __HAVE_ARCH_MEMCHR 423 void *memchr(const void *s, int c, size_t n) 424 { 425 const void *ret = s + n; 426 427 asm volatile( 428 " lgr 0,%[c]\n" 429 "0: srst %[ret],%[s]\n" 430 " jo 0b\n" 431 " jl 1f\n" 432 " la %[ret],0\n" 433 "1:" 434 : [ret] "+&a" (ret), [s] "+&a" (s) 435 : [c] "d" (c) 436 : "cc", "memory", "0"); 437 return (void *) ret; 438 } 439 EXPORT_SYMBOL(memchr); 440 #endif 441 442 /** 443 * memcmp - Compare two areas of memory 444 * @s1: One area of memory 445 * @s2: Another area of memory 446 * @n: The size of the area. 447 */ 448 #ifdef __HAVE_ARCH_MEMCMP 449 int memcmp(const void *s1, const void *s2, size_t n) 450 { 451 int ret; 452 453 ret = clcle(s1, n, s2, n); 454 if (ret) 455 ret = ret == 1 ? -1 : 1; 456 return ret; 457 } 458 EXPORT_SYMBOL(memcmp); 459 #endif 460 461 /** 462 * memscan - Find a character in an area of memory. 463 * @s: The memory area 464 * @c: The byte to search for 465 * @n: The size of the area. 466 * 467 * returns the address of the first occurrence of @c, or 1 byte past 468 * the area if @c is not found 469 */ 470 #ifdef __HAVE_ARCH_MEMSCAN 471 void *memscan(void *s, int c, size_t n) 472 { 473 const void *ret = s + n; 474 475 asm volatile( 476 " lgr 0,%[c]\n" 477 "0: srst %[ret],%[s]\n" 478 " jo 0b" 479 : [ret] "+&a" (ret), [s] "+&a" (s) 480 : [c] "d" (c) 481 : "cc", "memory", "0"); 482 return (void *)ret; 483 } 484 EXPORT_SYMBOL(memscan); 485 #endif 486