1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 8 * Copyright (C) 2007 Maciej W. Rozycki 9 * Copyright (C) 2014, Imagination Technologies Ltd. 10 */ 11 #ifndef _ASM_UACCESS_H 12 #define _ASM_UACCESS_H 13 14 #include <linux/kernel.h> 15 #include <linux/errno.h> 16 #include <linux/thread_info.h> 17 #include <asm/asm-eva.h> 18 19 /* 20 * The fs value determines whether argument validity checking should be 21 * performed or not. If get_fs() == USER_DS, checking is performed, with 22 * get_fs() == KERNEL_DS, checking is bypassed. 23 * 24 * For historical reasons, these macros are grossly misnamed. 25 */ 26 #ifdef CONFIG_32BIT 27 28 #ifdef CONFIG_KVM_GUEST 29 #define __UA_LIMIT 0x40000000UL 30 #else 31 #define __UA_LIMIT 0x80000000UL 32 #endif 33 34 #define __UA_ADDR ".word" 35 #define __UA_LA "la" 36 #define __UA_ADDU "addu" 37 #define __UA_t0 "$8" 38 #define __UA_t1 "$9" 39 40 #endif /* CONFIG_32BIT */ 41 42 #ifdef CONFIG_64BIT 43 44 extern u64 __ua_limit; 45 46 #define __UA_LIMIT __ua_limit 47 48 #define __UA_ADDR ".dword" 49 #define __UA_LA "dla" 50 #define __UA_ADDU "daddu" 51 #define __UA_t0 "$12" 52 #define __UA_t1 "$13" 53 54 #endif /* CONFIG_64BIT */ 55 56 /* 57 * USER_DS is a bitmask that has the bits set that may not be set in a valid 58 * userspace address. Note that we limit 32-bit userspace to 0x7fff8000 but 59 * the arithmetic we're doing only works if the limit is a power of two, so 60 * we use 0x80000000 here on 32-bit kernels. If a process passes an invalid 61 * address in this range it's the process's problem, not ours :-) 62 */ 63 64 #ifdef CONFIG_KVM_GUEST 65 #define KERNEL_DS ((mm_segment_t) { 0x80000000UL }) 66 #define USER_DS ((mm_segment_t) { 0xC0000000UL }) 67 #else 68 #define KERNEL_DS ((mm_segment_t) { 0UL }) 69 #define USER_DS ((mm_segment_t) { __UA_LIMIT }) 70 #endif 71 72 #define VERIFY_READ 0 73 #define VERIFY_WRITE 1 74 75 #define get_ds() (KERNEL_DS) 76 #define get_fs() (current_thread_info()->addr_limit) 77 #define set_fs(x) (current_thread_info()->addr_limit = (x)) 78 79 #define segment_eq(a, b) ((a).seg == (b).seg) 80 81 82 /* 83 * Is a address valid? This does a straighforward calculation rather 84 * than tests. 85 * 86 * Address valid if: 87 * - "addr" doesn't have any high-bits set 88 * - AND "size" doesn't have any high-bits set 89 * - AND "addr+size" doesn't have any high-bits set 90 * - OR we are in kernel mode. 91 * 92 * __ua_size() is a trick to avoid runtime checking of positive constant 93 * sizes; for those we already know at compile time that the size is ok. 94 */ 95 #define __ua_size(size) \ 96 ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size)) 97 98 /* 99 * access_ok: - Checks if a user space pointer is valid 100 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that 101 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe 102 * to write to a block, it is always safe to read from it. 103 * @addr: User space pointer to start of block to check 104 * @size: Size of block to check 105 * 106 * Context: User context only. This function may sleep. 107 * 108 * Checks if a pointer to a block of memory in user space is valid. 109 * 110 * Returns true (nonzero) if the memory block may be valid, false (zero) 111 * if it is definitely invalid. 112 * 113 * Note that, depending on architecture, this function probably just 114 * checks that the pointer is in the user space range - after calling 115 * this function, memory access functions may still return -EFAULT. 116 */ 117 118 #define __access_mask get_fs().seg 119 120 #define __access_ok(addr, size, mask) \ 121 ({ \ 122 unsigned long __addr = (unsigned long) (addr); \ 123 unsigned long __size = size; \ 124 unsigned long __mask = mask; \ 125 unsigned long __ok; \ 126 \ 127 __chk_user_ptr(addr); \ 128 __ok = (signed long)(__mask & (__addr | (__addr + __size) | \ 129 __ua_size(__size))); \ 130 __ok == 0; \ 131 }) 132 133 #define access_ok(type, addr, size) \ 134 likely(__access_ok((addr), (size), __access_mask)) 135 136 /* 137 * put_user: - Write a simple value into user space. 138 * @x: Value to copy to user space. 139 * @ptr: Destination address, in user space. 140 * 141 * Context: User context only. This function may sleep. 142 * 143 * This macro copies a single simple value from kernel space to user 144 * space. It supports simple types like char and int, but not larger 145 * data types like structures or arrays. 146 * 147 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 148 * to the result of dereferencing @ptr. 149 * 150 * Returns zero on success, or -EFAULT on error. 151 */ 152 #define put_user(x,ptr) \ 153 __put_user_check((x), (ptr), sizeof(*(ptr))) 154 155 /* 156 * get_user: - Get a simple variable from user space. 157 * @x: Variable to store result. 158 * @ptr: Source address, in user space. 159 * 160 * Context: User context only. This function may sleep. 161 * 162 * This macro copies a single simple variable from user space to kernel 163 * space. It supports simple types like char and int, but not larger 164 * data types like structures or arrays. 165 * 166 * @ptr must have pointer-to-simple-variable type, and the result of 167 * dereferencing @ptr must be assignable to @x without a cast. 168 * 169 * Returns zero on success, or -EFAULT on error. 170 * On error, the variable @x is set to zero. 171 */ 172 #define get_user(x,ptr) \ 173 __get_user_check((x), (ptr), sizeof(*(ptr))) 174 175 /* 176 * __put_user: - Write a simple value into user space, with less checking. 177 * @x: Value to copy to user space. 178 * @ptr: Destination address, in user space. 179 * 180 * Context: User context only. This function may sleep. 181 * 182 * This macro copies a single simple value from kernel space to user 183 * space. It supports simple types like char and int, but not larger 184 * data types like structures or arrays. 185 * 186 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 187 * to the result of dereferencing @ptr. 188 * 189 * Caller must check the pointer with access_ok() before calling this 190 * function. 191 * 192 * Returns zero on success, or -EFAULT on error. 193 */ 194 #define __put_user(x,ptr) \ 195 __put_user_nocheck((x), (ptr), sizeof(*(ptr))) 196 197 /* 198 * __get_user: - Get a simple variable from user space, with less checking. 199 * @x: Variable to store result. 200 * @ptr: Source address, in user space. 201 * 202 * Context: User context only. This function may sleep. 203 * 204 * This macro copies a single simple variable from user space to kernel 205 * space. It supports simple types like char and int, but not larger 206 * data types like structures or arrays. 207 * 208 * @ptr must have pointer-to-simple-variable type, and the result of 209 * dereferencing @ptr must be assignable to @x without a cast. 210 * 211 * Caller must check the pointer with access_ok() before calling this 212 * function. 213 * 214 * Returns zero on success, or -EFAULT on error. 215 * On error, the variable @x is set to zero. 216 */ 217 #define __get_user(x,ptr) \ 218 __get_user_nocheck((x), (ptr), sizeof(*(ptr))) 219 220 struct __large_struct { unsigned long buf[100]; }; 221 #define __m(x) (*(struct __large_struct __user *)(x)) 222 223 /* 224 * Yuck. We need two variants, one for 64bit operation and one 225 * for 32 bit mode and old iron. 226 */ 227 #ifndef CONFIG_EVA 228 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr) 229 #else 230 /* 231 * Kernel specific functions for EVA. We need to use normal load instructions 232 * to read data from kernel when operating in EVA mode. We use these macros to 233 * avoid redefining __get_user_asm for EVA. 234 */ 235 #undef _loadd 236 #undef _loadw 237 #undef _loadh 238 #undef _loadb 239 #ifdef CONFIG_32BIT 240 #define _loadd _loadw 241 #else 242 #define _loadd(reg, addr) "ld " reg ", " addr 243 #endif 244 #define _loadw(reg, addr) "lw " reg ", " addr 245 #define _loadh(reg, addr) "lh " reg ", " addr 246 #define _loadb(reg, addr) "lb " reg ", " addr 247 248 #define __get_kernel_common(val, size, ptr) \ 249 do { \ 250 switch (size) { \ 251 case 1: __get_data_asm(val, _loadb, ptr); break; \ 252 case 2: __get_data_asm(val, _loadh, ptr); break; \ 253 case 4: __get_data_asm(val, _loadw, ptr); break; \ 254 case 8: __GET_DW(val, _loadd, ptr); break; \ 255 default: __get_user_unknown(); break; \ 256 } \ 257 } while (0) 258 #endif 259 260 #ifdef CONFIG_32BIT 261 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr) 262 #endif 263 #ifdef CONFIG_64BIT 264 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr) 265 #endif 266 267 extern void __get_user_unknown(void); 268 269 #define __get_user_common(val, size, ptr) \ 270 do { \ 271 switch (size) { \ 272 case 1: __get_data_asm(val, user_lb, ptr); break; \ 273 case 2: __get_data_asm(val, user_lh, ptr); break; \ 274 case 4: __get_data_asm(val, user_lw, ptr); break; \ 275 case 8: __GET_DW(val, user_ld, ptr); break; \ 276 default: __get_user_unknown(); break; \ 277 } \ 278 } while (0) 279 280 #define __get_user_nocheck(x, ptr, size) \ 281 ({ \ 282 int __gu_err; \ 283 \ 284 if (segment_eq(get_fs(), get_ds())) { \ 285 __get_kernel_common((x), size, ptr); \ 286 } else { \ 287 __chk_user_ptr(ptr); \ 288 __get_user_common((x), size, ptr); \ 289 } \ 290 __gu_err; \ 291 }) 292 293 #define __get_user_check(x, ptr, size) \ 294 ({ \ 295 int __gu_err = -EFAULT; \ 296 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ 297 \ 298 might_fault(); \ 299 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) { \ 300 if (segment_eq(get_fs(), get_ds())) \ 301 __get_kernel_common((x), size, __gu_ptr); \ 302 else \ 303 __get_user_common((x), size, __gu_ptr); \ 304 } \ 305 \ 306 __gu_err; \ 307 }) 308 309 #define __get_data_asm(val, insn, addr) \ 310 { \ 311 long __gu_tmp; \ 312 \ 313 __asm__ __volatile__( \ 314 "1: "insn("%1", "%3")" \n" \ 315 "2: \n" \ 316 " .insn \n" \ 317 " .section .fixup,\"ax\" \n" \ 318 "3: li %0, %4 \n" \ 319 " j 2b \n" \ 320 " .previous \n" \ 321 " .section __ex_table,\"a\" \n" \ 322 " "__UA_ADDR "\t1b, 3b \n" \ 323 " .previous \n" \ 324 : "=r" (__gu_err), "=r" (__gu_tmp) \ 325 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \ 326 \ 327 (val) = (__typeof__(*(addr))) __gu_tmp; \ 328 } 329 330 /* 331 * Get a long long 64 using 32 bit registers. 332 */ 333 #define __get_data_asm_ll32(val, insn, addr) \ 334 { \ 335 union { \ 336 unsigned long long l; \ 337 __typeof__(*(addr)) t; \ 338 } __gu_tmp; \ 339 \ 340 __asm__ __volatile__( \ 341 "1: " insn("%1", "(%3)")" \n" \ 342 "2: " insn("%D1", "4(%3)")" \n" \ 343 "3: \n" \ 344 " .insn \n" \ 345 " .section .fixup,\"ax\" \n" \ 346 "4: li %0, %4 \n" \ 347 " move %1, $0 \n" \ 348 " move %D1, $0 \n" \ 349 " j 3b \n" \ 350 " .previous \n" \ 351 " .section __ex_table,\"a\" \n" \ 352 " " __UA_ADDR " 1b, 4b \n" \ 353 " " __UA_ADDR " 2b, 4b \n" \ 354 " .previous \n" \ 355 : "=r" (__gu_err), "=&r" (__gu_tmp.l) \ 356 : "0" (0), "r" (addr), "i" (-EFAULT)); \ 357 \ 358 (val) = __gu_tmp.t; \ 359 } 360 361 #ifndef CONFIG_EVA 362 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size) 363 #else 364 /* 365 * Kernel specific functions for EVA. We need to use normal load instructions 366 * to read data from kernel when operating in EVA mode. We use these macros to 367 * avoid redefining __get_data_asm for EVA. 368 */ 369 #undef _stored 370 #undef _storew 371 #undef _storeh 372 #undef _storeb 373 #ifdef CONFIG_32BIT 374 #define _stored _storew 375 #else 376 #define _stored(reg, addr) "ld " reg ", " addr 377 #endif 378 379 #define _storew(reg, addr) "sw " reg ", " addr 380 #define _storeh(reg, addr) "sh " reg ", " addr 381 #define _storeb(reg, addr) "sb " reg ", " addr 382 383 #define __put_kernel_common(ptr, size) \ 384 do { \ 385 switch (size) { \ 386 case 1: __put_data_asm(_storeb, ptr); break; \ 387 case 2: __put_data_asm(_storeh, ptr); break; \ 388 case 4: __put_data_asm(_storew, ptr); break; \ 389 case 8: __PUT_DW(_stored, ptr); break; \ 390 default: __put_user_unknown(); break; \ 391 } \ 392 } while(0) 393 #endif 394 395 /* 396 * Yuck. We need two variants, one for 64bit operation and one 397 * for 32 bit mode and old iron. 398 */ 399 #ifdef CONFIG_32BIT 400 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr) 401 #endif 402 #ifdef CONFIG_64BIT 403 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr) 404 #endif 405 406 #define __put_user_common(ptr, size) \ 407 do { \ 408 switch (size) { \ 409 case 1: __put_data_asm(user_sb, ptr); break; \ 410 case 2: __put_data_asm(user_sh, ptr); break; \ 411 case 4: __put_data_asm(user_sw, ptr); break; \ 412 case 8: __PUT_DW(user_sd, ptr); break; \ 413 default: __put_user_unknown(); break; \ 414 } \ 415 } while (0) 416 417 #define __put_user_nocheck(x, ptr, size) \ 418 ({ \ 419 __typeof__(*(ptr)) __pu_val; \ 420 int __pu_err = 0; \ 421 \ 422 __pu_val = (x); \ 423 if (segment_eq(get_fs(), get_ds())) { \ 424 __put_kernel_common(ptr, size); \ 425 } else { \ 426 __chk_user_ptr(ptr); \ 427 __put_user_common(ptr, size); \ 428 } \ 429 __pu_err; \ 430 }) 431 432 #define __put_user_check(x, ptr, size) \ 433 ({ \ 434 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 435 __typeof__(*(ptr)) __pu_val = (x); \ 436 int __pu_err = -EFAULT; \ 437 \ 438 might_fault(); \ 439 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) { \ 440 if (segment_eq(get_fs(), get_ds())) \ 441 __put_kernel_common(__pu_addr, size); \ 442 else \ 443 __put_user_common(__pu_addr, size); \ 444 } \ 445 \ 446 __pu_err; \ 447 }) 448 449 #define __put_data_asm(insn, ptr) \ 450 { \ 451 __asm__ __volatile__( \ 452 "1: "insn("%z2", "%3")" # __put_data_asm \n" \ 453 "2: \n" \ 454 " .insn \n" \ 455 " .section .fixup,\"ax\" \n" \ 456 "3: li %0, %4 \n" \ 457 " j 2b \n" \ 458 " .previous \n" \ 459 " .section __ex_table,\"a\" \n" \ 460 " " __UA_ADDR " 1b, 3b \n" \ 461 " .previous \n" \ 462 : "=r" (__pu_err) \ 463 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \ 464 "i" (-EFAULT)); \ 465 } 466 467 #define __put_data_asm_ll32(insn, ptr) \ 468 { \ 469 __asm__ __volatile__( \ 470 "1: "insn("%2", "(%3)")" # __put_data_asm_ll32 \n" \ 471 "2: "insn("%D2", "4(%3)")" \n" \ 472 "3: \n" \ 473 " .insn \n" \ 474 " .section .fixup,\"ax\" \n" \ 475 "4: li %0, %4 \n" \ 476 " j 3b \n" \ 477 " .previous \n" \ 478 " .section __ex_table,\"a\" \n" \ 479 " " __UA_ADDR " 1b, 4b \n" \ 480 " " __UA_ADDR " 2b, 4b \n" \ 481 " .previous" \ 482 : "=r" (__pu_err) \ 483 : "0" (0), "r" (__pu_val), "r" (ptr), \ 484 "i" (-EFAULT)); \ 485 } 486 487 extern void __put_user_unknown(void); 488 489 /* 490 * ul{b,h,w} are macros and there are no equivalent macros for EVA. 491 * EVA unaligned access is handled in the ADE exception handler. 492 */ 493 #ifndef CONFIG_EVA 494 /* 495 * put_user_unaligned: - Write a simple value into user space. 496 * @x: Value to copy to user space. 497 * @ptr: Destination address, in user space. 498 * 499 * Context: User context only. This function may sleep. 500 * 501 * This macro copies a single simple value from kernel space to user 502 * space. It supports simple types like char and int, but not larger 503 * data types like structures or arrays. 504 * 505 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 506 * to the result of dereferencing @ptr. 507 * 508 * Returns zero on success, or -EFAULT on error. 509 */ 510 #define put_user_unaligned(x,ptr) \ 511 __put_user_unaligned_check((x),(ptr),sizeof(*(ptr))) 512 513 /* 514 * get_user_unaligned: - Get a simple variable from user space. 515 * @x: Variable to store result. 516 * @ptr: Source address, in user space. 517 * 518 * Context: User context only. This function may sleep. 519 * 520 * This macro copies a single simple variable from user space to kernel 521 * space. It supports simple types like char and int, but not larger 522 * data types like structures or arrays. 523 * 524 * @ptr must have pointer-to-simple-variable type, and the result of 525 * dereferencing @ptr must be assignable to @x without a cast. 526 * 527 * Returns zero on success, or -EFAULT on error. 528 * On error, the variable @x is set to zero. 529 */ 530 #define get_user_unaligned(x,ptr) \ 531 __get_user_unaligned_check((x),(ptr),sizeof(*(ptr))) 532 533 /* 534 * __put_user_unaligned: - Write a simple value into user space, with less checking. 535 * @x: Value to copy to user space. 536 * @ptr: Destination address, in user space. 537 * 538 * Context: User context only. This function may sleep. 539 * 540 * This macro copies a single simple value from kernel space to user 541 * space. It supports simple types like char and int, but not larger 542 * data types like structures or arrays. 543 * 544 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 545 * to the result of dereferencing @ptr. 546 * 547 * Caller must check the pointer with access_ok() before calling this 548 * function. 549 * 550 * Returns zero on success, or -EFAULT on error. 551 */ 552 #define __put_user_unaligned(x,ptr) \ 553 __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr))) 554 555 /* 556 * __get_user_unaligned: - Get a simple variable from user space, with less checking. 557 * @x: Variable to store result. 558 * @ptr: Source address, in user space. 559 * 560 * Context: User context only. This function may sleep. 561 * 562 * This macro copies a single simple variable from user space to kernel 563 * space. It supports simple types like char and int, but not larger 564 * data types like structures or arrays. 565 * 566 * @ptr must have pointer-to-simple-variable type, and the result of 567 * dereferencing @ptr must be assignable to @x without a cast. 568 * 569 * Caller must check the pointer with access_ok() before calling this 570 * function. 571 * 572 * Returns zero on success, or -EFAULT on error. 573 * On error, the variable @x is set to zero. 574 */ 575 #define __get_user_unaligned(x,ptr) \ 576 __get_user__unalignednocheck((x),(ptr),sizeof(*(ptr))) 577 578 /* 579 * Yuck. We need two variants, one for 64bit operation and one 580 * for 32 bit mode and old iron. 581 */ 582 #ifdef CONFIG_32BIT 583 #define __GET_USER_UNALIGNED_DW(val, ptr) \ 584 __get_user_unaligned_asm_ll32(val, ptr) 585 #endif 586 #ifdef CONFIG_64BIT 587 #define __GET_USER_UNALIGNED_DW(val, ptr) \ 588 __get_user_unaligned_asm(val, "uld", ptr) 589 #endif 590 591 extern void __get_user_unaligned_unknown(void); 592 593 #define __get_user_unaligned_common(val, size, ptr) \ 594 do { \ 595 switch (size) { \ 596 case 1: __get_data_asm(val, "lb", ptr); break; \ 597 case 2: __get_user_unaligned_asm(val, "ulh", ptr); break; \ 598 case 4: __get_user_unaligned_asm(val, "ulw", ptr); break; \ 599 case 8: __GET_USER_UNALIGNED_DW(val, ptr); break; \ 600 default: __get_user_unaligned_unknown(); break; \ 601 } \ 602 } while (0) 603 604 #define __get_user_unaligned_nocheck(x,ptr,size) \ 605 ({ \ 606 int __gu_err; \ 607 \ 608 __get_user_unaligned_common((x), size, ptr); \ 609 __gu_err; \ 610 }) 611 612 #define __get_user_unaligned_check(x,ptr,size) \ 613 ({ \ 614 int __gu_err = -EFAULT; \ 615 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ 616 \ 617 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) \ 618 __get_user_unaligned_common((x), size, __gu_ptr); \ 619 \ 620 __gu_err; \ 621 }) 622 623 #define __get_data_unaligned_asm(val, insn, addr) \ 624 { \ 625 long __gu_tmp; \ 626 \ 627 __asm__ __volatile__( \ 628 "1: " insn " %1, %3 \n" \ 629 "2: \n" \ 630 " .insn \n" \ 631 " .section .fixup,\"ax\" \n" \ 632 "3: li %0, %4 \n" \ 633 " j 2b \n" \ 634 " .previous \n" \ 635 " .section __ex_table,\"a\" \n" \ 636 " "__UA_ADDR "\t1b, 3b \n" \ 637 " "__UA_ADDR "\t1b + 4, 3b \n" \ 638 " .previous \n" \ 639 : "=r" (__gu_err), "=r" (__gu_tmp) \ 640 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \ 641 \ 642 (val) = (__typeof__(*(addr))) __gu_tmp; \ 643 } 644 645 /* 646 * Get a long long 64 using 32 bit registers. 647 */ 648 #define __get_user_unaligned_asm_ll32(val, addr) \ 649 { \ 650 unsigned long long __gu_tmp; \ 651 \ 652 __asm__ __volatile__( \ 653 "1: ulw %1, (%3) \n" \ 654 "2: ulw %D1, 4(%3) \n" \ 655 " move %0, $0 \n" \ 656 "3: \n" \ 657 " .insn \n" \ 658 " .section .fixup,\"ax\" \n" \ 659 "4: li %0, %4 \n" \ 660 " move %1, $0 \n" \ 661 " move %D1, $0 \n" \ 662 " j 3b \n" \ 663 " .previous \n" \ 664 " .section __ex_table,\"a\" \n" \ 665 " " __UA_ADDR " 1b, 4b \n" \ 666 " " __UA_ADDR " 1b + 4, 4b \n" \ 667 " " __UA_ADDR " 2b, 4b \n" \ 668 " " __UA_ADDR " 2b + 4, 4b \n" \ 669 " .previous \n" \ 670 : "=r" (__gu_err), "=&r" (__gu_tmp) \ 671 : "0" (0), "r" (addr), "i" (-EFAULT)); \ 672 (val) = (__typeof__(*(addr))) __gu_tmp; \ 673 } 674 675 /* 676 * Yuck. We need two variants, one for 64bit operation and one 677 * for 32 bit mode and old iron. 678 */ 679 #ifdef CONFIG_32BIT 680 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr) 681 #endif 682 #ifdef CONFIG_64BIT 683 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr) 684 #endif 685 686 #define __put_user_unaligned_common(ptr, size) \ 687 do { \ 688 switch (size) { \ 689 case 1: __put_data_asm("sb", ptr); break; \ 690 case 2: __put_user_unaligned_asm("ush", ptr); break; \ 691 case 4: __put_user_unaligned_asm("usw", ptr); break; \ 692 case 8: __PUT_USER_UNALIGNED_DW(ptr); break; \ 693 default: __put_user_unaligned_unknown(); break; \ 694 } while (0) 695 696 #define __put_user_unaligned_nocheck(x,ptr,size) \ 697 ({ \ 698 __typeof__(*(ptr)) __pu_val; \ 699 int __pu_err = 0; \ 700 \ 701 __pu_val = (x); \ 702 __put_user_unaligned_common(ptr, size); \ 703 __pu_err; \ 704 }) 705 706 #define __put_user_unaligned_check(x,ptr,size) \ 707 ({ \ 708 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 709 __typeof__(*(ptr)) __pu_val = (x); \ 710 int __pu_err = -EFAULT; \ 711 \ 712 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) \ 713 __put_user_unaligned_common(__pu_addr, size); \ 714 \ 715 __pu_err; \ 716 }) 717 718 #define __put_user_unaligned_asm(insn, ptr) \ 719 { \ 720 __asm__ __volatile__( \ 721 "1: " insn " %z2, %3 # __put_user_unaligned_asm\n" \ 722 "2: \n" \ 723 " .insn \n" \ 724 " .section .fixup,\"ax\" \n" \ 725 "3: li %0, %4 \n" \ 726 " j 2b \n" \ 727 " .previous \n" \ 728 " .section __ex_table,\"a\" \n" \ 729 " " __UA_ADDR " 1b, 3b \n" \ 730 " .previous \n" \ 731 : "=r" (__pu_err) \ 732 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \ 733 "i" (-EFAULT)); \ 734 } 735 736 #define __put_user_unaligned_asm_ll32(ptr) \ 737 { \ 738 __asm__ __volatile__( \ 739 "1: sw %2, (%3) # __put_user_unaligned_asm_ll32 \n" \ 740 "2: sw %D2, 4(%3) \n" \ 741 "3: \n" \ 742 " .insn \n" \ 743 " .section .fixup,\"ax\" \n" \ 744 "4: li %0, %4 \n" \ 745 " j 3b \n" \ 746 " .previous \n" \ 747 " .section __ex_table,\"a\" \n" \ 748 " " __UA_ADDR " 1b, 4b \n" \ 749 " " __UA_ADDR " 1b + 4, 4b \n" \ 750 " " __UA_ADDR " 2b, 4b \n" \ 751 " " __UA_ADDR " 2b + 4, 4b \n" \ 752 " .previous" \ 753 : "=r" (__pu_err) \ 754 : "0" (0), "r" (__pu_val), "r" (ptr), \ 755 "i" (-EFAULT)); \ 756 } 757 758 extern void __put_user_unaligned_unknown(void); 759 #endif 760 761 /* 762 * We're generating jump to subroutines which will be outside the range of 763 * jump instructions 764 */ 765 #ifdef MODULE 766 #define __MODULE_JAL(destination) \ 767 ".set\tnoat\n\t" \ 768 __UA_LA "\t$1, " #destination "\n\t" \ 769 "jalr\t$1\n\t" \ 770 ".set\tat\n\t" 771 #else 772 #define __MODULE_JAL(destination) \ 773 "jal\t" #destination "\n\t" 774 #endif 775 776 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS 777 #define DADDI_SCRATCH "$0" 778 #else 779 #define DADDI_SCRATCH "$3" 780 #endif 781 782 extern size_t __copy_user(void *__to, const void *__from, size_t __n); 783 784 #ifndef CONFIG_EVA 785 #define __invoke_copy_to_user(to, from, n) \ 786 ({ \ 787 register void __user *__cu_to_r __asm__("$4"); \ 788 register const void *__cu_from_r __asm__("$5"); \ 789 register long __cu_len_r __asm__("$6"); \ 790 \ 791 __cu_to_r = (to); \ 792 __cu_from_r = (from); \ 793 __cu_len_r = (n); \ 794 __asm__ __volatile__( \ 795 __MODULE_JAL(__copy_user) \ 796 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 797 : \ 798 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 799 DADDI_SCRATCH, "memory"); \ 800 __cu_len_r; \ 801 }) 802 803 #define __invoke_copy_to_kernel(to, from, n) \ 804 __invoke_copy_to_user(to, from, n) 805 806 #endif 807 808 /* 809 * __copy_to_user: - Copy a block of data into user space, with less checking. 810 * @to: Destination address, in user space. 811 * @from: Source address, in kernel space. 812 * @n: Number of bytes to copy. 813 * 814 * Context: User context only. This function may sleep. 815 * 816 * Copy data from kernel space to user space. Caller must check 817 * the specified block with access_ok() before calling this function. 818 * 819 * Returns number of bytes that could not be copied. 820 * On success, this will be zero. 821 */ 822 #define __copy_to_user(to, from, n) \ 823 ({ \ 824 void __user *__cu_to; \ 825 const void *__cu_from; \ 826 long __cu_len; \ 827 \ 828 __cu_to = (to); \ 829 __cu_from = (from); \ 830 __cu_len = (n); \ 831 might_fault(); \ 832 if (segment_eq(get_fs(), get_ds())) \ 833 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \ 834 __cu_len); \ 835 else \ 836 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ 837 __cu_len); \ 838 __cu_len; \ 839 }) 840 841 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); 842 843 #define __copy_to_user_inatomic(to, from, n) \ 844 ({ \ 845 void __user *__cu_to; \ 846 const void *__cu_from; \ 847 long __cu_len; \ 848 \ 849 __cu_to = (to); \ 850 __cu_from = (from); \ 851 __cu_len = (n); \ 852 if (segment_eq(get_fs(), get_ds())) \ 853 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \ 854 __cu_len); \ 855 else \ 856 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ 857 __cu_len); \ 858 __cu_len; \ 859 }) 860 861 #define __copy_from_user_inatomic(to, from, n) \ 862 ({ \ 863 void *__cu_to; \ 864 const void __user *__cu_from; \ 865 long __cu_len; \ 866 \ 867 __cu_to = (to); \ 868 __cu_from = (from); \ 869 __cu_len = (n); \ 870 if (segment_eq(get_fs(), get_ds())) \ 871 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to, \ 872 __cu_from,\ 873 __cu_len);\ 874 else \ 875 __cu_len = __invoke_copy_from_user_inatomic(__cu_to, \ 876 __cu_from, \ 877 __cu_len); \ 878 __cu_len; \ 879 }) 880 881 /* 882 * copy_to_user: - Copy a block of data into user space. 883 * @to: Destination address, in user space. 884 * @from: Source address, in kernel space. 885 * @n: Number of bytes to copy. 886 * 887 * Context: User context only. This function may sleep. 888 * 889 * Copy data from kernel space to user space. 890 * 891 * Returns number of bytes that could not be copied. 892 * On success, this will be zero. 893 */ 894 #define copy_to_user(to, from, n) \ 895 ({ \ 896 void __user *__cu_to; \ 897 const void *__cu_from; \ 898 long __cu_len; \ 899 \ 900 __cu_to = (to); \ 901 __cu_from = (from); \ 902 __cu_len = (n); \ 903 if (segment_eq(get_fs(), get_ds())) { \ 904 __cu_len = __invoke_copy_to_kernel(__cu_to, \ 905 __cu_from, \ 906 __cu_len); \ 907 } else { \ 908 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \ 909 might_fault(); \ 910 __cu_len = __invoke_copy_to_user(__cu_to, \ 911 __cu_from, \ 912 __cu_len); \ 913 } \ 914 } \ 915 __cu_len; \ 916 }) 917 918 #ifndef CONFIG_EVA 919 920 #define __invoke_copy_from_user(to, from, n) \ 921 ({ \ 922 register void *__cu_to_r __asm__("$4"); \ 923 register const void __user *__cu_from_r __asm__("$5"); \ 924 register long __cu_len_r __asm__("$6"); \ 925 \ 926 __cu_to_r = (to); \ 927 __cu_from_r = (from); \ 928 __cu_len_r = (n); \ 929 __asm__ __volatile__( \ 930 ".set\tnoreorder\n\t" \ 931 __MODULE_JAL(__copy_user) \ 932 ".set\tnoat\n\t" \ 933 __UA_ADDU "\t$1, %1, %2\n\t" \ 934 ".set\tat\n\t" \ 935 ".set\treorder" \ 936 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 937 : \ 938 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 939 DADDI_SCRATCH, "memory"); \ 940 __cu_len_r; \ 941 }) 942 943 #define __invoke_copy_from_kernel(to, from, n) \ 944 __invoke_copy_from_user(to, from, n) 945 946 /* For userland <-> userland operations */ 947 #define ___invoke_copy_in_user(to, from, n) \ 948 __invoke_copy_from_user(to, from, n) 949 950 /* For kernel <-> kernel operations */ 951 #define ___invoke_copy_in_kernel(to, from, n) \ 952 __invoke_copy_from_user(to, from, n) 953 954 #define __invoke_copy_from_user_inatomic(to, from, n) \ 955 ({ \ 956 register void *__cu_to_r __asm__("$4"); \ 957 register const void __user *__cu_from_r __asm__("$5"); \ 958 register long __cu_len_r __asm__("$6"); \ 959 \ 960 __cu_to_r = (to); \ 961 __cu_from_r = (from); \ 962 __cu_len_r = (n); \ 963 __asm__ __volatile__( \ 964 ".set\tnoreorder\n\t" \ 965 __MODULE_JAL(__copy_user_inatomic) \ 966 ".set\tnoat\n\t" \ 967 __UA_ADDU "\t$1, %1, %2\n\t" \ 968 ".set\tat\n\t" \ 969 ".set\treorder" \ 970 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 971 : \ 972 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 973 DADDI_SCRATCH, "memory"); \ 974 __cu_len_r; \ 975 }) 976 977 #define __invoke_copy_from_kernel_inatomic(to, from, n) \ 978 __invoke_copy_from_user_inatomic(to, from, n) \ 979 980 #else 981 982 /* EVA specific functions */ 983 984 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from, 985 size_t __n); 986 extern size_t __copy_from_user_eva(void *__to, const void *__from, 987 size_t __n); 988 extern size_t __copy_to_user_eva(void *__to, const void *__from, 989 size_t __n); 990 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); 991 992 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr) \ 993 ({ \ 994 register void *__cu_to_r __asm__("$4"); \ 995 register const void __user *__cu_from_r __asm__("$5"); \ 996 register long __cu_len_r __asm__("$6"); \ 997 \ 998 __cu_to_r = (to); \ 999 __cu_from_r = (from); \ 1000 __cu_len_r = (n); \ 1001 __asm__ __volatile__( \ 1002 ".set\tnoreorder\n\t" \ 1003 __MODULE_JAL(func_ptr) \ 1004 ".set\tnoat\n\t" \ 1005 __UA_ADDU "\t$1, %1, %2\n\t" \ 1006 ".set\tat\n\t" \ 1007 ".set\treorder" \ 1008 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 1009 : \ 1010 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 1011 DADDI_SCRATCH, "memory"); \ 1012 __cu_len_r; \ 1013 }) 1014 1015 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr) \ 1016 ({ \ 1017 register void *__cu_to_r __asm__("$4"); \ 1018 register const void __user *__cu_from_r __asm__("$5"); \ 1019 register long __cu_len_r __asm__("$6"); \ 1020 \ 1021 __cu_to_r = (to); \ 1022 __cu_from_r = (from); \ 1023 __cu_len_r = (n); \ 1024 __asm__ __volatile__( \ 1025 __MODULE_JAL(func_ptr) \ 1026 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 1027 : \ 1028 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 1029 DADDI_SCRATCH, "memory"); \ 1030 __cu_len_r; \ 1031 }) 1032 1033 /* 1034 * Source or destination address is in userland. We need to go through 1035 * the TLB 1036 */ 1037 #define __invoke_copy_from_user(to, from, n) \ 1038 __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva) 1039 1040 #define __invoke_copy_from_user_inatomic(to, from, n) \ 1041 __invoke_copy_from_user_eva_generic(to, from, n, \ 1042 __copy_user_inatomic_eva) 1043 1044 #define __invoke_copy_to_user(to, from, n) \ 1045 __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva) 1046 1047 #define ___invoke_copy_in_user(to, from, n) \ 1048 __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva) 1049 1050 /* 1051 * Source or destination address in the kernel. We are not going through 1052 * the TLB 1053 */ 1054 #define __invoke_copy_from_kernel(to, from, n) \ 1055 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user) 1056 1057 #define __invoke_copy_from_kernel_inatomic(to, from, n) \ 1058 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic) 1059 1060 #define __invoke_copy_to_kernel(to, from, n) \ 1061 __invoke_copy_to_user_eva_generic(to, from, n, __copy_user) 1062 1063 #define ___invoke_copy_in_kernel(to, from, n) \ 1064 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user) 1065 1066 #endif /* CONFIG_EVA */ 1067 1068 /* 1069 * __copy_from_user: - Copy a block of data from user space, with less checking. 1070 * @to: Destination address, in kernel space. 1071 * @from: Source address, in user space. 1072 * @n: Number of bytes to copy. 1073 * 1074 * Context: User context only. This function may sleep. 1075 * 1076 * Copy data from user space to kernel space. Caller must check 1077 * the specified block with access_ok() before calling this function. 1078 * 1079 * Returns number of bytes that could not be copied. 1080 * On success, this will be zero. 1081 * 1082 * If some data could not be copied, this function will pad the copied 1083 * data to the requested size using zero bytes. 1084 */ 1085 #define __copy_from_user(to, from, n) \ 1086 ({ \ 1087 void *__cu_to; \ 1088 const void __user *__cu_from; \ 1089 long __cu_len; \ 1090 \ 1091 __cu_to = (to); \ 1092 __cu_from = (from); \ 1093 __cu_len = (n); \ 1094 might_fault(); \ 1095 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ 1096 __cu_len); \ 1097 __cu_len; \ 1098 }) 1099 1100 /* 1101 * copy_from_user: - Copy a block of data from user space. 1102 * @to: Destination address, in kernel space. 1103 * @from: Source address, in user space. 1104 * @n: Number of bytes to copy. 1105 * 1106 * Context: User context only. This function may sleep. 1107 * 1108 * Copy data from user space to kernel space. 1109 * 1110 * Returns number of bytes that could not be copied. 1111 * On success, this will be zero. 1112 * 1113 * If some data could not be copied, this function will pad the copied 1114 * data to the requested size using zero bytes. 1115 */ 1116 #define copy_from_user(to, from, n) \ 1117 ({ \ 1118 void *__cu_to; \ 1119 const void __user *__cu_from; \ 1120 long __cu_len; \ 1121 \ 1122 __cu_to = (to); \ 1123 __cu_from = (from); \ 1124 __cu_len = (n); \ 1125 if (segment_eq(get_fs(), get_ds())) { \ 1126 __cu_len = __invoke_copy_from_kernel(__cu_to, \ 1127 __cu_from, \ 1128 __cu_len); \ 1129 } else { \ 1130 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \ 1131 might_fault(); \ 1132 __cu_len = __invoke_copy_from_user(__cu_to, \ 1133 __cu_from, \ 1134 __cu_len); \ 1135 } \ 1136 } \ 1137 __cu_len; \ 1138 }) 1139 1140 #define __copy_in_user(to, from, n) \ 1141 ({ \ 1142 void __user *__cu_to; \ 1143 const void __user *__cu_from; \ 1144 long __cu_len; \ 1145 \ 1146 __cu_to = (to); \ 1147 __cu_from = (from); \ 1148 __cu_len = (n); \ 1149 if (segment_eq(get_fs(), get_ds())) { \ 1150 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \ 1151 __cu_len); \ 1152 } else { \ 1153 might_fault(); \ 1154 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from, \ 1155 __cu_len); \ 1156 } \ 1157 __cu_len; \ 1158 }) 1159 1160 #define copy_in_user(to, from, n) \ 1161 ({ \ 1162 void __user *__cu_to; \ 1163 const void __user *__cu_from; \ 1164 long __cu_len; \ 1165 \ 1166 __cu_to = (to); \ 1167 __cu_from = (from); \ 1168 __cu_len = (n); \ 1169 if (segment_eq(get_fs(), get_ds())) { \ 1170 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from, \ 1171 __cu_len); \ 1172 } else { \ 1173 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\ 1174 access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\ 1175 might_fault(); \ 1176 __cu_len = ___invoke_copy_in_user(__cu_to, \ 1177 __cu_from, \ 1178 __cu_len); \ 1179 } \ 1180 } \ 1181 __cu_len; \ 1182 }) 1183 1184 /* 1185 * __clear_user: - Zero a block of memory in user space, with less checking. 1186 * @to: Destination address, in user space. 1187 * @n: Number of bytes to zero. 1188 * 1189 * Zero a block of memory in user space. Caller must check 1190 * the specified block with access_ok() before calling this function. 1191 * 1192 * Returns number of bytes that could not be cleared. 1193 * On success, this will be zero. 1194 */ 1195 static inline __kernel_size_t 1196 __clear_user(void __user *addr, __kernel_size_t size) 1197 { 1198 __kernel_size_t res; 1199 1200 might_fault(); 1201 __asm__ __volatile__( 1202 "move\t$4, %1\n\t" 1203 "move\t$5, $0\n\t" 1204 "move\t$6, %2\n\t" 1205 __MODULE_JAL(__bzero) 1206 "move\t%0, $6" 1207 : "=r" (res) 1208 : "r" (addr), "r" (size) 1209 : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31"); 1210 1211 return res; 1212 } 1213 1214 #define clear_user(addr,n) \ 1215 ({ \ 1216 void __user * __cl_addr = (addr); \ 1217 unsigned long __cl_size = (n); \ 1218 if (__cl_size && access_ok(VERIFY_WRITE, \ 1219 __cl_addr, __cl_size)) \ 1220 __cl_size = __clear_user(__cl_addr, __cl_size); \ 1221 __cl_size; \ 1222 }) 1223 1224 /* 1225 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. 1226 * @dst: Destination address, in kernel space. This buffer must be at 1227 * least @count bytes long. 1228 * @src: Source address, in user space. 1229 * @count: Maximum number of bytes to copy, including the trailing NUL. 1230 * 1231 * Copies a NUL-terminated string from userspace to kernel space. 1232 * Caller must check the specified block with access_ok() before calling 1233 * this function. 1234 * 1235 * On success, returns the length of the string (not including the trailing 1236 * NUL). 1237 * 1238 * If access to userspace fails, returns -EFAULT (some data may have been 1239 * copied). 1240 * 1241 * If @count is smaller than the length of the string, copies @count bytes 1242 * and returns @count. 1243 */ 1244 static inline long 1245 __strncpy_from_user(char *__to, const char __user *__from, long __len) 1246 { 1247 long res; 1248 1249 if (segment_eq(get_fs(), get_ds())) { 1250 __asm__ __volatile__( 1251 "move\t$4, %1\n\t" 1252 "move\t$5, %2\n\t" 1253 "move\t$6, %3\n\t" 1254 __MODULE_JAL(__strncpy_from_kernel_nocheck_asm) 1255 "move\t%0, $2" 1256 : "=r" (res) 1257 : "r" (__to), "r" (__from), "r" (__len) 1258 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1259 } else { 1260 might_fault(); 1261 __asm__ __volatile__( 1262 "move\t$4, %1\n\t" 1263 "move\t$5, %2\n\t" 1264 "move\t$6, %3\n\t" 1265 __MODULE_JAL(__strncpy_from_user_nocheck_asm) 1266 "move\t%0, $2" 1267 : "=r" (res) 1268 : "r" (__to), "r" (__from), "r" (__len) 1269 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1270 } 1271 1272 return res; 1273 } 1274 1275 /* 1276 * strncpy_from_user: - Copy a NUL terminated string from userspace. 1277 * @dst: Destination address, in kernel space. This buffer must be at 1278 * least @count bytes long. 1279 * @src: Source address, in user space. 1280 * @count: Maximum number of bytes to copy, including the trailing NUL. 1281 * 1282 * Copies a NUL-terminated string from userspace to kernel space. 1283 * 1284 * On success, returns the length of the string (not including the trailing 1285 * NUL). 1286 * 1287 * If access to userspace fails, returns -EFAULT (some data may have been 1288 * copied). 1289 * 1290 * If @count is smaller than the length of the string, copies @count bytes 1291 * and returns @count. 1292 */ 1293 static inline long 1294 strncpy_from_user(char *__to, const char __user *__from, long __len) 1295 { 1296 long res; 1297 1298 if (segment_eq(get_fs(), get_ds())) { 1299 __asm__ __volatile__( 1300 "move\t$4, %1\n\t" 1301 "move\t$5, %2\n\t" 1302 "move\t$6, %3\n\t" 1303 __MODULE_JAL(__strncpy_from_kernel_asm) 1304 "move\t%0, $2" 1305 : "=r" (res) 1306 : "r" (__to), "r" (__from), "r" (__len) 1307 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1308 } else { 1309 might_fault(); 1310 __asm__ __volatile__( 1311 "move\t$4, %1\n\t" 1312 "move\t$5, %2\n\t" 1313 "move\t$6, %3\n\t" 1314 __MODULE_JAL(__strncpy_from_user_asm) 1315 "move\t%0, $2" 1316 : "=r" (res) 1317 : "r" (__to), "r" (__from), "r" (__len) 1318 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1319 } 1320 1321 return res; 1322 } 1323 1324 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */ 1325 static inline long __strlen_user(const char __user *s) 1326 { 1327 long res; 1328 1329 if (segment_eq(get_fs(), get_ds())) { 1330 __asm__ __volatile__( 1331 "move\t$4, %1\n\t" 1332 __MODULE_JAL(__strlen_kernel_nocheck_asm) 1333 "move\t%0, $2" 1334 : "=r" (res) 1335 : "r" (s) 1336 : "$2", "$4", __UA_t0, "$31"); 1337 } else { 1338 might_fault(); 1339 __asm__ __volatile__( 1340 "move\t$4, %1\n\t" 1341 __MODULE_JAL(__strlen_user_nocheck_asm) 1342 "move\t%0, $2" 1343 : "=r" (res) 1344 : "r" (s) 1345 : "$2", "$4", __UA_t0, "$31"); 1346 } 1347 1348 return res; 1349 } 1350 1351 /* 1352 * strlen_user: - Get the size of a string in user space. 1353 * @str: The string to measure. 1354 * 1355 * Context: User context only. This function may sleep. 1356 * 1357 * Get the size of a NUL-terminated string in user space. 1358 * 1359 * Returns the size of the string INCLUDING the terminating NUL. 1360 * On exception, returns 0. 1361 * 1362 * If there is a limit on the length of a valid string, you may wish to 1363 * consider using strnlen_user() instead. 1364 */ 1365 static inline long strlen_user(const char __user *s) 1366 { 1367 long res; 1368 1369 if (segment_eq(get_fs(), get_ds())) { 1370 __asm__ __volatile__( 1371 "move\t$4, %1\n\t" 1372 __MODULE_JAL(__strlen_kernel_asm) 1373 "move\t%0, $2" 1374 : "=r" (res) 1375 : "r" (s) 1376 : "$2", "$4", __UA_t0, "$31"); 1377 } else { 1378 might_fault(); 1379 __asm__ __volatile__( 1380 "move\t$4, %1\n\t" 1381 __MODULE_JAL(__strlen_kernel_asm) 1382 "move\t%0, $2" 1383 : "=r" (res) 1384 : "r" (s) 1385 : "$2", "$4", __UA_t0, "$31"); 1386 } 1387 1388 return res; 1389 } 1390 1391 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */ 1392 static inline long __strnlen_user(const char __user *s, long n) 1393 { 1394 long res; 1395 1396 if (segment_eq(get_fs(), get_ds())) { 1397 __asm__ __volatile__( 1398 "move\t$4, %1\n\t" 1399 "move\t$5, %2\n\t" 1400 __MODULE_JAL(__strnlen_kernel_nocheck_asm) 1401 "move\t%0, $2" 1402 : "=r" (res) 1403 : "r" (s), "r" (n) 1404 : "$2", "$4", "$5", __UA_t0, "$31"); 1405 } else { 1406 might_fault(); 1407 __asm__ __volatile__( 1408 "move\t$4, %1\n\t" 1409 "move\t$5, %2\n\t" 1410 __MODULE_JAL(__strnlen_user_nocheck_asm) 1411 "move\t%0, $2" 1412 : "=r" (res) 1413 : "r" (s), "r" (n) 1414 : "$2", "$4", "$5", __UA_t0, "$31"); 1415 } 1416 1417 return res; 1418 } 1419 1420 /* 1421 * strlen_user: - Get the size of a string in user space. 1422 * @str: The string to measure. 1423 * 1424 * Context: User context only. This function may sleep. 1425 * 1426 * Get the size of a NUL-terminated string in user space. 1427 * 1428 * Returns the size of the string INCLUDING the terminating NUL. 1429 * On exception, returns 0. 1430 * 1431 * If there is a limit on the length of a valid string, you may wish to 1432 * consider using strnlen_user() instead. 1433 */ 1434 static inline long strnlen_user(const char __user *s, long n) 1435 { 1436 long res; 1437 1438 might_fault(); 1439 if (segment_eq(get_fs(), get_ds())) { 1440 __asm__ __volatile__( 1441 "move\t$4, %1\n\t" 1442 "move\t$5, %2\n\t" 1443 __MODULE_JAL(__strnlen_kernel_asm) 1444 "move\t%0, $2" 1445 : "=r" (res) 1446 : "r" (s), "r" (n) 1447 : "$2", "$4", "$5", __UA_t0, "$31"); 1448 } else { 1449 __asm__ __volatile__( 1450 "move\t$4, %1\n\t" 1451 "move\t$5, %2\n\t" 1452 __MODULE_JAL(__strnlen_user_asm) 1453 "move\t%0, $2" 1454 : "=r" (res) 1455 : "r" (s), "r" (n) 1456 : "$2", "$4", "$5", __UA_t0, "$31"); 1457 } 1458 1459 return res; 1460 } 1461 1462 struct exception_table_entry 1463 { 1464 unsigned long insn; 1465 unsigned long nextinsn; 1466 }; 1467 1468 extern int fixup_exception(struct pt_regs *regs); 1469 1470 #endif /* _ASM_UACCESS_H */ 1471