1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * Copyright (c) 2009, Intel Corporation 28 * All rights reserved. 29 */ 30 31/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ 32/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ 33/* All Rights Reserved */ 34 35/* Copyright (c) 1987, 1988 Microsoft Corporation */ 36/* All Rights Reserved */ 37 38/* 39 * Copyright 2016 Joyent, Inc. 40 */ 41 42#include <sys/errno.h> 43#include <sys/asm_linkage.h> 44 45#if defined(__lint) 46#include <sys/types.h> 47#include <sys/systm.h> 48#else /* __lint */ 49#include "assym.h" 50#endif /* __lint */ 51 52#define KCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */ 53#define XCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */ 54/* 55 * Non-temopral access (NTA) alignment requirement 56 */ 57#define NTA_ALIGN_SIZE 4 /* Must be at least 4-byte aligned */ 58#define NTA_ALIGN_MASK _CONST(NTA_ALIGN_SIZE-1) 59#define COUNT_ALIGN_SIZE 16 /* Must be at least 16-byte aligned */ 60#define COUNT_ALIGN_MASK _CONST(COUNT_ALIGN_SIZE-1) 61 62/* 63 * The optimal 64-bit bcopy and kcopy for modern x86 processors uses 64 * "rep smovq" for large sizes. Performance data shows that many calls to 65 * bcopy/kcopy/bzero/kzero operate on small buffers. For best performance for 66 * these small sizes unrolled code is used. For medium sizes loops writing 67 * 64-bytes per loop are used. Transition points were determined experimentally. 68 */ 69#define BZERO_USE_REP (1024) 70#define BCOPY_DFLT_REP (128) 71#define BCOPY_NHM_REP (768) 72 73/* 74 * Copy a block of storage, returning an error code if `from' or 75 * `to' takes a kernel pagefault which cannot be resolved. 76 * Returns errno value on pagefault error, 0 if all ok 77 */ 78 79#if defined(__lint) 80 81/* ARGSUSED */ 82int 83kcopy(const void *from, void *to, size_t count) 84{ return (0); } 85 86#else /* __lint */ 87 88 .globl kernelbase 89 .globl postbootkernelbase 90 91#if defined(__amd64) 92 93 ENTRY(kcopy) 94 pushq %rbp 95 movq %rsp, %rbp 96#ifdef DEBUG 97 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */ 98 jb 0f 99 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */ 100 jnb 1f 1010: leaq .kcopy_panic_msg(%rip), %rdi 102 xorl %eax, %eax 103 call panic 1041: 105#endif 106 /* 107 * pass lofault value as 4th argument to do_copy_fault 108 */ 109 leaq _kcopy_copyerr(%rip), %rcx 110 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */ 111 112do_copy_fault: 113 movq T_LOFAULT(%r9), %r11 /* save the current lofault */ 114 movq %rcx, T_LOFAULT(%r9) /* new lofault */ 115 call bcopy_altentry 116 xorl %eax, %eax /* return 0 (success) */ 117 118 /* 119 * A fault during do_copy_fault is indicated through an errno value 120 * in %rax and we iretq from the trap handler to here. 121 */ 122_kcopy_copyerr: 123 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 124 leave 125 ret 126 SET_SIZE(kcopy) 127 128#elif defined(__i386) 129 130#define ARG_FROM 8 131#define ARG_TO 12 132#define ARG_COUNT 16 133 134 ENTRY(kcopy) 135#ifdef DEBUG 136 pushl %ebp 137 movl %esp, %ebp 138 movl postbootkernelbase, %eax 139 cmpl %eax, ARG_FROM(%ebp) 140 jb 0f 141 cmpl %eax, ARG_TO(%ebp) 142 jnb 1f 1430: pushl $.kcopy_panic_msg 144 call panic 1451: popl %ebp 146#endif 147 lea _kcopy_copyerr, %eax /* lofault value */ 148 movl %gs:CPU_THREAD, %edx 149 150do_copy_fault: 151 pushl %ebp 152 movl %esp, %ebp /* setup stack frame */ 153 pushl %esi 154 pushl %edi /* save registers */ 155 156 movl T_LOFAULT(%edx), %edi 157 pushl %edi /* save the current lofault */ 158 movl %eax, T_LOFAULT(%edx) /* new lofault */ 159 160 movl ARG_COUNT(%ebp), %ecx 161 movl ARG_FROM(%ebp), %esi 162 movl ARG_TO(%ebp), %edi 163 shrl $2, %ecx /* word count */ 164 rep 165 smovl 166 movl ARG_COUNT(%ebp), %ecx 167 andl $3, %ecx /* bytes left over */ 168 rep 169 smovb 170 xorl %eax, %eax 171 172 /* 173 * A fault during do_copy_fault is indicated through an errno value 174 * in %eax and we iret from the trap handler to here. 175 */ 176_kcopy_copyerr: 177 popl %ecx 178 popl %edi 179 movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */ 180 popl %esi 181 popl %ebp 182 ret 183 SET_SIZE(kcopy) 184 185#undef ARG_FROM 186#undef ARG_TO 187#undef ARG_COUNT 188 189#endif /* __i386 */ 190#endif /* __lint */ 191 192#if defined(__lint) 193 194/* 195 * Copy a block of storage. Similar to kcopy but uses non-temporal 196 * instructions. 197 */ 198 199/* ARGSUSED */ 200int 201kcopy_nta(const void *from, void *to, size_t count, int copy_cached) 202{ return (0); } 203 204#else /* __lint */ 205 206#if defined(__amd64) 207 208#define COPY_LOOP_INIT(src, dst, cnt) \ 209 addq cnt, src; \ 210 addq cnt, dst; \ 211 shrq $3, cnt; \ 212 neg cnt 213 214 /* Copy 16 bytes per loop. Uses %rax and %r8 */ 215#define COPY_LOOP_BODY(src, dst, cnt) \ 216 prefetchnta 0x100(src, cnt, 8); \ 217 movq (src, cnt, 8), %rax; \ 218 movq 0x8(src, cnt, 8), %r8; \ 219 movnti %rax, (dst, cnt, 8); \ 220 movnti %r8, 0x8(dst, cnt, 8); \ 221 addq $2, cnt 222 223 ENTRY(kcopy_nta) 224 pushq %rbp 225 movq %rsp, %rbp 226#ifdef DEBUG 227 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */ 228 jb 0f 229 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */ 230 jnb 1f 2310: leaq .kcopy_panic_msg(%rip), %rdi 232 xorl %eax, %eax 233 call panic 2341: 235#endif 236 237 movq %gs:CPU_THREAD, %r9 238 cmpq $0, %rcx /* No non-temporal access? */ 239 /* 240 * pass lofault value as 4th argument to do_copy_fault 241 */ 242 leaq _kcopy_nta_copyerr(%rip), %rcx /* doesn't set rflags */ 243 jnz do_copy_fault /* use regular access */ 244 /* 245 * Make sure cnt is >= KCOPY_MIN_SIZE 246 */ 247 cmpq $KCOPY_MIN_SIZE, %rdx 248 jb do_copy_fault 249 250 /* 251 * Make sure src and dst are NTA_ALIGN_SIZE aligned, 252 * count is COUNT_ALIGN_SIZE aligned. 253 */ 254 movq %rdi, %r10 255 orq %rsi, %r10 256 andq $NTA_ALIGN_MASK, %r10 257 orq %rdx, %r10 258 andq $COUNT_ALIGN_MASK, %r10 259 jnz do_copy_fault 260 261 ALTENTRY(do_copy_fault_nta) 262 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */ 263 movq T_LOFAULT(%r9), %r11 /* save the current lofault */ 264 movq %rcx, T_LOFAULT(%r9) /* new lofault */ 265 266 /* 267 * COPY_LOOP_BODY uses %rax and %r8 268 */ 269 COPY_LOOP_INIT(%rdi, %rsi, %rdx) 2702: COPY_LOOP_BODY(%rdi, %rsi, %rdx) 271 jnz 2b 272 273 mfence 274 xorl %eax, %eax /* return 0 (success) */ 275 276_kcopy_nta_copyerr: 277 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 278 leave 279 ret 280 SET_SIZE(do_copy_fault_nta) 281 SET_SIZE(kcopy_nta) 282 283#elif defined(__i386) 284 285#define ARG_FROM 8 286#define ARG_TO 12 287#define ARG_COUNT 16 288 289#define COPY_LOOP_INIT(src, dst, cnt) \ 290 addl cnt, src; \ 291 addl cnt, dst; \ 292 shrl $3, cnt; \ 293 neg cnt 294 295#define COPY_LOOP_BODY(src, dst, cnt) \ 296 prefetchnta 0x100(src, cnt, 8); \ 297 movl (src, cnt, 8), %esi; \ 298 movnti %esi, (dst, cnt, 8); \ 299 movl 0x4(src, cnt, 8), %esi; \ 300 movnti %esi, 0x4(dst, cnt, 8); \ 301 movl 0x8(src, cnt, 8), %esi; \ 302 movnti %esi, 0x8(dst, cnt, 8); \ 303 movl 0xc(src, cnt, 8), %esi; \ 304 movnti %esi, 0xc(dst, cnt, 8); \ 305 addl $2, cnt 306 307 /* 308 * kcopy_nta is not implemented for 32-bit as no performance 309 * improvement was shown. We simply jump directly to kcopy 310 * and discard the 4 arguments. 311 */ 312 ENTRY(kcopy_nta) 313 jmp kcopy 314 315 lea _kcopy_nta_copyerr, %eax /* lofault value */ 316 ALTENTRY(do_copy_fault_nta) 317 pushl %ebp 318 movl %esp, %ebp /* setup stack frame */ 319 pushl %esi 320 pushl %edi 321 322 movl %gs:CPU_THREAD, %edx 323 movl T_LOFAULT(%edx), %edi 324 pushl %edi /* save the current lofault */ 325 movl %eax, T_LOFAULT(%edx) /* new lofault */ 326 327 /* COPY_LOOP_BODY needs to use %esi */ 328 movl ARG_COUNT(%ebp), %ecx 329 movl ARG_FROM(%ebp), %edi 330 movl ARG_TO(%ebp), %eax 331 COPY_LOOP_INIT(%edi, %eax, %ecx) 3321: COPY_LOOP_BODY(%edi, %eax, %ecx) 333 jnz 1b 334 mfence 335 336 xorl %eax, %eax 337_kcopy_nta_copyerr: 338 popl %ecx 339 popl %edi 340 movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */ 341 popl %esi 342 leave 343 ret 344 SET_SIZE(do_copy_fault_nta) 345 SET_SIZE(kcopy_nta) 346 347#undef ARG_FROM 348#undef ARG_TO 349#undef ARG_COUNT 350 351#endif /* __i386 */ 352#endif /* __lint */ 353 354#if defined(__lint) 355 356/* ARGSUSED */ 357void 358bcopy(const void *from, void *to, size_t count) 359{} 360 361#else /* __lint */ 362 363#if defined(__amd64) 364 365 ENTRY(bcopy) 366#ifdef DEBUG 367 orq %rdx, %rdx /* %rdx = count */ 368 jz 1f 369 cmpq postbootkernelbase(%rip), %rdi /* %rdi = from */ 370 jb 0f 371 cmpq postbootkernelbase(%rip), %rsi /* %rsi = to */ 372 jnb 1f 3730: leaq .bcopy_panic_msg(%rip), %rdi 374 jmp call_panic /* setup stack and call panic */ 3751: 376#endif 377 /* 378 * bcopy_altentry() is called from kcopy, i.e., do_copy_fault. 379 * kcopy assumes that bcopy doesn't touch %r9 and %r11. If bcopy 380 * uses these registers in future they must be saved and restored. 381 */ 382 ALTENTRY(bcopy_altentry) 383do_copy: 384#define L(s) .bcopy/**/s 385 cmpq $0x50, %rdx /* 80 */ 386 jae bcopy_ck_size 387 388 /* 389 * Performance data shows many caller's copy small buffers. So for 390 * best perf for these sizes unrolled code is used. Store data without 391 * worrying about alignment. 392 */ 393 leaq L(fwdPxQx)(%rip), %r10 394 addq %rdx, %rdi 395 addq %rdx, %rsi 396 movslq (%r10,%rdx,4), %rcx 397 leaq (%rcx,%r10,1), %r10 398 jmpq *%r10 399 400 .p2align 4 401L(fwdPxQx): 402 .int L(P0Q0)-L(fwdPxQx) /* 0 */ 403 .int L(P1Q0)-L(fwdPxQx) 404 .int L(P2Q0)-L(fwdPxQx) 405 .int L(P3Q0)-L(fwdPxQx) 406 .int L(P4Q0)-L(fwdPxQx) 407 .int L(P5Q0)-L(fwdPxQx) 408 .int L(P6Q0)-L(fwdPxQx) 409 .int L(P7Q0)-L(fwdPxQx) 410 411 .int L(P0Q1)-L(fwdPxQx) /* 8 */ 412 .int L(P1Q1)-L(fwdPxQx) 413 .int L(P2Q1)-L(fwdPxQx) 414 .int L(P3Q1)-L(fwdPxQx) 415 .int L(P4Q1)-L(fwdPxQx) 416 .int L(P5Q1)-L(fwdPxQx) 417 .int L(P6Q1)-L(fwdPxQx) 418 .int L(P7Q1)-L(fwdPxQx) 419 420 .int L(P0Q2)-L(fwdPxQx) /* 16 */ 421 .int L(P1Q2)-L(fwdPxQx) 422 .int L(P2Q2)-L(fwdPxQx) 423 .int L(P3Q2)-L(fwdPxQx) 424 .int L(P4Q2)-L(fwdPxQx) 425 .int L(P5Q2)-L(fwdPxQx) 426 .int L(P6Q2)-L(fwdPxQx) 427 .int L(P7Q2)-L(fwdPxQx) 428 429 .int L(P0Q3)-L(fwdPxQx) /* 24 */ 430 .int L(P1Q3)-L(fwdPxQx) 431 .int L(P2Q3)-L(fwdPxQx) 432 .int L(P3Q3)-L(fwdPxQx) 433 .int L(P4Q3)-L(fwdPxQx) 434 .int L(P5Q3)-L(fwdPxQx) 435 .int L(P6Q3)-L(fwdPxQx) 436 .int L(P7Q3)-L(fwdPxQx) 437 438 .int L(P0Q4)-L(fwdPxQx) /* 32 */ 439 .int L(P1Q4)-L(fwdPxQx) 440 .int L(P2Q4)-L(fwdPxQx) 441 .int L(P3Q4)-L(fwdPxQx) 442 .int L(P4Q4)-L(fwdPxQx) 443 .int L(P5Q4)-L(fwdPxQx) 444 .int L(P6Q4)-L(fwdPxQx) 445 .int L(P7Q4)-L(fwdPxQx) 446 447 .int L(P0Q5)-L(fwdPxQx) /* 40 */ 448 .int L(P1Q5)-L(fwdPxQx) 449 .int L(P2Q5)-L(fwdPxQx) 450 .int L(P3Q5)-L(fwdPxQx) 451 .int L(P4Q5)-L(fwdPxQx) 452 .int L(P5Q5)-L(fwdPxQx) 453 .int L(P6Q5)-L(fwdPxQx) 454 .int L(P7Q5)-L(fwdPxQx) 455 456 .int L(P0Q6)-L(fwdPxQx) /* 48 */ 457 .int L(P1Q6)-L(fwdPxQx) 458 .int L(P2Q6)-L(fwdPxQx) 459 .int L(P3Q6)-L(fwdPxQx) 460 .int L(P4Q6)-L(fwdPxQx) 461 .int L(P5Q6)-L(fwdPxQx) 462 .int L(P6Q6)-L(fwdPxQx) 463 .int L(P7Q6)-L(fwdPxQx) 464 465 .int L(P0Q7)-L(fwdPxQx) /* 56 */ 466 .int L(P1Q7)-L(fwdPxQx) 467 .int L(P2Q7)-L(fwdPxQx) 468 .int L(P3Q7)-L(fwdPxQx) 469 .int L(P4Q7)-L(fwdPxQx) 470 .int L(P5Q7)-L(fwdPxQx) 471 .int L(P6Q7)-L(fwdPxQx) 472 .int L(P7Q7)-L(fwdPxQx) 473 474 .int L(P0Q8)-L(fwdPxQx) /* 64 */ 475 .int L(P1Q8)-L(fwdPxQx) 476 .int L(P2Q8)-L(fwdPxQx) 477 .int L(P3Q8)-L(fwdPxQx) 478 .int L(P4Q8)-L(fwdPxQx) 479 .int L(P5Q8)-L(fwdPxQx) 480 .int L(P6Q8)-L(fwdPxQx) 481 .int L(P7Q8)-L(fwdPxQx) 482 483 .int L(P0Q9)-L(fwdPxQx) /* 72 */ 484 .int L(P1Q9)-L(fwdPxQx) 485 .int L(P2Q9)-L(fwdPxQx) 486 .int L(P3Q9)-L(fwdPxQx) 487 .int L(P4Q9)-L(fwdPxQx) 488 .int L(P5Q9)-L(fwdPxQx) 489 .int L(P6Q9)-L(fwdPxQx) 490 .int L(P7Q9)-L(fwdPxQx) /* 79 */ 491 492 .p2align 4 493L(P0Q9): 494 mov -0x48(%rdi), %rcx 495 mov %rcx, -0x48(%rsi) 496L(P0Q8): 497 mov -0x40(%rdi), %r10 498 mov %r10, -0x40(%rsi) 499L(P0Q7): 500 mov -0x38(%rdi), %r8 501 mov %r8, -0x38(%rsi) 502L(P0Q6): 503 mov -0x30(%rdi), %rcx 504 mov %rcx, -0x30(%rsi) 505L(P0Q5): 506 mov -0x28(%rdi), %r10 507 mov %r10, -0x28(%rsi) 508L(P0Q4): 509 mov -0x20(%rdi), %r8 510 mov %r8, -0x20(%rsi) 511L(P0Q3): 512 mov -0x18(%rdi), %rcx 513 mov %rcx, -0x18(%rsi) 514L(P0Q2): 515 mov -0x10(%rdi), %r10 516 mov %r10, -0x10(%rsi) 517L(P0Q1): 518 mov -0x8(%rdi), %r8 519 mov %r8, -0x8(%rsi) 520L(P0Q0): 521 ret 522 523 .p2align 4 524L(P1Q9): 525 mov -0x49(%rdi), %r8 526 mov %r8, -0x49(%rsi) 527L(P1Q8): 528 mov -0x41(%rdi), %rcx 529 mov %rcx, -0x41(%rsi) 530L(P1Q7): 531 mov -0x39(%rdi), %r10 532 mov %r10, -0x39(%rsi) 533L(P1Q6): 534 mov -0x31(%rdi), %r8 535 mov %r8, -0x31(%rsi) 536L(P1Q5): 537 mov -0x29(%rdi), %rcx 538 mov %rcx, -0x29(%rsi) 539L(P1Q4): 540 mov -0x21(%rdi), %r10 541 mov %r10, -0x21(%rsi) 542L(P1Q3): 543 mov -0x19(%rdi), %r8 544 mov %r8, -0x19(%rsi) 545L(P1Q2): 546 mov -0x11(%rdi), %rcx 547 mov %rcx, -0x11(%rsi) 548L(P1Q1): 549 mov -0x9(%rdi), %r10 550 mov %r10, -0x9(%rsi) 551L(P1Q0): 552 movzbq -0x1(%rdi), %r8 553 mov %r8b, -0x1(%rsi) 554 ret 555 556 .p2align 4 557L(P2Q9): 558 mov -0x4a(%rdi), %r8 559 mov %r8, -0x4a(%rsi) 560L(P2Q8): 561 mov -0x42(%rdi), %rcx 562 mov %rcx, -0x42(%rsi) 563L(P2Q7): 564 mov -0x3a(%rdi), %r10 565 mov %r10, -0x3a(%rsi) 566L(P2Q6): 567 mov -0x32(%rdi), %r8 568 mov %r8, -0x32(%rsi) 569L(P2Q5): 570 mov -0x2a(%rdi), %rcx 571 mov %rcx, -0x2a(%rsi) 572L(P2Q4): 573 mov -0x22(%rdi), %r10 574 mov %r10, -0x22(%rsi) 575L(P2Q3): 576 mov -0x1a(%rdi), %r8 577 mov %r8, -0x1a(%rsi) 578L(P2Q2): 579 mov -0x12(%rdi), %rcx 580 mov %rcx, -0x12(%rsi) 581L(P2Q1): 582 mov -0xa(%rdi), %r10 583 mov %r10, -0xa(%rsi) 584L(P2Q0): 585 movzwq -0x2(%rdi), %r8 586 mov %r8w, -0x2(%rsi) 587 ret 588 589 .p2align 4 590L(P3Q9): 591 mov -0x4b(%rdi), %r8 592 mov %r8, -0x4b(%rsi) 593L(P3Q8): 594 mov -0x43(%rdi), %rcx 595 mov %rcx, -0x43(%rsi) 596L(P3Q7): 597 mov -0x3b(%rdi), %r10 598 mov %r10, -0x3b(%rsi) 599L(P3Q6): 600 mov -0x33(%rdi), %r8 601 mov %r8, -0x33(%rsi) 602L(P3Q5): 603 mov -0x2b(%rdi), %rcx 604 mov %rcx, -0x2b(%rsi) 605L(P3Q4): 606 mov -0x23(%rdi), %r10 607 mov %r10, -0x23(%rsi) 608L(P3Q3): 609 mov -0x1b(%rdi), %r8 610 mov %r8, -0x1b(%rsi) 611L(P3Q2): 612 mov -0x13(%rdi), %rcx 613 mov %rcx, -0x13(%rsi) 614L(P3Q1): 615 mov -0xb(%rdi), %r10 616 mov %r10, -0xb(%rsi) 617 /* 618 * These trailing loads/stores have to do all their loads 1st, 619 * then do the stores. 620 */ 621L(P3Q0): 622 movzwq -0x3(%rdi), %r8 623 movzbq -0x1(%rdi), %r10 624 mov %r8w, -0x3(%rsi) 625 mov %r10b, -0x1(%rsi) 626 ret 627 628 .p2align 4 629L(P4Q9): 630 mov -0x4c(%rdi), %r8 631 mov %r8, -0x4c(%rsi) 632L(P4Q8): 633 mov -0x44(%rdi), %rcx 634 mov %rcx, -0x44(%rsi) 635L(P4Q7): 636 mov -0x3c(%rdi), %r10 637 mov %r10, -0x3c(%rsi) 638L(P4Q6): 639 mov -0x34(%rdi), %r8 640 mov %r8, -0x34(%rsi) 641L(P4Q5): 642 mov -0x2c(%rdi), %rcx 643 mov %rcx, -0x2c(%rsi) 644L(P4Q4): 645 mov -0x24(%rdi), %r10 646 mov %r10, -0x24(%rsi) 647L(P4Q3): 648 mov -0x1c(%rdi), %r8 649 mov %r8, -0x1c(%rsi) 650L(P4Q2): 651 mov -0x14(%rdi), %rcx 652 mov %rcx, -0x14(%rsi) 653L(P4Q1): 654 mov -0xc(%rdi), %r10 655 mov %r10, -0xc(%rsi) 656L(P4Q0): 657 mov -0x4(%rdi), %r8d 658 mov %r8d, -0x4(%rsi) 659 ret 660 661 .p2align 4 662L(P5Q9): 663 mov -0x4d(%rdi), %r8 664 mov %r8, -0x4d(%rsi) 665L(P5Q8): 666 mov -0x45(%rdi), %rcx 667 mov %rcx, -0x45(%rsi) 668L(P5Q7): 669 mov -0x3d(%rdi), %r10 670 mov %r10, -0x3d(%rsi) 671L(P5Q6): 672 mov -0x35(%rdi), %r8 673 mov %r8, -0x35(%rsi) 674L(P5Q5): 675 mov -0x2d(%rdi), %rcx 676 mov %rcx, -0x2d(%rsi) 677L(P5Q4): 678 mov -0x25(%rdi), %r10 679 mov %r10, -0x25(%rsi) 680L(P5Q3): 681 mov -0x1d(%rdi), %r8 682 mov %r8, -0x1d(%rsi) 683L(P5Q2): 684 mov -0x15(%rdi), %rcx 685 mov %rcx, -0x15(%rsi) 686L(P5Q1): 687 mov -0xd(%rdi), %r10 688 mov %r10, -0xd(%rsi) 689L(P5Q0): 690 mov -0x5(%rdi), %r8d 691 movzbq -0x1(%rdi), %r10 692 mov %r8d, -0x5(%rsi) 693 mov %r10b, -0x1(%rsi) 694 ret 695 696 .p2align 4 697L(P6Q9): 698 mov -0x4e(%rdi), %r8 699 mov %r8, -0x4e(%rsi) 700L(P6Q8): 701 mov -0x46(%rdi), %rcx 702 mov %rcx, -0x46(%rsi) 703L(P6Q7): 704 mov -0x3e(%rdi), %r10 705 mov %r10, -0x3e(%rsi) 706L(P6Q6): 707 mov -0x36(%rdi), %r8 708 mov %r8, -0x36(%rsi) 709L(P6Q5): 710 mov -0x2e(%rdi), %rcx 711 mov %rcx, -0x2e(%rsi) 712L(P6Q4): 713 mov -0x26(%rdi), %r10 714 mov %r10, -0x26(%rsi) 715L(P6Q3): 716 mov -0x1e(%rdi), %r8 717 mov %r8, -0x1e(%rsi) 718L(P6Q2): 719 mov -0x16(%rdi), %rcx 720 mov %rcx, -0x16(%rsi) 721L(P6Q1): 722 mov -0xe(%rdi), %r10 723 mov %r10, -0xe(%rsi) 724L(P6Q0): 725 mov -0x6(%rdi), %r8d 726 movzwq -0x2(%rdi), %r10 727 mov %r8d, -0x6(%rsi) 728 mov %r10w, -0x2(%rsi) 729 ret 730 731 .p2align 4 732L(P7Q9): 733 mov -0x4f(%rdi), %r8 734 mov %r8, -0x4f(%rsi) 735L(P7Q8): 736 mov -0x47(%rdi), %rcx 737 mov %rcx, -0x47(%rsi) 738L(P7Q7): 739 mov -0x3f(%rdi), %r10 740 mov %r10, -0x3f(%rsi) 741L(P7Q6): 742 mov -0x37(%rdi), %r8 743 mov %r8, -0x37(%rsi) 744L(P7Q5): 745 mov -0x2f(%rdi), %rcx 746 mov %rcx, -0x2f(%rsi) 747L(P7Q4): 748 mov -0x27(%rdi), %r10 749 mov %r10, -0x27(%rsi) 750L(P7Q3): 751 mov -0x1f(%rdi), %r8 752 mov %r8, -0x1f(%rsi) 753L(P7Q2): 754 mov -0x17(%rdi), %rcx 755 mov %rcx, -0x17(%rsi) 756L(P7Q1): 757 mov -0xf(%rdi), %r10 758 mov %r10, -0xf(%rsi) 759L(P7Q0): 760 mov -0x7(%rdi), %r8d 761 movzwq -0x3(%rdi), %r10 762 movzbq -0x1(%rdi), %rcx 763 mov %r8d, -0x7(%rsi) 764 mov %r10w, -0x3(%rsi) 765 mov %cl, -0x1(%rsi) 766 ret 767 768 /* 769 * For large sizes rep smovq is fastest. 770 * Transition point determined experimentally as measured on 771 * Intel Xeon processors (incl. Nehalem and previous generations) and 772 * AMD Opteron. The transition value is patched at boot time to avoid 773 * memory reference hit. 774 */ 775 .globl bcopy_patch_start 776bcopy_patch_start: 777 cmpq $BCOPY_NHM_REP, %rdx 778 .globl bcopy_patch_end 779bcopy_patch_end: 780 781 .p2align 4 782 .globl bcopy_ck_size 783bcopy_ck_size: 784 cmpq $BCOPY_DFLT_REP, %rdx 785 jae L(use_rep) 786 787 /* 788 * Align to a 8-byte boundary. Avoids penalties from unaligned stores 789 * as well as from stores spanning cachelines. 790 */ 791 test $0x7, %rsi 792 jz L(aligned_loop) 793 test $0x1, %rsi 794 jz 2f 795 movzbq (%rdi), %r8 796 dec %rdx 797 inc %rdi 798 mov %r8b, (%rsi) 799 inc %rsi 8002: 801 test $0x2, %rsi 802 jz 4f 803 movzwq (%rdi), %r8 804 sub $0x2, %rdx 805 add $0x2, %rdi 806 mov %r8w, (%rsi) 807 add $0x2, %rsi 8084: 809 test $0x4, %rsi 810 jz L(aligned_loop) 811 mov (%rdi), %r8d 812 sub $0x4, %rdx 813 add $0x4, %rdi 814 mov %r8d, (%rsi) 815 add $0x4, %rsi 816 817 /* 818 * Copy 64-bytes per loop 819 */ 820 .p2align 4 821L(aligned_loop): 822 mov (%rdi), %r8 823 mov 0x8(%rdi), %r10 824 lea -0x40(%rdx), %rdx 825 mov %r8, (%rsi) 826 mov %r10, 0x8(%rsi) 827 mov 0x10(%rdi), %rcx 828 mov 0x18(%rdi), %r8 829 mov %rcx, 0x10(%rsi) 830 mov %r8, 0x18(%rsi) 831 832 cmp $0x40, %rdx 833 mov 0x20(%rdi), %r10 834 mov 0x28(%rdi), %rcx 835 mov %r10, 0x20(%rsi) 836 mov %rcx, 0x28(%rsi) 837 mov 0x30(%rdi), %r8 838 mov 0x38(%rdi), %r10 839 lea 0x40(%rdi), %rdi 840 mov %r8, 0x30(%rsi) 841 mov %r10, 0x38(%rsi) 842 lea 0x40(%rsi), %rsi 843 jae L(aligned_loop) 844 845 /* 846 * Copy remaining bytes (0-63) 847 */ 848L(do_remainder): 849 leaq L(fwdPxQx)(%rip), %r10 850 addq %rdx, %rdi 851 addq %rdx, %rsi 852 movslq (%r10,%rdx,4), %rcx 853 leaq (%rcx,%r10,1), %r10 854 jmpq *%r10 855 856 /* 857 * Use rep smovq. Clear remainder via unrolled code 858 */ 859 .p2align 4 860L(use_rep): 861 xchgq %rdi, %rsi /* %rsi = source, %rdi = destination */ 862 movq %rdx, %rcx /* %rcx = count */ 863 shrq $3, %rcx /* 8-byte word count */ 864 rep 865 smovq 866 867 xchgq %rsi, %rdi /* %rdi = src, %rsi = destination */ 868 andq $7, %rdx /* remainder */ 869 jnz L(do_remainder) 870 ret 871#undef L 872 873#ifdef DEBUG 874 /* 875 * Setup frame on the run-time stack. The end of the input argument 876 * area must be aligned on a 16 byte boundary. The stack pointer %rsp, 877 * always points to the end of the latest allocated stack frame. 878 * panic(const char *format, ...) is a varargs function. When a 879 * function taking variable arguments is called, %rax must be set 880 * to eight times the number of floating point parameters passed 881 * to the function in SSE registers. 882 */ 883call_panic: 884 pushq %rbp /* align stack properly */ 885 movq %rsp, %rbp 886 xorl %eax, %eax /* no variable arguments */ 887 call panic /* %rdi = format string */ 888#endif 889 SET_SIZE(bcopy_altentry) 890 SET_SIZE(bcopy) 891 892#elif defined(__i386) 893 894#define ARG_FROM 4 895#define ARG_TO 8 896#define ARG_COUNT 12 897 898 ENTRY(bcopy) 899#ifdef DEBUG 900 movl ARG_COUNT(%esp), %eax 901 orl %eax, %eax 902 jz 1f 903 movl postbootkernelbase, %eax 904 cmpl %eax, ARG_FROM(%esp) 905 jb 0f 906 cmpl %eax, ARG_TO(%esp) 907 jnb 1f 9080: pushl %ebp 909 movl %esp, %ebp 910 pushl $.bcopy_panic_msg 911 call panic 9121: 913#endif 914do_copy: 915 movl %esi, %eax /* save registers */ 916 movl %edi, %edx 917 movl ARG_COUNT(%esp), %ecx 918 movl ARG_FROM(%esp), %esi 919 movl ARG_TO(%esp), %edi 920 921 shrl $2, %ecx /* word count */ 922 rep 923 smovl 924 movl ARG_COUNT(%esp), %ecx 925 andl $3, %ecx /* bytes left over */ 926 rep 927 smovb 928 movl %eax, %esi /* restore registers */ 929 movl %edx, %edi 930 ret 931 SET_SIZE(bcopy) 932 933#undef ARG_COUNT 934#undef ARG_FROM 935#undef ARG_TO 936 937#endif /* __i386 */ 938#endif /* __lint */ 939 940 941/* 942 * Zero a block of storage, returning an error code if we 943 * take a kernel pagefault which cannot be resolved. 944 * Returns errno value on pagefault error, 0 if all ok 945 */ 946 947#if defined(__lint) 948 949/* ARGSUSED */ 950int 951kzero(void *addr, size_t count) 952{ return (0); } 953 954#else /* __lint */ 955 956#if defined(__amd64) 957 958 ENTRY(kzero) 959#ifdef DEBUG 960 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */ 961 jnb 0f 962 leaq .kzero_panic_msg(%rip), %rdi 963 jmp call_panic /* setup stack and call panic */ 9640: 965#endif 966 /* 967 * pass lofault value as 3rd argument for fault return 968 */ 969 leaq _kzeroerr(%rip), %rdx 970 971 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */ 972 movq T_LOFAULT(%r9), %r11 /* save the current lofault */ 973 movq %rdx, T_LOFAULT(%r9) /* new lofault */ 974 call bzero_altentry 975 xorl %eax, %eax 976 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */ 977 ret 978 /* 979 * A fault during bzero is indicated through an errno value 980 * in %rax when we iretq to here. 981 */ 982_kzeroerr: 983 addq $8, %rsp /* pop bzero_altentry call ret addr */ 984 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */ 985 ret 986 SET_SIZE(kzero) 987 988#elif defined(__i386) 989 990#define ARG_ADDR 8 991#define ARG_COUNT 12 992 993 ENTRY(kzero) 994#ifdef DEBUG 995 pushl %ebp 996 movl %esp, %ebp 997 movl postbootkernelbase, %eax 998 cmpl %eax, ARG_ADDR(%ebp) 999 jnb 0f 1000 pushl $.kzero_panic_msg 1001 call panic 10020: popl %ebp 1003#endif 1004 lea _kzeroerr, %eax /* kzeroerr is lofault value */ 1005 1006 pushl %ebp /* save stack base */ 1007 movl %esp, %ebp /* set new stack base */ 1008 pushl %edi /* save %edi */ 1009 1010 mov %gs:CPU_THREAD, %edx 1011 movl T_LOFAULT(%edx), %edi 1012 pushl %edi /* save the current lofault */ 1013 movl %eax, T_LOFAULT(%edx) /* new lofault */ 1014 1015 movl ARG_COUNT(%ebp), %ecx /* get size in bytes */ 1016 movl ARG_ADDR(%ebp), %edi /* %edi <- address of bytes to clear */ 1017 shrl $2, %ecx /* Count of double words to zero */ 1018 xorl %eax, %eax /* sstol val */ 1019 rep 1020 sstol /* %ecx contains words to clear (%eax=0) */ 1021 1022 movl ARG_COUNT(%ebp), %ecx /* get size in bytes */ 1023 andl $3, %ecx /* do mod 4 */ 1024 rep 1025 sstob /* %ecx contains residual bytes to clear */ 1026 1027 /* 1028 * A fault during kzero is indicated through an errno value 1029 * in %eax when we iret to here. 1030 */ 1031_kzeroerr: 1032 popl %edi 1033 movl %edi, T_LOFAULT(%edx) /* restore the original lofault */ 1034 popl %edi 1035 popl %ebp 1036 ret 1037 SET_SIZE(kzero) 1038 1039#undef ARG_ADDR 1040#undef ARG_COUNT 1041 1042#endif /* __i386 */ 1043#endif /* __lint */ 1044 1045/* 1046 * Zero a block of storage. 1047 */ 1048 1049#if defined(__lint) 1050 1051/* ARGSUSED */ 1052void 1053bzero(void *addr, size_t count) 1054{} 1055 1056#else /* __lint */ 1057 1058#if defined(__amd64) 1059 1060 ENTRY(bzero) 1061#ifdef DEBUG 1062 cmpq postbootkernelbase(%rip), %rdi /* %rdi = addr */ 1063 jnb 0f 1064 leaq .bzero_panic_msg(%rip), %rdi 1065 jmp call_panic /* setup stack and call panic */ 10660: 1067#endif 1068 ALTENTRY(bzero_altentry) 1069do_zero: 1070#define L(s) .bzero/**/s 1071 xorl %eax, %eax 1072 1073 cmpq $0x50, %rsi /* 80 */ 1074 jae L(ck_align) 1075 1076 /* 1077 * Performance data shows many caller's are zeroing small buffers. So 1078 * for best perf for these sizes unrolled code is used. Store zeros 1079 * without worrying about alignment. 1080 */ 1081 leaq L(setPxQx)(%rip), %r10 1082 addq %rsi, %rdi 1083 movslq (%r10,%rsi,4), %rcx 1084 leaq (%rcx,%r10,1), %r10 1085 jmpq *%r10 1086 1087 .p2align 4 1088L(setPxQx): 1089 .int L(P0Q0)-L(setPxQx) /* 0 */ 1090 .int L(P1Q0)-L(setPxQx) 1091 .int L(P2Q0)-L(setPxQx) 1092 .int L(P3Q0)-L(setPxQx) 1093 .int L(P4Q0)-L(setPxQx) 1094 .int L(P5Q0)-L(setPxQx) 1095 .int L(P6Q0)-L(setPxQx) 1096 .int L(P7Q0)-L(setPxQx) 1097 1098 .int L(P0Q1)-L(setPxQx) /* 8 */ 1099 .int L(P1Q1)-L(setPxQx) 1100 .int L(P2Q1)-L(setPxQx) 1101 .int L(P3Q1)-L(setPxQx) 1102 .int L(P4Q1)-L(setPxQx) 1103 .int L(P5Q1)-L(setPxQx) 1104 .int L(P6Q1)-L(setPxQx) 1105 .int L(P7Q1)-L(setPxQx) 1106 1107 .int L(P0Q2)-L(setPxQx) /* 16 */ 1108 .int L(P1Q2)-L(setPxQx) 1109 .int L(P2Q2)-L(setPxQx) 1110 .int L(P3Q2)-L(setPxQx) 1111 .int L(P4Q2)-L(setPxQx) 1112 .int L(P5Q2)-L(setPxQx) 1113 .int L(P6Q2)-L(setPxQx) 1114 .int L(P7Q2)-L(setPxQx) 1115 1116 .int L(P0Q3)-L(setPxQx) /* 24 */ 1117 .int L(P1Q3)-L(setPxQx) 1118 .int L(P2Q3)-L(setPxQx) 1119 .int L(P3Q3)-L(setPxQx) 1120 .int L(P4Q3)-L(setPxQx) 1121 .int L(P5Q3)-L(setPxQx) 1122 .int L(P6Q3)-L(setPxQx) 1123 .int L(P7Q3)-L(setPxQx) 1124 1125 .int L(P0Q4)-L(setPxQx) /* 32 */ 1126 .int L(P1Q4)-L(setPxQx) 1127 .int L(P2Q4)-L(setPxQx) 1128 .int L(P3Q4)-L(setPxQx) 1129 .int L(P4Q4)-L(setPxQx) 1130 .int L(P5Q4)-L(setPxQx) 1131 .int L(P6Q4)-L(setPxQx) 1132 .int L(P7Q4)-L(setPxQx) 1133 1134 .int L(P0Q5)-L(setPxQx) /* 40 */ 1135 .int L(P1Q5)-L(setPxQx) 1136 .int L(P2Q5)-L(setPxQx) 1137 .int L(P3Q5)-L(setPxQx) 1138 .int L(P4Q5)-L(setPxQx) 1139 .int L(P5Q5)-L(setPxQx) 1140 .int L(P6Q5)-L(setPxQx) 1141 .int L(P7Q5)-L(setPxQx) 1142 1143 .int L(P0Q6)-L(setPxQx) /* 48 */ 1144 .int L(P1Q6)-L(setPxQx) 1145 .int L(P2Q6)-L(setPxQx) 1146 .int L(P3Q6)-L(setPxQx) 1147 .int L(P4Q6)-L(setPxQx) 1148 .int L(P5Q6)-L(setPxQx) 1149 .int L(P6Q6)-L(setPxQx) 1150 .int L(P7Q6)-L(setPxQx) 1151 1152 .int L(P0Q7)-L(setPxQx) /* 56 */ 1153 .int L(P1Q7)-L(setPxQx) 1154 .int L(P2Q7)-L(setPxQx) 1155 .int L(P3Q7)-L(setPxQx) 1156 .int L(P4Q7)-L(setPxQx) 1157 .int L(P5Q7)-L(setPxQx) 1158 .int L(P6Q7)-L(setPxQx) 1159 .int L(P7Q7)-L(setPxQx) 1160 1161 .int L(P0Q8)-L(setPxQx) /* 64 */ 1162 .int L(P1Q8)-L(setPxQx) 1163 .int L(P2Q8)-L(setPxQx) 1164 .int L(P3Q8)-L(setPxQx) 1165 .int L(P4Q8)-L(setPxQx) 1166 .int L(P5Q8)-L(setPxQx) 1167 .int L(P6Q8)-L(setPxQx) 1168 .int L(P7Q8)-L(setPxQx) 1169 1170 .int L(P0Q9)-L(setPxQx) /* 72 */ 1171 .int L(P1Q9)-L(setPxQx) 1172 .int L(P2Q9)-L(setPxQx) 1173 .int L(P3Q9)-L(setPxQx) 1174 .int L(P4Q9)-L(setPxQx) 1175 .int L(P5Q9)-L(setPxQx) 1176 .int L(P6Q9)-L(setPxQx) 1177 .int L(P7Q9)-L(setPxQx) /* 79 */ 1178 1179 .p2align 4 1180L(P0Q9): mov %rax, -0x48(%rdi) 1181L(P0Q8): mov %rax, -0x40(%rdi) 1182L(P0Q7): mov %rax, -0x38(%rdi) 1183L(P0Q6): mov %rax, -0x30(%rdi) 1184L(P0Q5): mov %rax, -0x28(%rdi) 1185L(P0Q4): mov %rax, -0x20(%rdi) 1186L(P0Q3): mov %rax, -0x18(%rdi) 1187L(P0Q2): mov %rax, -0x10(%rdi) 1188L(P0Q1): mov %rax, -0x8(%rdi) 1189L(P0Q0): 1190 ret 1191 1192 .p2align 4 1193L(P1Q9): mov %rax, -0x49(%rdi) 1194L(P1Q8): mov %rax, -0x41(%rdi) 1195L(P1Q7): mov %rax, -0x39(%rdi) 1196L(P1Q6): mov %rax, -0x31(%rdi) 1197L(P1Q5): mov %rax, -0x29(%rdi) 1198L(P1Q4): mov %rax, -0x21(%rdi) 1199L(P1Q3): mov %rax, -0x19(%rdi) 1200L(P1Q2): mov %rax, -0x11(%rdi) 1201L(P1Q1): mov %rax, -0x9(%rdi) 1202L(P1Q0): mov %al, -0x1(%rdi) 1203 ret 1204 1205 .p2align 4 1206L(P2Q9): mov %rax, -0x4a(%rdi) 1207L(P2Q8): mov %rax, -0x42(%rdi) 1208L(P2Q7): mov %rax, -0x3a(%rdi) 1209L(P2Q6): mov %rax, -0x32(%rdi) 1210L(P2Q5): mov %rax, -0x2a(%rdi) 1211L(P2Q4): mov %rax, -0x22(%rdi) 1212L(P2Q3): mov %rax, -0x1a(%rdi) 1213L(P2Q2): mov %rax, -0x12(%rdi) 1214L(P2Q1): mov %rax, -0xa(%rdi) 1215L(P2Q0): mov %ax, -0x2(%rdi) 1216 ret 1217 1218 .p2align 4 1219L(P3Q9): mov %rax, -0x4b(%rdi) 1220L(P3Q8): mov %rax, -0x43(%rdi) 1221L(P3Q7): mov %rax, -0x3b(%rdi) 1222L(P3Q6): mov %rax, -0x33(%rdi) 1223L(P3Q5): mov %rax, -0x2b(%rdi) 1224L(P3Q4): mov %rax, -0x23(%rdi) 1225L(P3Q3): mov %rax, -0x1b(%rdi) 1226L(P3Q2): mov %rax, -0x13(%rdi) 1227L(P3Q1): mov %rax, -0xb(%rdi) 1228L(P3Q0): mov %ax, -0x3(%rdi) 1229 mov %al, -0x1(%rdi) 1230 ret 1231 1232 .p2align 4 1233L(P4Q9): mov %rax, -0x4c(%rdi) 1234L(P4Q8): mov %rax, -0x44(%rdi) 1235L(P4Q7): mov %rax, -0x3c(%rdi) 1236L(P4Q6): mov %rax, -0x34(%rdi) 1237L(P4Q5): mov %rax, -0x2c(%rdi) 1238L(P4Q4): mov %rax, -0x24(%rdi) 1239L(P4Q3): mov %rax, -0x1c(%rdi) 1240L(P4Q2): mov %rax, -0x14(%rdi) 1241L(P4Q1): mov %rax, -0xc(%rdi) 1242L(P4Q0): mov %eax, -0x4(%rdi) 1243 ret 1244 1245 .p2align 4 1246L(P5Q9): mov %rax, -0x4d(%rdi) 1247L(P5Q8): mov %rax, -0x45(%rdi) 1248L(P5Q7): mov %rax, -0x3d(%rdi) 1249L(P5Q6): mov %rax, -0x35(%rdi) 1250L(P5Q5): mov %rax, -0x2d(%rdi) 1251L(P5Q4): mov %rax, -0x25(%rdi) 1252L(P5Q3): mov %rax, -0x1d(%rdi) 1253L(P5Q2): mov %rax, -0x15(%rdi) 1254L(P5Q1): mov %rax, -0xd(%rdi) 1255L(P5Q0): mov %eax, -0x5(%rdi) 1256 mov %al, -0x1(%rdi) 1257 ret 1258 1259 .p2align 4 1260L(P6Q9): mov %rax, -0x4e(%rdi) 1261L(P6Q8): mov %rax, -0x46(%rdi) 1262L(P6Q7): mov %rax, -0x3e(%rdi) 1263L(P6Q6): mov %rax, -0x36(%rdi) 1264L(P6Q5): mov %rax, -0x2e(%rdi) 1265L(P6Q4): mov %rax, -0x26(%rdi) 1266L(P6Q3): mov %rax, -0x1e(%rdi) 1267L(P6Q2): mov %rax, -0x16(%rdi) 1268L(P6Q1): mov %rax, -0xe(%rdi) 1269L(P6Q0): mov %eax, -0x6(%rdi) 1270 mov %ax, -0x2(%rdi) 1271 ret 1272 1273 .p2align 4 1274L(P7Q9): mov %rax, -0x4f(%rdi) 1275L(P7Q8): mov %rax, -0x47(%rdi) 1276L(P7Q7): mov %rax, -0x3f(%rdi) 1277L(P7Q6): mov %rax, -0x37(%rdi) 1278L(P7Q5): mov %rax, -0x2f(%rdi) 1279L(P7Q4): mov %rax, -0x27(%rdi) 1280L(P7Q3): mov %rax, -0x1f(%rdi) 1281L(P7Q2): mov %rax, -0x17(%rdi) 1282L(P7Q1): mov %rax, -0xf(%rdi) 1283L(P7Q0): mov %eax, -0x7(%rdi) 1284 mov %ax, -0x3(%rdi) 1285 mov %al, -0x1(%rdi) 1286 ret 1287 1288 /* 1289 * Align to a 16-byte boundary. Avoids penalties from unaligned stores 1290 * as well as from stores spanning cachelines. Note 16-byte alignment 1291 * is better in case where rep sstosq is used. 1292 */ 1293 .p2align 4 1294L(ck_align): 1295 test $0xf, %rdi 1296 jz L(aligned_now) 1297 test $1, %rdi 1298 jz 2f 1299 mov %al, (%rdi) 1300 dec %rsi 1301 lea 1(%rdi),%rdi 13022: 1303 test $2, %rdi 1304 jz 4f 1305 mov %ax, (%rdi) 1306 sub $2, %rsi 1307 lea 2(%rdi),%rdi 13084: 1309 test $4, %rdi 1310 jz 8f 1311 mov %eax, (%rdi) 1312 sub $4, %rsi 1313 lea 4(%rdi),%rdi 13148: 1315 test $8, %rdi 1316 jz L(aligned_now) 1317 mov %rax, (%rdi) 1318 sub $8, %rsi 1319 lea 8(%rdi),%rdi 1320 1321 /* 1322 * For large sizes rep sstoq is fastest. 1323 * Transition point determined experimentally as measured on 1324 * Intel Xeon processors (incl. Nehalem) and AMD Opteron. 1325 */ 1326L(aligned_now): 1327 cmp $BZERO_USE_REP, %rsi 1328 ja L(use_rep) 1329 1330 /* 1331 * zero 64-bytes per loop 1332 */ 1333 .p2align 4 1334L(bzero_loop): 1335 leaq -0x40(%rsi), %rsi 1336 cmpq $0x40, %rsi 1337 movq %rax, (%rdi) 1338 movq %rax, 0x8(%rdi) 1339 movq %rax, 0x10(%rdi) 1340 movq %rax, 0x18(%rdi) 1341 movq %rax, 0x20(%rdi) 1342 movq %rax, 0x28(%rdi) 1343 movq %rax, 0x30(%rdi) 1344 movq %rax, 0x38(%rdi) 1345 leaq 0x40(%rdi), %rdi 1346 jae L(bzero_loop) 1347 1348 /* 1349 * Clear any remaining bytes.. 1350 */ 13519: 1352 leaq L(setPxQx)(%rip), %r10 1353 addq %rsi, %rdi 1354 movslq (%r10,%rsi,4), %rcx 1355 leaq (%rcx,%r10,1), %r10 1356 jmpq *%r10 1357 1358 /* 1359 * Use rep sstoq. Clear any remainder via unrolled code 1360 */ 1361 .p2align 4 1362L(use_rep): 1363 movq %rsi, %rcx /* get size in bytes */ 1364 shrq $3, %rcx /* count of 8-byte words to zero */ 1365 rep 1366 sstoq /* %rcx = words to clear (%rax=0) */ 1367 andq $7, %rsi /* remaining bytes */ 1368 jnz 9b 1369 ret 1370#undef L 1371 SET_SIZE(bzero_altentry) 1372 SET_SIZE(bzero) 1373 1374#elif defined(__i386) 1375 1376#define ARG_ADDR 4 1377#define ARG_COUNT 8 1378 1379 ENTRY(bzero) 1380#ifdef DEBUG 1381 movl postbootkernelbase, %eax 1382 cmpl %eax, ARG_ADDR(%esp) 1383 jnb 0f 1384 pushl %ebp 1385 movl %esp, %ebp 1386 pushl $.bzero_panic_msg 1387 call panic 13880: 1389#endif 1390do_zero: 1391 movl %edi, %edx 1392 movl ARG_COUNT(%esp), %ecx 1393 movl ARG_ADDR(%esp), %edi 1394 shrl $2, %ecx 1395 xorl %eax, %eax 1396 rep 1397 sstol 1398 movl ARG_COUNT(%esp), %ecx 1399 andl $3, %ecx 1400 rep 1401 sstob 1402 movl %edx, %edi 1403 ret 1404 SET_SIZE(bzero) 1405 1406#undef ARG_ADDR 1407#undef ARG_COUNT 1408 1409#endif /* __i386 */ 1410#endif /* __lint */ 1411 1412/* 1413 * Transfer data to and from user space - 1414 * Note that these routines can cause faults 1415 * It is assumed that the kernel has nothing at 1416 * less than KERNELBASE in the virtual address space. 1417 * 1418 * Note that copyin(9F) and copyout(9F) are part of the 1419 * DDI/DKI which specifies that they return '-1' on "errors." 1420 * 1421 * Sigh. 1422 * 1423 * So there's two extremely similar routines - xcopyin_nta() and 1424 * xcopyout_nta() which return the errno that we've faithfully computed. 1425 * This allows other callers (e.g. uiomove(9F)) to work correctly. 1426 * Given that these are used pretty heavily, we expand the calling 1427 * sequences inline for all flavours (rather than making wrappers). 1428 */ 1429 1430/* 1431 * Copy user data to kernel space. 1432 */ 1433 1434#if defined(__lint) 1435 1436/* ARGSUSED */ 1437int 1438copyin(const void *uaddr, void *kaddr, size_t count) 1439{ return (0); } 1440 1441#else /* lint */ 1442 1443#if defined(__amd64) 1444 1445 ENTRY(copyin) 1446 pushq %rbp 1447 movq %rsp, %rbp 1448 subq $24, %rsp 1449 1450 /* 1451 * save args in case we trap and need to rerun as a copyop 1452 */ 1453 movq %rdi, (%rsp) 1454 movq %rsi, 0x8(%rsp) 1455 movq %rdx, 0x10(%rsp) 1456 1457 movq kernelbase(%rip), %rax 1458#ifdef DEBUG 1459 cmpq %rax, %rsi /* %rsi = kaddr */ 1460 jnb 1f 1461 leaq .copyin_panic_msg(%rip), %rdi 1462 xorl %eax, %eax 1463 call panic 14641: 1465#endif 1466 /* 1467 * pass lofault value as 4th argument to do_copy_fault 1468 */ 1469 leaq _copyin_err(%rip), %rcx 1470 1471 movq %gs:CPU_THREAD, %r9 1472 cmpq %rax, %rdi /* test uaddr < kernelbase */ 1473 jb do_copy_fault 1474 jmp 3f 1475 1476_copyin_err: 1477 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 1478 addq $8, %rsp /* pop bcopy_altentry call ret addr */ 14793: 1480 movq T_COPYOPS(%r9), %rax 1481 cmpq $0, %rax 1482 jz 2f 1483 /* 1484 * reload args for the copyop 1485 */ 1486 movq (%rsp), %rdi 1487 movq 0x8(%rsp), %rsi 1488 movq 0x10(%rsp), %rdx 1489 leave 1490 jmp *CP_COPYIN(%rax) 1491 14922: movl $-1, %eax 1493 leave 1494 ret 1495 SET_SIZE(copyin) 1496 1497#elif defined(__i386) 1498 1499#define ARG_UADDR 4 1500#define ARG_KADDR 8 1501 1502 ENTRY(copyin) 1503 movl kernelbase, %ecx 1504#ifdef DEBUG 1505 cmpl %ecx, ARG_KADDR(%esp) 1506 jnb 1f 1507 pushl %ebp 1508 movl %esp, %ebp 1509 pushl $.copyin_panic_msg 1510 call panic 15111: 1512#endif 1513 lea _copyin_err, %eax 1514 1515 movl %gs:CPU_THREAD, %edx 1516 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */ 1517 jb do_copy_fault 1518 jmp 3f 1519 1520_copyin_err: 1521 popl %ecx 1522 popl %edi 1523 movl %ecx, T_LOFAULT(%edx) /* restore original lofault */ 1524 popl %esi 1525 popl %ebp 15263: 1527 movl T_COPYOPS(%edx), %eax 1528 cmpl $0, %eax 1529 jz 2f 1530 jmp *CP_COPYIN(%eax) 1531 15322: movl $-1, %eax 1533 ret 1534 SET_SIZE(copyin) 1535 1536#undef ARG_UADDR 1537#undef ARG_KADDR 1538 1539#endif /* __i386 */ 1540#endif /* __lint */ 1541 1542#if defined(__lint) 1543 1544/* ARGSUSED */ 1545int 1546xcopyin_nta(const void *uaddr, void *kaddr, size_t count, int copy_cached) 1547{ return (0); } 1548 1549#else /* __lint */ 1550 1551#if defined(__amd64) 1552 1553 ENTRY(xcopyin_nta) 1554 pushq %rbp 1555 movq %rsp, %rbp 1556 subq $24, %rsp 1557 1558 /* 1559 * save args in case we trap and need to rerun as a copyop 1560 * %rcx is consumed in this routine so we don't need to save 1561 * it. 1562 */ 1563 movq %rdi, (%rsp) 1564 movq %rsi, 0x8(%rsp) 1565 movq %rdx, 0x10(%rsp) 1566 1567 movq kernelbase(%rip), %rax 1568#ifdef DEBUG 1569 cmpq %rax, %rsi /* %rsi = kaddr */ 1570 jnb 1f 1571 leaq .xcopyin_panic_msg(%rip), %rdi 1572 xorl %eax, %eax 1573 call panic 15741: 1575#endif 1576 movq %gs:CPU_THREAD, %r9 1577 cmpq %rax, %rdi /* test uaddr < kernelbase */ 1578 jae 4f 1579 cmpq $0, %rcx /* No non-temporal access? */ 1580 /* 1581 * pass lofault value as 4th argument to do_copy_fault 1582 */ 1583 leaq _xcopyin_err(%rip), %rcx /* doesn't set rflags */ 1584 jnz do_copy_fault /* use regular access */ 1585 /* 1586 * Make sure cnt is >= XCOPY_MIN_SIZE bytes 1587 */ 1588 cmpq $XCOPY_MIN_SIZE, %rdx 1589 jb do_copy_fault 1590 1591 /* 1592 * Make sure src and dst are NTA_ALIGN_SIZE aligned, 1593 * count is COUNT_ALIGN_SIZE aligned. 1594 */ 1595 movq %rdi, %r10 1596 orq %rsi, %r10 1597 andq $NTA_ALIGN_MASK, %r10 1598 orq %rdx, %r10 1599 andq $COUNT_ALIGN_MASK, %r10 1600 jnz do_copy_fault 1601 leaq _xcopyin_nta_err(%rip), %rcx /* doesn't set rflags */ 1602 jmp do_copy_fault_nta /* use non-temporal access */ 1603 16044: 1605 movl $EFAULT, %eax 1606 jmp 3f 1607 1608 /* 1609 * A fault during do_copy_fault or do_copy_fault_nta is 1610 * indicated through an errno value in %rax and we iret from the 1611 * trap handler to here. 1612 */ 1613_xcopyin_err: 1614 addq $8, %rsp /* pop bcopy_altentry call ret addr */ 1615_xcopyin_nta_err: 1616 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 16173: 1618 movq T_COPYOPS(%r9), %r8 1619 cmpq $0, %r8 1620 jz 2f 1621 1622 /* 1623 * reload args for the copyop 1624 */ 1625 movq (%rsp), %rdi 1626 movq 0x8(%rsp), %rsi 1627 movq 0x10(%rsp), %rdx 1628 leave 1629 jmp *CP_XCOPYIN(%r8) 1630 16312: leave 1632 ret 1633 SET_SIZE(xcopyin_nta) 1634 1635#elif defined(__i386) 1636 1637#define ARG_UADDR 4 1638#define ARG_KADDR 8 1639#define ARG_COUNT 12 1640#define ARG_CACHED 16 1641 1642 .globl use_sse_copy 1643 1644 ENTRY(xcopyin_nta) 1645 movl kernelbase, %ecx 1646 lea _xcopyin_err, %eax 1647 movl %gs:CPU_THREAD, %edx 1648 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */ 1649 jae 4f 1650 1651 cmpl $0, use_sse_copy /* no sse support */ 1652 jz do_copy_fault 1653 1654 cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */ 1655 jnz do_copy_fault 1656 1657 /* 1658 * Make sure cnt is >= XCOPY_MIN_SIZE bytes 1659 */ 1660 cmpl $XCOPY_MIN_SIZE, ARG_COUNT(%esp) 1661 jb do_copy_fault 1662 1663 /* 1664 * Make sure src and dst are NTA_ALIGN_SIZE aligned, 1665 * count is COUNT_ALIGN_SIZE aligned. 1666 */ 1667 movl ARG_UADDR(%esp), %ecx 1668 orl ARG_KADDR(%esp), %ecx 1669 andl $NTA_ALIGN_MASK, %ecx 1670 orl ARG_COUNT(%esp), %ecx 1671 andl $COUNT_ALIGN_MASK, %ecx 1672 jnz do_copy_fault 1673 1674 jmp do_copy_fault_nta /* use regular access */ 1675 16764: 1677 movl $EFAULT, %eax 1678 jmp 3f 1679 1680 /* 1681 * A fault during do_copy_fault or do_copy_fault_nta is 1682 * indicated through an errno value in %eax and we iret from the 1683 * trap handler to here. 1684 */ 1685_xcopyin_err: 1686 popl %ecx 1687 popl %edi 1688 movl %ecx, T_LOFAULT(%edx) /* restore original lofault */ 1689 popl %esi 1690 popl %ebp 16913: 1692 cmpl $0, T_COPYOPS(%edx) 1693 jz 2f 1694 movl T_COPYOPS(%edx), %eax 1695 jmp *CP_XCOPYIN(%eax) 1696 16972: rep; ret /* use 2 byte return instruction when branch target */ 1698 /* AMD Software Optimization Guide - Section 6.2 */ 1699 SET_SIZE(xcopyin_nta) 1700 1701#undef ARG_UADDR 1702#undef ARG_KADDR 1703#undef ARG_COUNT 1704#undef ARG_CACHED 1705 1706#endif /* __i386 */ 1707#endif /* __lint */ 1708 1709/* 1710 * Copy kernel data to user space. 1711 */ 1712 1713#if defined(__lint) 1714 1715/* ARGSUSED */ 1716int 1717copyout(const void *kaddr, void *uaddr, size_t count) 1718{ return (0); } 1719 1720#else /* __lint */ 1721 1722#if defined(__amd64) 1723 1724 ENTRY(copyout) 1725 pushq %rbp 1726 movq %rsp, %rbp 1727 subq $24, %rsp 1728 1729 /* 1730 * save args in case we trap and need to rerun as a copyop 1731 */ 1732 movq %rdi, (%rsp) 1733 movq %rsi, 0x8(%rsp) 1734 movq %rdx, 0x10(%rsp) 1735 1736 movq kernelbase(%rip), %rax 1737#ifdef DEBUG 1738 cmpq %rax, %rdi /* %rdi = kaddr */ 1739 jnb 1f 1740 leaq .copyout_panic_msg(%rip), %rdi 1741 xorl %eax, %eax 1742 call panic 17431: 1744#endif 1745 /* 1746 * pass lofault value as 4th argument to do_copy_fault 1747 */ 1748 leaq _copyout_err(%rip), %rcx 1749 1750 movq %gs:CPU_THREAD, %r9 1751 cmpq %rax, %rsi /* test uaddr < kernelbase */ 1752 jb do_copy_fault 1753 jmp 3f 1754 1755_copyout_err: 1756 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 1757 addq $8, %rsp /* pop bcopy_altentry call ret addr */ 17583: 1759 movq T_COPYOPS(%r9), %rax 1760 cmpq $0, %rax 1761 jz 2f 1762 1763 /* 1764 * reload args for the copyop 1765 */ 1766 movq (%rsp), %rdi 1767 movq 0x8(%rsp), %rsi 1768 movq 0x10(%rsp), %rdx 1769 leave 1770 jmp *CP_COPYOUT(%rax) 1771 17722: movl $-1, %eax 1773 leave 1774 ret 1775 SET_SIZE(copyout) 1776 1777#elif defined(__i386) 1778 1779#define ARG_KADDR 4 1780#define ARG_UADDR 8 1781 1782 ENTRY(copyout) 1783 movl kernelbase, %ecx 1784#ifdef DEBUG 1785 cmpl %ecx, ARG_KADDR(%esp) 1786 jnb 1f 1787 pushl %ebp 1788 movl %esp, %ebp 1789 pushl $.copyout_panic_msg 1790 call panic 17911: 1792#endif 1793 lea _copyout_err, %eax 1794 movl %gs:CPU_THREAD, %edx 1795 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */ 1796 jb do_copy_fault 1797 jmp 3f 1798 1799_copyout_err: 1800 popl %ecx 1801 popl %edi 1802 movl %ecx, T_LOFAULT(%edx) /* restore original lofault */ 1803 popl %esi 1804 popl %ebp 18053: 1806 movl T_COPYOPS(%edx), %eax 1807 cmpl $0, %eax 1808 jz 2f 1809 jmp *CP_COPYOUT(%eax) 1810 18112: movl $-1, %eax 1812 ret 1813 SET_SIZE(copyout) 1814 1815#undef ARG_UADDR 1816#undef ARG_KADDR 1817 1818#endif /* __i386 */ 1819#endif /* __lint */ 1820 1821#if defined(__lint) 1822 1823/* ARGSUSED */ 1824int 1825xcopyout_nta(const void *kaddr, void *uaddr, size_t count, int copy_cached) 1826{ return (0); } 1827 1828#else /* __lint */ 1829 1830#if defined(__amd64) 1831 1832 ENTRY(xcopyout_nta) 1833 pushq %rbp 1834 movq %rsp, %rbp 1835 subq $24, %rsp 1836 1837 /* 1838 * save args in case we trap and need to rerun as a copyop 1839 */ 1840 movq %rdi, (%rsp) 1841 movq %rsi, 0x8(%rsp) 1842 movq %rdx, 0x10(%rsp) 1843 1844 movq kernelbase(%rip), %rax 1845#ifdef DEBUG 1846 cmpq %rax, %rdi /* %rdi = kaddr */ 1847 jnb 1f 1848 leaq .xcopyout_panic_msg(%rip), %rdi 1849 xorl %eax, %eax 1850 call panic 18511: 1852#endif 1853 movq %gs:CPU_THREAD, %r9 1854 cmpq %rax, %rsi /* test uaddr < kernelbase */ 1855 jae 4f 1856 1857 cmpq $0, %rcx /* No non-temporal access? */ 1858 /* 1859 * pass lofault value as 4th argument to do_copy_fault 1860 */ 1861 leaq _xcopyout_err(%rip), %rcx 1862 jnz do_copy_fault 1863 /* 1864 * Make sure cnt is >= XCOPY_MIN_SIZE bytes 1865 */ 1866 cmpq $XCOPY_MIN_SIZE, %rdx 1867 jb do_copy_fault 1868 1869 /* 1870 * Make sure src and dst are NTA_ALIGN_SIZE aligned, 1871 * count is COUNT_ALIGN_SIZE aligned. 1872 */ 1873 movq %rdi, %r10 1874 orq %rsi, %r10 1875 andq $NTA_ALIGN_MASK, %r10 1876 orq %rdx, %r10 1877 andq $COUNT_ALIGN_MASK, %r10 1878 jnz do_copy_fault 1879 leaq _xcopyout_nta_err(%rip), %rcx 1880 jmp do_copy_fault_nta 1881 18824: 1883 movl $EFAULT, %eax 1884 jmp 3f 1885 1886 /* 1887 * A fault during do_copy_fault or do_copy_fault_nta is 1888 * indicated through an errno value in %rax and we iret from the 1889 * trap handler to here. 1890 */ 1891_xcopyout_err: 1892 addq $8, %rsp /* pop bcopy_altentry call ret addr */ 1893_xcopyout_nta_err: 1894 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 18953: 1896 movq T_COPYOPS(%r9), %r8 1897 cmpq $0, %r8 1898 jz 2f 1899 1900 /* 1901 * reload args for the copyop 1902 */ 1903 movq (%rsp), %rdi 1904 movq 0x8(%rsp), %rsi 1905 movq 0x10(%rsp), %rdx 1906 leave 1907 jmp *CP_XCOPYOUT(%r8) 1908 19092: leave 1910 ret 1911 SET_SIZE(xcopyout_nta) 1912 1913#elif defined(__i386) 1914 1915#define ARG_KADDR 4 1916#define ARG_UADDR 8 1917#define ARG_COUNT 12 1918#define ARG_CACHED 16 1919 1920 ENTRY(xcopyout_nta) 1921 movl kernelbase, %ecx 1922 lea _xcopyout_err, %eax 1923 movl %gs:CPU_THREAD, %edx 1924 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */ 1925 jae 4f 1926 1927 cmpl $0, use_sse_copy /* no sse support */ 1928 jz do_copy_fault 1929 1930 cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */ 1931 jnz do_copy_fault 1932 1933 /* 1934 * Make sure cnt is >= XCOPY_MIN_SIZE bytes 1935 */ 1936 cmpl $XCOPY_MIN_SIZE, %edx 1937 jb do_copy_fault 1938 1939 /* 1940 * Make sure src and dst are NTA_ALIGN_SIZE aligned, 1941 * count is COUNT_ALIGN_SIZE aligned. 1942 */ 1943 movl ARG_UADDR(%esp), %ecx 1944 orl ARG_KADDR(%esp), %ecx 1945 andl $NTA_ALIGN_MASK, %ecx 1946 orl ARG_COUNT(%esp), %ecx 1947 andl $COUNT_ALIGN_MASK, %ecx 1948 jnz do_copy_fault 1949 jmp do_copy_fault_nta 1950 19514: 1952 movl $EFAULT, %eax 1953 jmp 3f 1954 1955 /* 1956 * A fault during do_copy_fault or do_copy_fault_nta is 1957 * indicated through an errno value in %eax and we iret from the 1958 * trap handler to here. 1959 */ 1960_xcopyout_err: 1961 / restore the original lofault 1962 popl %ecx 1963 popl %edi 1964 movl %ecx, T_LOFAULT(%edx) / original lofault 1965 popl %esi 1966 popl %ebp 19673: 1968 cmpl $0, T_COPYOPS(%edx) 1969 jz 2f 1970 movl T_COPYOPS(%edx), %eax 1971 jmp *CP_XCOPYOUT(%eax) 1972 19732: rep; ret /* use 2 byte return instruction when branch target */ 1974 /* AMD Software Optimization Guide - Section 6.2 */ 1975 SET_SIZE(xcopyout_nta) 1976 1977#undef ARG_UADDR 1978#undef ARG_KADDR 1979#undef ARG_COUNT 1980#undef ARG_CACHED 1981 1982#endif /* __i386 */ 1983#endif /* __lint */ 1984 1985/* 1986 * Copy a null terminated string from one point to another in 1987 * the kernel address space. 1988 */ 1989 1990#if defined(__lint) 1991 1992/* ARGSUSED */ 1993int 1994copystr(const char *from, char *to, size_t maxlength, size_t *lencopied) 1995{ return (0); } 1996 1997#else /* __lint */ 1998 1999#if defined(__amd64) 2000 2001 ENTRY(copystr) 2002 pushq %rbp 2003 movq %rsp, %rbp 2004#ifdef DEBUG 2005 movq kernelbase(%rip), %rax 2006 cmpq %rax, %rdi /* %rdi = from */ 2007 jb 0f 2008 cmpq %rax, %rsi /* %rsi = to */ 2009 jnb 1f 20100: leaq .copystr_panic_msg(%rip), %rdi 2011 xorl %eax, %eax 2012 call panic 20131: 2014#endif 2015 movq %gs:CPU_THREAD, %r9 2016 movq T_LOFAULT(%r9), %r8 /* pass current lofault value as */ 2017 /* 5th argument to do_copystr */ 2018do_copystr: 2019 movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */ 2020 movq T_LOFAULT(%r9), %r11 /* save the current lofault */ 2021 movq %r8, T_LOFAULT(%r9) /* new lofault */ 2022 2023 movq %rdx, %r8 /* save maxlength */ 2024 2025 cmpq $0, %rdx /* %rdx = maxlength */ 2026 je copystr_enametoolong /* maxlength == 0 */ 2027 2028copystr_loop: 2029 decq %r8 2030 movb (%rdi), %al 2031 incq %rdi 2032 movb %al, (%rsi) 2033 incq %rsi 2034 cmpb $0, %al 2035 je copystr_null /* null char */ 2036 cmpq $0, %r8 2037 jne copystr_loop 2038 2039copystr_enametoolong: 2040 movl $ENAMETOOLONG, %eax 2041 jmp copystr_out 2042 2043copystr_null: 2044 xorl %eax, %eax /* no error */ 2045 2046copystr_out: 2047 cmpq $0, %rcx /* want length? */ 2048 je copystr_done /* no */ 2049 subq %r8, %rdx /* compute length and store it */ 2050 movq %rdx, (%rcx) 2051 2052copystr_done: 2053 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */ 2054 leave 2055 ret 2056 SET_SIZE(copystr) 2057 2058#elif defined(__i386) 2059 2060#define ARG_FROM 8 2061#define ARG_TO 12 2062#define ARG_MAXLEN 16 2063#define ARG_LENCOPIED 20 2064 2065 ENTRY(copystr) 2066#ifdef DEBUG 2067 pushl %ebp 2068 movl %esp, %ebp 2069 movl kernelbase, %eax 2070 cmpl %eax, ARG_FROM(%esp) 2071 jb 0f 2072 cmpl %eax, ARG_TO(%esp) 2073 jnb 1f 20740: pushl $.copystr_panic_msg 2075 call panic 20761: popl %ebp 2077#endif 2078 /* get the current lofault address */ 2079 movl %gs:CPU_THREAD, %eax 2080 movl T_LOFAULT(%eax), %eax 2081do_copystr: 2082 pushl %ebp /* setup stack frame */ 2083 movl %esp, %ebp 2084 pushl %ebx /* save registers */ 2085 pushl %edi 2086 2087 movl %gs:CPU_THREAD, %ebx 2088 movl T_LOFAULT(%ebx), %edi 2089 pushl %edi /* save the current lofault */ 2090 movl %eax, T_LOFAULT(%ebx) /* new lofault */ 2091 2092 movl ARG_MAXLEN(%ebp), %ecx 2093 cmpl $0, %ecx 2094 je copystr_enametoolong /* maxlength == 0 */ 2095 2096 movl ARG_FROM(%ebp), %ebx /* source address */ 2097 movl ARG_TO(%ebp), %edx /* destination address */ 2098 2099copystr_loop: 2100 decl %ecx 2101 movb (%ebx), %al 2102 incl %ebx 2103 movb %al, (%edx) 2104 incl %edx 2105 cmpb $0, %al 2106 je copystr_null /* null char */ 2107 cmpl $0, %ecx 2108 jne copystr_loop 2109 2110copystr_enametoolong: 2111 movl $ENAMETOOLONG, %eax 2112 jmp copystr_out 2113 2114copystr_null: 2115 xorl %eax, %eax /* no error */ 2116 2117copystr_out: 2118 cmpl $0, ARG_LENCOPIED(%ebp) /* want length? */ 2119 je copystr_done /* no */ 2120 movl ARG_MAXLEN(%ebp), %edx 2121 subl %ecx, %edx /* compute length and store it */ 2122 movl ARG_LENCOPIED(%ebp), %ecx 2123 movl %edx, (%ecx) 2124 2125copystr_done: 2126 popl %edi 2127 movl %gs:CPU_THREAD, %ebx 2128 movl %edi, T_LOFAULT(%ebx) /* restore the original lofault */ 2129 2130 popl %edi 2131 popl %ebx 2132 popl %ebp 2133 ret 2134 SET_SIZE(copystr) 2135 2136#undef ARG_FROM 2137#undef ARG_TO 2138#undef ARG_MAXLEN 2139#undef ARG_LENCOPIED 2140 2141#endif /* __i386 */ 2142#endif /* __lint */ 2143 2144/* 2145 * Copy a null terminated string from the user address space into 2146 * the kernel address space. 2147 */ 2148 2149#if defined(__lint) 2150 2151/* ARGSUSED */ 2152int 2153copyinstr(const char *uaddr, char *kaddr, size_t maxlength, 2154 size_t *lencopied) 2155{ return (0); } 2156 2157#else /* __lint */ 2158 2159#if defined(__amd64) 2160 2161 ENTRY(copyinstr) 2162 pushq %rbp 2163 movq %rsp, %rbp 2164 subq $32, %rsp 2165 2166 /* 2167 * save args in case we trap and need to rerun as a copyop 2168 */ 2169 movq %rdi, (%rsp) 2170 movq %rsi, 0x8(%rsp) 2171 movq %rdx, 0x10(%rsp) 2172 movq %rcx, 0x18(%rsp) 2173 2174 movq kernelbase(%rip), %rax 2175#ifdef DEBUG 2176 cmpq %rax, %rsi /* %rsi = kaddr */ 2177 jnb 1f 2178 leaq .copyinstr_panic_msg(%rip), %rdi 2179 xorl %eax, %eax 2180 call panic 21811: 2182#endif 2183 /* 2184 * pass lofault value as 5th argument to do_copystr 2185 */ 2186 leaq _copyinstr_error(%rip), %r8 2187 2188 cmpq %rax, %rdi /* test uaddr < kernelbase */ 2189 jb do_copystr 2190 movq %gs:CPU_THREAD, %r9 2191 jmp 3f 2192 2193_copyinstr_error: 2194 movq %r11, T_LOFAULT(%r9) /* restore original lofault */ 21953: 2196 movq T_COPYOPS(%r9), %rax 2197 cmpq $0, %rax 2198 jz 2f 2199 2200 /* 2201 * reload args for the copyop 2202 */ 2203 movq (%rsp), %rdi 2204 movq 0x8(%rsp), %rsi 2205 movq 0x10(%rsp), %rdx 2206 movq 0x18(%rsp), %rcx 2207 leave 2208 jmp *CP_COPYINSTR(%rax) 2209 22102: movl $EFAULT, %eax /* return EFAULT */ 2211 leave 2212 ret 2213 SET_SIZE(copyinstr) 2214 2215#elif defined(__i386) 2216 2217#define ARG_UADDR 4 2218#define ARG_KADDR 8 2219 2220 ENTRY(copyinstr) 2221 movl kernelbase, %ecx 2222#ifdef DEBUG 2223 cmpl %ecx, ARG_KADDR(%esp) 2224 jnb 1f 2225 pushl %ebp 2226 movl %esp, %ebp 2227 pushl $.copyinstr_panic_msg 2228 call panic 22291: 2230#endif 2231 lea _copyinstr_error, %eax 2232 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */ 2233 jb do_copystr 2234 movl %gs:CPU_THREAD, %edx 2235 jmp 3f 2236 2237_copyinstr_error: 2238 popl %edi 2239 movl %gs:CPU_THREAD, %edx 2240 movl %edi, T_LOFAULT(%edx) /* original lofault */ 2241 2242 popl %edi 2243 popl %ebx 2244 popl %ebp 22453: 2246 movl T_COPYOPS(%edx), %eax 2247 cmpl $0, %eax 2248 jz 2f 2249 jmp *CP_COPYINSTR(%eax) 2250 22512: movl $EFAULT, %eax /* return EFAULT */ 2252 ret 2253 SET_SIZE(copyinstr) 2254 2255#undef ARG_UADDR 2256#undef ARG_KADDR 2257 2258#endif /* __i386 */ 2259#endif /* __lint */ 2260 2261/* 2262 * Copy a null terminated string from the kernel 2263 * address space to the user address space. 2264 */ 2265 2266#if defined(__lint) 2267 2268/* ARGSUSED */ 2269int 2270copyoutstr(const char *kaddr, char *uaddr, size_t maxlength, 2271 size_t *lencopied) 2272{ return (0); } 2273 2274#else /* __lint */ 2275 2276#if defined(__amd64) 2277 2278 ENTRY(copyoutstr) 2279 pushq %rbp 2280 movq %rsp, %rbp 2281 subq $32, %rsp 2282 2283 /* 2284 * save args in case we trap and need to rerun as a copyop 2285 */ 2286 movq %rdi, (%rsp) 2287 movq %rsi, 0x8(%rsp) 2288 movq %rdx, 0x10(%rsp) 2289 movq %rcx, 0x18(%rsp) 2290 2291 movq kernelbase(%rip), %rax 2292#ifdef DEBUG 2293 cmpq %rax, %rdi /* %rdi = kaddr */ 2294 jnb 1f 2295 leaq .copyoutstr_panic_msg(%rip), %rdi 2296 jmp call_panic /* setup stack and call panic */ 22971: 2298#endif 2299 /* 2300 * pass lofault value as 5th argument to do_copystr 2301 */ 2302 leaq _copyoutstr_error(%rip), %r8 2303 2304 cmpq %rax, %rsi /* test uaddr < kernelbase */ 2305 jb do_copystr 2306 movq %gs:CPU_THREAD, %r9 2307 jmp 3f 2308 2309_copyoutstr_error: 2310 movq %r11, T_LOFAULT(%r9) /* restore the original lofault */ 23113: 2312 movq T_COPYOPS(%r9), %rax 2313 cmpq $0, %rax 2314 jz 2f 2315 2316 /* 2317 * reload args for the copyop 2318 */ 2319 movq (%rsp), %rdi 2320 movq 0x8(%rsp), %rsi 2321 movq 0x10(%rsp), %rdx 2322 movq 0x18(%rsp), %rcx 2323 leave 2324 jmp *CP_COPYOUTSTR(%rax) 2325 23262: movl $EFAULT, %eax /* return EFAULT */ 2327 leave 2328 ret 2329 SET_SIZE(copyoutstr) 2330 2331#elif defined(__i386) 2332 2333#define ARG_KADDR 4 2334#define ARG_UADDR 8 2335 2336 ENTRY(copyoutstr) 2337 movl kernelbase, %ecx 2338#ifdef DEBUG 2339 cmpl %ecx, ARG_KADDR(%esp) 2340 jnb 1f 2341 pushl %ebp 2342 movl %esp, %ebp 2343 pushl $.copyoutstr_panic_msg 2344 call panic 23451: 2346#endif 2347 lea _copyoutstr_error, %eax 2348 cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */ 2349 jb do_copystr 2350 movl %gs:CPU_THREAD, %edx 2351 jmp 3f 2352 2353_copyoutstr_error: 2354 popl %edi 2355 movl %gs:CPU_THREAD, %edx 2356 movl %edi, T_LOFAULT(%edx) /* restore the original lofault */ 2357 2358 popl %edi 2359 popl %ebx 2360 popl %ebp 23613: 2362 movl T_COPYOPS(%edx), %eax 2363 cmpl $0, %eax 2364 jz 2f 2365 jmp *CP_COPYOUTSTR(%eax) 2366 23672: movl $EFAULT, %eax /* return EFAULT */ 2368 ret 2369 SET_SIZE(copyoutstr) 2370 2371#undef ARG_KADDR 2372#undef ARG_UADDR 2373 2374#endif /* __i386 */ 2375#endif /* __lint */ 2376 2377/* 2378 * Since all of the fuword() variants are so similar, we have a macro to spit 2379 * them out. This allows us to create DTrace-unobservable functions easily. 2380 */ 2381 2382#if defined(__lint) 2383 2384#if defined(__amd64) 2385 2386/* ARGSUSED */ 2387int 2388fuword64(const void *addr, uint64_t *dst) 2389{ return (0); } 2390 2391#endif 2392 2393/* ARGSUSED */ 2394int 2395fuword32(const void *addr, uint32_t *dst) 2396{ return (0); } 2397 2398/* ARGSUSED */ 2399int 2400fuword16(const void *addr, uint16_t *dst) 2401{ return (0); } 2402 2403/* ARGSUSED */ 2404int 2405fuword8(const void *addr, uint8_t *dst) 2406{ return (0); } 2407 2408#else /* __lint */ 2409 2410#if defined(__amd64) 2411 2412/* 2413 * (Note that we don't save and reload the arguments here 2414 * because their values are not altered in the copy path) 2415 */ 2416 2417#define FUWORD(NAME, INSTR, REG, COPYOP) \ 2418 ENTRY(NAME) \ 2419 movq %gs:CPU_THREAD, %r9; \ 2420 cmpq kernelbase(%rip), %rdi; \ 2421 jae 1f; \ 2422 leaq _flt_/**/NAME, %rdx; \ 2423 movq %rdx, T_LOFAULT(%r9); \ 2424 INSTR (%rdi), REG; \ 2425 movq $0, T_LOFAULT(%r9); \ 2426 INSTR REG, (%rsi); \ 2427 xorl %eax, %eax; \ 2428 ret; \ 2429_flt_/**/NAME: \ 2430 movq $0, T_LOFAULT(%r9); \ 24311: \ 2432 movq T_COPYOPS(%r9), %rax; \ 2433 cmpq $0, %rax; \ 2434 jz 2f; \ 2435 jmp *COPYOP(%rax); \ 24362: \ 2437 movl $-1, %eax; \ 2438 ret; \ 2439 SET_SIZE(NAME) 2440 2441 FUWORD(fuword64, movq, %rax, CP_FUWORD64) 2442 FUWORD(fuword32, movl, %eax, CP_FUWORD32) 2443 FUWORD(fuword16, movw, %ax, CP_FUWORD16) 2444 FUWORD(fuword8, movb, %al, CP_FUWORD8) 2445 2446#elif defined(__i386) 2447 2448#define FUWORD(NAME, INSTR, REG, COPYOP) \ 2449 ENTRY(NAME) \ 2450 movl %gs:CPU_THREAD, %ecx; \ 2451 movl kernelbase, %eax; \ 2452 cmpl %eax, 4(%esp); \ 2453 jae 1f; \ 2454 lea _flt_/**/NAME, %edx; \ 2455 movl %edx, T_LOFAULT(%ecx); \ 2456 movl 4(%esp), %eax; \ 2457 movl 8(%esp), %edx; \ 2458 INSTR (%eax), REG; \ 2459 movl $0, T_LOFAULT(%ecx); \ 2460 INSTR REG, (%edx); \ 2461 xorl %eax, %eax; \ 2462 ret; \ 2463_flt_/**/NAME: \ 2464 movl $0, T_LOFAULT(%ecx); \ 24651: \ 2466 movl T_COPYOPS(%ecx), %eax; \ 2467 cmpl $0, %eax; \ 2468 jz 2f; \ 2469 jmp *COPYOP(%eax); \ 24702: \ 2471 movl $-1, %eax; \ 2472 ret; \ 2473 SET_SIZE(NAME) 2474 2475 FUWORD(fuword32, movl, %eax, CP_FUWORD32) 2476 FUWORD(fuword16, movw, %ax, CP_FUWORD16) 2477 FUWORD(fuword8, movb, %al, CP_FUWORD8) 2478 2479#endif /* __i386 */ 2480 2481#undef FUWORD 2482 2483#endif /* __lint */ 2484 2485/* 2486 * Set user word. 2487 */ 2488 2489#if defined(__lint) 2490 2491#if defined(__amd64) 2492 2493/* ARGSUSED */ 2494int 2495suword64(void *addr, uint64_t value) 2496{ return (0); } 2497 2498#endif 2499 2500/* ARGSUSED */ 2501int 2502suword32(void *addr, uint32_t value) 2503{ return (0); } 2504 2505/* ARGSUSED */ 2506int 2507suword16(void *addr, uint16_t value) 2508{ return (0); } 2509 2510/* ARGSUSED */ 2511int 2512suword8(void *addr, uint8_t value) 2513{ return (0); } 2514 2515#else /* lint */ 2516 2517#if defined(__amd64) 2518 2519/* 2520 * (Note that we don't save and reload the arguments here 2521 * because their values are not altered in the copy path) 2522 */ 2523 2524#define SUWORD(NAME, INSTR, REG, COPYOP) \ 2525 ENTRY(NAME) \ 2526 movq %gs:CPU_THREAD, %r9; \ 2527 cmpq kernelbase(%rip), %rdi; \ 2528 jae 1f; \ 2529 leaq _flt_/**/NAME, %rdx; \ 2530 movq %rdx, T_LOFAULT(%r9); \ 2531 INSTR REG, (%rdi); \ 2532 movq $0, T_LOFAULT(%r9); \ 2533 xorl %eax, %eax; \ 2534 ret; \ 2535_flt_/**/NAME: \ 2536 movq $0, T_LOFAULT(%r9); \ 25371: \ 2538 movq T_COPYOPS(%r9), %rax; \ 2539 cmpq $0, %rax; \ 2540 jz 3f; \ 2541 jmp *COPYOP(%rax); \ 25423: \ 2543 movl $-1, %eax; \ 2544 ret; \ 2545 SET_SIZE(NAME) 2546 2547 SUWORD(suword64, movq, %rsi, CP_SUWORD64) 2548 SUWORD(suword32, movl, %esi, CP_SUWORD32) 2549 SUWORD(suword16, movw, %si, CP_SUWORD16) 2550 SUWORD(suword8, movb, %sil, CP_SUWORD8) 2551 2552#elif defined(__i386) 2553 2554#define SUWORD(NAME, INSTR, REG, COPYOP) \ 2555 ENTRY(NAME) \ 2556 movl %gs:CPU_THREAD, %ecx; \ 2557 movl kernelbase, %eax; \ 2558 cmpl %eax, 4(%esp); \ 2559 jae 1f; \ 2560 lea _flt_/**/NAME, %edx; \ 2561 movl %edx, T_LOFAULT(%ecx); \ 2562 movl 4(%esp), %eax; \ 2563 movl 8(%esp), %edx; \ 2564 INSTR REG, (%eax); \ 2565 movl $0, T_LOFAULT(%ecx); \ 2566 xorl %eax, %eax; \ 2567 ret; \ 2568_flt_/**/NAME: \ 2569 movl $0, T_LOFAULT(%ecx); \ 25701: \ 2571 movl T_COPYOPS(%ecx), %eax; \ 2572 cmpl $0, %eax; \ 2573 jz 3f; \ 2574 movl COPYOP(%eax), %ecx; \ 2575 jmp *%ecx; \ 25763: \ 2577 movl $-1, %eax; \ 2578 ret; \ 2579 SET_SIZE(NAME) 2580 2581 SUWORD(suword32, movl, %edx, CP_SUWORD32) 2582 SUWORD(suword16, movw, %dx, CP_SUWORD16) 2583 SUWORD(suword8, movb, %dl, CP_SUWORD8) 2584 2585#endif /* __i386 */ 2586 2587#undef SUWORD 2588 2589#endif /* __lint */ 2590 2591#if defined(__lint) 2592 2593#if defined(__amd64) 2594 2595/*ARGSUSED*/ 2596void 2597fuword64_noerr(const void *addr, uint64_t *dst) 2598{} 2599 2600#endif 2601 2602/*ARGSUSED*/ 2603void 2604fuword32_noerr(const void *addr, uint32_t *dst) 2605{} 2606 2607/*ARGSUSED*/ 2608void 2609fuword8_noerr(const void *addr, uint8_t *dst) 2610{} 2611 2612/*ARGSUSED*/ 2613void 2614fuword16_noerr(const void *addr, uint16_t *dst) 2615{} 2616 2617#else /* __lint */ 2618 2619#if defined(__amd64) 2620 2621#define FUWORD_NOERR(NAME, INSTR, REG) \ 2622 ENTRY(NAME) \ 2623 cmpq kernelbase(%rip), %rdi; \ 2624 cmovnbq kernelbase(%rip), %rdi; \ 2625 INSTR (%rdi), REG; \ 2626 INSTR REG, (%rsi); \ 2627 ret; \ 2628 SET_SIZE(NAME) 2629 2630 FUWORD_NOERR(fuword64_noerr, movq, %rax) 2631 FUWORD_NOERR(fuword32_noerr, movl, %eax) 2632 FUWORD_NOERR(fuword16_noerr, movw, %ax) 2633 FUWORD_NOERR(fuword8_noerr, movb, %al) 2634 2635#elif defined(__i386) 2636 2637#define FUWORD_NOERR(NAME, INSTR, REG) \ 2638 ENTRY(NAME) \ 2639 movl 4(%esp), %eax; \ 2640 cmpl kernelbase, %eax; \ 2641 jb 1f; \ 2642 movl kernelbase, %eax; \ 26431: movl 8(%esp), %edx; \ 2644 INSTR (%eax), REG; \ 2645 INSTR REG, (%edx); \ 2646 ret; \ 2647 SET_SIZE(NAME) 2648 2649 FUWORD_NOERR(fuword32_noerr, movl, %ecx) 2650 FUWORD_NOERR(fuword16_noerr, movw, %cx) 2651 FUWORD_NOERR(fuword8_noerr, movb, %cl) 2652 2653#endif /* __i386 */ 2654 2655#undef FUWORD_NOERR 2656 2657#endif /* __lint */ 2658 2659#if defined(__lint) 2660 2661#if defined(__amd64) 2662 2663/*ARGSUSED*/ 2664void 2665suword64_noerr(void *addr, uint64_t value) 2666{} 2667 2668#endif 2669 2670/*ARGSUSED*/ 2671void 2672suword32_noerr(void *addr, uint32_t value) 2673{} 2674 2675/*ARGSUSED*/ 2676void 2677suword16_noerr(void *addr, uint16_t value) 2678{} 2679 2680/*ARGSUSED*/ 2681void 2682suword8_noerr(void *addr, uint8_t value) 2683{} 2684 2685#else /* lint */ 2686 2687#if defined(__amd64) 2688 2689#define SUWORD_NOERR(NAME, INSTR, REG) \ 2690 ENTRY(NAME) \ 2691 cmpq kernelbase(%rip), %rdi; \ 2692 cmovnbq kernelbase(%rip), %rdi; \ 2693 INSTR REG, (%rdi); \ 2694 ret; \ 2695 SET_SIZE(NAME) 2696 2697 SUWORD_NOERR(suword64_noerr, movq, %rsi) 2698 SUWORD_NOERR(suword32_noerr, movl, %esi) 2699 SUWORD_NOERR(suword16_noerr, movw, %si) 2700 SUWORD_NOERR(suword8_noerr, movb, %sil) 2701 2702#elif defined(__i386) 2703 2704#define SUWORD_NOERR(NAME, INSTR, REG) \ 2705 ENTRY(NAME) \ 2706 movl 4(%esp), %eax; \ 2707 cmpl kernelbase, %eax; \ 2708 jb 1f; \ 2709 movl kernelbase, %eax; \ 27101: \ 2711 movl 8(%esp), %edx; \ 2712 INSTR REG, (%eax); \ 2713 ret; \ 2714 SET_SIZE(NAME) 2715 2716 SUWORD_NOERR(suword32_noerr, movl, %edx) 2717 SUWORD_NOERR(suword16_noerr, movw, %dx) 2718 SUWORD_NOERR(suword8_noerr, movb, %dl) 2719 2720#endif /* __i386 */ 2721 2722#undef SUWORD_NOERR 2723 2724#endif /* lint */ 2725 2726 2727#if defined(__lint) 2728 2729/*ARGSUSED*/ 2730int 2731subyte(void *addr, uchar_t value) 2732{ return (0); } 2733 2734/*ARGSUSED*/ 2735void 2736subyte_noerr(void *addr, uchar_t value) 2737{} 2738 2739/*ARGSUSED*/ 2740int 2741fulword(const void *addr, ulong_t *valuep) 2742{ return (0); } 2743 2744/*ARGSUSED*/ 2745void 2746fulword_noerr(const void *addr, ulong_t *valuep) 2747{} 2748 2749/*ARGSUSED*/ 2750int 2751sulword(void *addr, ulong_t valuep) 2752{ return (0); } 2753 2754/*ARGSUSED*/ 2755void 2756sulword_noerr(void *addr, ulong_t valuep) 2757{} 2758 2759#else 2760 2761 .weak subyte 2762 subyte=suword8 2763 .weak subyte_noerr 2764 subyte_noerr=suword8_noerr 2765 2766#if defined(__amd64) 2767 2768 .weak fulword 2769 fulword=fuword64 2770 .weak fulword_noerr 2771 fulword_noerr=fuword64_noerr 2772 .weak sulword 2773 sulword=suword64 2774 .weak sulword_noerr 2775 sulword_noerr=suword64_noerr 2776 2777#elif defined(__i386) 2778 2779 .weak fulword 2780 fulword=fuword32 2781 .weak fulword_noerr 2782 fulword_noerr=fuword32_noerr 2783 .weak sulword 2784 sulword=suword32 2785 .weak sulword_noerr 2786 sulword_noerr=suword32_noerr 2787 2788#endif /* __i386 */ 2789 2790#endif /* __lint */ 2791 2792#if defined(__lint) 2793 2794/* 2795 * Copy a block of storage - must not overlap (from + len <= to). 2796 * No fault handler installed (to be called under on_fault()) 2797 */ 2798 2799/* ARGSUSED */ 2800void 2801copyout_noerr(const void *kfrom, void *uto, size_t count) 2802{} 2803 2804/* ARGSUSED */ 2805void 2806copyin_noerr(const void *ufrom, void *kto, size_t count) 2807{} 2808 2809/* 2810 * Zero a block of storage in user space 2811 */ 2812 2813/* ARGSUSED */ 2814void 2815uzero(void *addr, size_t count) 2816{} 2817 2818/* 2819 * copy a block of storage in user space 2820 */ 2821 2822/* ARGSUSED */ 2823void 2824ucopy(const void *ufrom, void *uto, size_t ulength) 2825{} 2826 2827/* 2828 * copy a string in user space 2829 */ 2830 2831/* ARGSUSED */ 2832void 2833ucopystr(const char *ufrom, char *uto, size_t umaxlength, size_t *lencopied) 2834{} 2835 2836#else /* __lint */ 2837 2838#if defined(__amd64) 2839 2840 ENTRY(copyin_noerr) 2841 movq kernelbase(%rip), %rax 2842#ifdef DEBUG 2843 cmpq %rax, %rsi /* %rsi = kto */ 2844 jae 1f 2845 leaq .cpyin_ne_pmsg(%rip), %rdi 2846 jmp call_panic /* setup stack and call panic */ 28471: 2848#endif 2849 cmpq %rax, %rdi /* ufrom < kernelbase */ 2850 jb do_copy 2851 movq %rax, %rdi /* force fault at kernelbase */ 2852 jmp do_copy 2853 SET_SIZE(copyin_noerr) 2854 2855 ENTRY(copyout_noerr) 2856 movq kernelbase(%rip), %rax 2857#ifdef DEBUG 2858 cmpq %rax, %rdi /* %rdi = kfrom */ 2859 jae 1f 2860 leaq .cpyout_ne_pmsg(%rip), %rdi 2861 jmp call_panic /* setup stack and call panic */ 28621: 2863#endif 2864 cmpq %rax, %rsi /* uto < kernelbase */ 2865 jb do_copy 2866 movq %rax, %rsi /* force fault at kernelbase */ 2867 jmp do_copy 2868 SET_SIZE(copyout_noerr) 2869 2870 ENTRY(uzero) 2871 movq kernelbase(%rip), %rax 2872 cmpq %rax, %rdi 2873 jb do_zero 2874 movq %rax, %rdi /* force fault at kernelbase */ 2875 jmp do_zero 2876 SET_SIZE(uzero) 2877 2878 ENTRY(ucopy) 2879 movq kernelbase(%rip), %rax 2880 cmpq %rax, %rdi 2881 cmovaeq %rax, %rdi /* force fault at kernelbase */ 2882 cmpq %rax, %rsi 2883 cmovaeq %rax, %rsi /* force fault at kernelbase */ 2884 jmp do_copy 2885 SET_SIZE(ucopy) 2886 2887 ENTRY(ucopystr) 2888 pushq %rbp 2889 movq %rsp, %rbp 2890 movq kernelbase(%rip), %rax 2891 cmpq %rax, %rdi 2892 cmovaeq %rax, %rdi /* force fault at kernelbase */ 2893 cmpq %rax, %rsi 2894 cmovaeq %rax, %rsi /* force fault at kernelbase */ 2895 /* do_copystr expects lofault address in %r8 */ 2896 movq %gs:CPU_THREAD, %r8 2897 movq T_LOFAULT(%r8), %r8 2898 jmp do_copystr 2899 SET_SIZE(ucopystr) 2900 2901#elif defined(__i386) 2902 2903 ENTRY(copyin_noerr) 2904 movl kernelbase, %eax 2905#ifdef DEBUG 2906 cmpl %eax, 8(%esp) 2907 jae 1f 2908 pushl $.cpyin_ne_pmsg 2909 call panic 29101: 2911#endif 2912 cmpl %eax, 4(%esp) 2913 jb do_copy 2914 movl %eax, 4(%esp) /* force fault at kernelbase */ 2915 jmp do_copy 2916 SET_SIZE(copyin_noerr) 2917 2918 ENTRY(copyout_noerr) 2919 movl kernelbase, %eax 2920#ifdef DEBUG 2921 cmpl %eax, 4(%esp) 2922 jae 1f 2923 pushl $.cpyout_ne_pmsg 2924 call panic 29251: 2926#endif 2927 cmpl %eax, 8(%esp) 2928 jb do_copy 2929 movl %eax, 8(%esp) /* force fault at kernelbase */ 2930 jmp do_copy 2931 SET_SIZE(copyout_noerr) 2932 2933 ENTRY(uzero) 2934 movl kernelbase, %eax 2935 cmpl %eax, 4(%esp) 2936 jb do_zero 2937 movl %eax, 4(%esp) /* force fault at kernelbase */ 2938 jmp do_zero 2939 SET_SIZE(uzero) 2940 2941 ENTRY(ucopy) 2942 movl kernelbase, %eax 2943 cmpl %eax, 4(%esp) 2944 jb 1f 2945 movl %eax, 4(%esp) /* force fault at kernelbase */ 29461: 2947 cmpl %eax, 8(%esp) 2948 jb do_copy 2949 movl %eax, 8(%esp) /* force fault at kernelbase */ 2950 jmp do_copy 2951 SET_SIZE(ucopy) 2952 2953 ENTRY(ucopystr) 2954 movl kernelbase, %eax 2955 cmpl %eax, 4(%esp) 2956 jb 1f 2957 movl %eax, 4(%esp) /* force fault at kernelbase */ 29581: 2959 cmpl %eax, 8(%esp) 2960 jb 2f 2961 movl %eax, 8(%esp) /* force fault at kernelbase */ 29622: 2963 /* do_copystr expects the lofault address in %eax */ 2964 movl %gs:CPU_THREAD, %eax 2965 movl T_LOFAULT(%eax), %eax 2966 jmp do_copystr 2967 SET_SIZE(ucopystr) 2968 2969#endif /* __i386 */ 2970 2971#ifdef DEBUG 2972 .data 2973.kcopy_panic_msg: 2974 .string "kcopy: arguments below kernelbase" 2975.bcopy_panic_msg: 2976 .string "bcopy: arguments below kernelbase" 2977.kzero_panic_msg: 2978 .string "kzero: arguments below kernelbase" 2979.bzero_panic_msg: 2980 .string "bzero: arguments below kernelbase" 2981.copyin_panic_msg: 2982 .string "copyin: kaddr argument below kernelbase" 2983.xcopyin_panic_msg: 2984 .string "xcopyin: kaddr argument below kernelbase" 2985.copyout_panic_msg: 2986 .string "copyout: kaddr argument below kernelbase" 2987.xcopyout_panic_msg: 2988 .string "xcopyout: kaddr argument below kernelbase" 2989.copystr_panic_msg: 2990 .string "copystr: arguments in user space" 2991.copyinstr_panic_msg: 2992 .string "copyinstr: kaddr argument not in kernel address space" 2993.copyoutstr_panic_msg: 2994 .string "copyoutstr: kaddr argument not in kernel address space" 2995.cpyin_ne_pmsg: 2996 .string "copyin_noerr: argument not in kernel address space" 2997.cpyout_ne_pmsg: 2998 .string "copyout_noerr: argument not in kernel address space" 2999#endif 3000 3001#endif /* __lint */ 3002