1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#pragma ident "%Z%%M% %I% %E% SMI" 28 29#include <sys/param.h> 30#include <sys/errno.h> 31#include <sys/asm_linkage.h> 32#include <sys/vtrace.h> 33#include <sys/machthread.h> 34#include <sys/clock.h> 35#include <sys/asi.h> 36#include <sys/fsr.h> 37#include <sys/privregs.h> 38 39#if !defined(lint) 40#include "assym.h" 41#endif /* lint */ 42 43#define FP_USED 1 44#define LOFAULT_SET 2 45 46/* 47 * Error barrier: 48 * We use membar sync to establish an error barrier for 49 * deferred errors. Membar syncs are added before any update 50 * to t_lofault to ensure that deferred errors from earlier 51 * accesses will not be reported after the membar. This error 52 * isolation is important when we try to recover from async 53 * errors which tries to distinguish kernel accesses to user 54 * data. 55 */ 56 57/* 58 * Zero a block of storage. 59 * 60 * uzero is used by the kernel to zero a block in user address space. 61 */ 62 63#if defined(lint) 64 65/* ARGSUSED */ 66int 67kzero(void *addr, size_t count) 68{ return(0); } 69 70/* ARGSUSED */ 71void 72uzero(void *addr, size_t count) 73{} 74 75#else /* lint */ 76 77 ENTRY(uzero) 78 ! 79 ! Set a new lo_fault handler only if we came in with one 80 ! already specified. 81 ! 82 wr %g0, ASI_USER, %asi 83 ldn [THREAD_REG + T_LOFAULT], %o5 84 tst %o5 85 bz,pt %ncc, .do_zero 86 sethi %hi(.zeroerr), %o2 87 or %o2, %lo(.zeroerr), %o2 88 membar #Sync 89 ba,pt %ncc, .do_zero 90 stn %o2, [THREAD_REG + T_LOFAULT] 91 92 ENTRY(kzero) 93 ! 94 ! Always set a lo_fault handler 95 ! 96 wr %g0, ASI_P, %asi 97 ldn [THREAD_REG + T_LOFAULT], %o5 98 sethi %hi(.zeroerr), %o2 99 or %o5, LOFAULT_SET, %o5 100 or %o2, %lo(.zeroerr), %o2 101 membar #Sync 102 ba,pt %ncc, .do_zero 103 stn %o2, [THREAD_REG + T_LOFAULT] 104 105/* 106 * We got here because of a fault during kzero or if 107 * uzero or bzero was called with t_lofault non-zero. 108 * Otherwise we've already run screaming from the room. 109 * Errno value is in %g1. Note that we're here iff 110 * we did set t_lofault. 111 */ 112.zeroerr: 113 ! 114 ! Undo asi register setting. Just set it to be the 115 ! kernel default without checking. 116 ! 117 wr %g0, ASI_P, %asi 118 ! 119 ! If saved t_lofault has FP_USED set, clear the %fprs register 120 ! 121 btst FP_USED, %o5 122 bz,pt %ncc, 1f ! skip if not used 123 nop 124 membar #Sync 125 wr %g0, %g0, %fprs ! clear fprs 126 andn %o5, FP_USED, %o5 ! turn off flag bit 127 ! 128 ! We did set t_lofault. It may well have been zero coming in. 129 ! 1301: 131 tst %o5 132 membar #Sync 133 bne,pn %ncc, 3f 134 andncc %o5, LOFAULT_SET, %o5 1352: 136 ! 137 ! Old handler was zero. Just return the error. 138 ! 139 retl ! return 140 mov %g1, %o0 ! error code from %g1 1413: 142 ! 143 ! We're here because %o5 was non-zero. It was non-zero 144 ! because either LOFAULT_SET was present, a previous fault 145 ! handler was present or both. In all cases we need to reset 146 ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET 147 ! before we either simply return the error or we invoke the 148 ! previously specified handler. 149 ! 150 be %ncc, 2b 151 stn %o5, [THREAD_REG + T_LOFAULT] 152 jmp %o5 ! goto real handler 153 nop 154 SET_SIZE(kzero) 155 SET_SIZE(uzero) 156 157#endif /* lint */ 158 159/* 160 * Zero a block of storage. 161 */ 162 163#if defined(lint) 164 165/* ARGSUSED */ 166void 167bzero(void *addr, size_t count) 168{} 169 170#else /* lint */ 171 172 ENTRY(bzero) 173 wr %g0, ASI_P, %asi 174 175 ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector 176 tst %o5 177 bz,pt %ncc, .do_zero 178 sethi %hi(.zeroerr), %o2 179 or %o2, %lo(.zeroerr), %o2 180 membar #Sync ! sync error barrier 181 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector 182 183.do_zero: 184 cmp %o1, 15 ! check for small counts 185 blu,pn %ncc, .byteclr ! just clear bytes 186 nop 187 188 cmp %o1, 192 ! check for large counts 189 blu %ncc, .bzero_small 190 nop 191 192 sethi %hi(use_hw_bzero), %o2 193 ld [%o2 + %lo(use_hw_bzero)], %o2 194 tst %o2 195 bz %icc, .bzero_small 196 nop 197 198 rd %fprs, %o2 ! check for unused fp 199 btst FPRS_FEF, %o2 200 bnz %icc, .bzero_small 201 nop 202 203 ldn [THREAD_REG + T_LWP], %o2 204 tst %o2 205 bz,pn %ncc, .bzero_small 206 nop 207 208 ! Check for block alignment 209 btst (64-1), %o0 210 bz %icc, .bzl_block 211 nop 212 213 ! Check for double-word alignment 214 btst (8-1), %o0 215 bz %icc, .bzl_dword 216 nop 217 218 ! Check for word alignment 219 btst (4-1), %o0 220 bz %icc, .bzl_word 221 nop 222 223 ! Clear bytes until word aligned 224.bzl_byte: 225 stba %g0, [%o0]%asi 226 add %o0, 1, %o0 227 btst (4-1), %o0 228 bnz %icc, .bzl_byte 229 sub %o1, 1, %o1 230 231 ! Check for dword-aligned 232 btst (8-1), %o0 233 bz %icc, .bzl_dword 234 nop 235 236 ! Clear words until double-word aligned 237.bzl_word: 238 sta %g0, [%o0]%asi 239 add %o0, 4, %o0 240 btst (8-1), %o0 241 bnz %icc, .bzl_word 242 sub %o1, 4, %o1 243 244.bzl_dword: 245 ! Clear dwords until block aligned 246 stxa %g0, [%o0]%asi 247 add %o0, 8, %o0 248 btst (64-1), %o0 249 bnz %icc, .bzl_dword 250 sub %o1, 8, %o1 251 252.bzl_block: 253 membar #StoreStore|#StoreLoad|#LoadStore 254 wr %g0, FPRS_FEF, %fprs 255 256 ! Set the lower bit in the saved t_lofault to indicate 257 ! that we need to clear the %fprs register on the way 258 ! out 259 or %o5, FP_USED, %o5 260 261 ! Clear block 262 fzero %d0 263 fzero %d2 264 fzero %d4 265 fzero %d6 266 fzero %d8 267 fzero %d10 268 fzero %d12 269 fzero %d14 270 rd %asi, %o3 271 wr %g0, ASI_BLK_P, %asi 272 cmp %o3, ASI_P 273 bne,a %icc, 1f 274 wr %g0, ASI_BLK_AIUS, %asi 2751: 276 mov 256, %o3 277 ba,pt %ncc, .bzl_doblock 278 nop 279 280.bzl_blkstart: 281 ! stda %d0, [%o0+192]%asi ! in dly slot of branch that got us here 282 stda %d0, [%o0+128]%asi 283 stda %d0, [%o0+64]%asi 284 stda %d0, [%o0]%asi 285.bzl_zinst: 286 add %o0, %o3, %o0 287 sub %o1, %o3, %o1 288.bzl_doblock: 289 cmp %o1, 256 290 bgeu,a %ncc, .bzl_blkstart 291 stda %d0, [%o0+192]%asi 292 293 cmp %o1, 64 294 blu %ncc, .bzl_finish 295 296 andn %o1, (64-1), %o3 297 srl %o3, 4, %o2 ! using blocks, 1 instr / 16 words 298 set .bzl_zinst, %o4 299 sub %o4, %o2, %o4 300 jmp %o4 301 nop 302 303.bzl_finish: 304 membar #StoreLoad|#StoreStore 305 wr %g0, %g0, %fprs 306 andn %o5, FP_USED, %o5 307 308 rd %asi, %o4 309 wr %g0, ASI_P, %asi 310 cmp %o4, ASI_BLK_P 311 bne,a %icc, 1f 312 wr %g0, ASI_USER, %asi 3131: 314 315.bzlf_dword: 316 ! double words 317 cmp %o1, 8 318 blu %ncc, .bzlf_word 319 nop 320 stxa %g0, [%o0]%asi 321 add %o0, 8, %o0 322 sub %o1, 8, %o1 323 ba,pt %ncc, .bzlf_dword 324 nop 325 326.bzlf_word: 327 ! words 328 cmp %o1, 4 329 blu %ncc, .bzlf_byte 330 nop 331 sta %g0, [%o0]%asi 332 add %o0, 4, %o0 333 sub %o1, 4, %o1 334 ba,pt %ncc, .bzlf_word 335 nop 336 3371: 338 add %o0, 1, %o0 ! increment address 339.bzlf_byte: 340 subcc %o1, 1, %o1 ! decrement count 341 bgeu,a %ncc, 1b 342 stba %g0, [%o0]%asi ! zero a byte 343 344 ! 345 ! If we used the FP registers, that bit was turned 346 ! off after we were finished. We're just concerned with 347 ! whether t_lofault was set when we came in. We end up 348 ! here from either kzero() or bzero(). kzero() *always* 349 ! sets a lofault handler. It ors LOFAULT_SET into %o5 350 ! to indicate it has done this even if the value of %o5 351 ! is otherwise zero. bzero() sets a lofault handler *only* 352 ! if one was previously set. Accordingly we need to examine 353 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET 354 ! before resetting the error handler. 355 ! 356 tst %o5 357 bz,pt %ncc, 1f 358 andn %o5, LOFAULT_SET, %o5 359 membar #Sync ! sync error barrier 360 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 3611: 362 retl 363 clr %o0 ! return (0) 364 365.bzero_small: 366 367 ! 368 ! Check for word alignment. 369 ! 370 btst 3, %o0 371 bz .bzero_probe 372 mov 0x100, %o3 ! constant size of main loop 373 ! 374 ! 375 ! clear bytes until word aligned 376 ! 3771: stba %g0,[%o0]%asi 378 add %o0, 1, %o0 379 btst 3, %o0 380 bnz 1b 381 sub %o1, 1, %o1 382.bzero_probe: 383 384 ! 385 ! if needed move a word to become double-word aligned. 386 ! 387 btst 7, %o0 ! is double aligned? 388 bz %icc, .bzero_nobuf 389 nop 390 sta %g0, [%o0]%asi ! clr to double boundry 391 sub %o1, 4, %o1 392 ba,pt %ncc, .bzero_nobuf 393 add %o0, 4, %o0 394 395 !stxa %g0, [%o0+0xf8]%asi 396.bzero_blk: 397 stxa %g0, [%o0+0xf0]%asi 398 stxa %g0, [%o0+0xe8]%asi 399 stxa %g0, [%o0+0xe0]%asi 400 stxa %g0, [%o0+0xd8]%asi 401 stxa %g0, [%o0+0xd0]%asi 402 stxa %g0, [%o0+0xc8]%asi 403 stxa %g0, [%o0+0xc0]%asi 404 stxa %g0, [%o0+0xb8]%asi 405 stxa %g0, [%o0+0xb0]%asi 406 stxa %g0, [%o0+0xa8]%asi 407 stxa %g0, [%o0+0xa0]%asi 408 stxa %g0, [%o0+0x98]%asi 409 stxa %g0, [%o0+0x90]%asi 410 stxa %g0, [%o0+0x88]%asi 411 stxa %g0, [%o0+0x80]%asi 412 stxa %g0, [%o0+0x78]%asi 413 stxa %g0, [%o0+0x70]%asi 414 stxa %g0, [%o0+0x68]%asi 415 stxa %g0, [%o0+0x60]%asi 416 stxa %g0, [%o0+0x58]%asi 417 stxa %g0, [%o0+0x50]%asi 418 stxa %g0, [%o0+0x48]%asi 419 stxa %g0, [%o0+0x40]%asi 420 stxa %g0, [%o0+0x38]%asi 421 stxa %g0, [%o0+0x30]%asi 422 stxa %g0, [%o0+0x28]%asi 423 stxa %g0, [%o0+0x20]%asi 424 stxa %g0, [%o0+0x18]%asi 425 stxa %g0, [%o0+0x10]%asi 426 stxa %g0, [%o0+0x08]%asi 427 stxa %g0, [%o0]%asi 428.zinst: 429 add %o0, %o3, %o0 ! increment source address 430 sub %o1, %o3, %o1 ! decrement count 431.bzero_nobuf: 432 cmp %o1, 0x100 ! can we do whole chunk? 433 bgeu,a %ncc, .bzero_blk 434 stxa %g0, [%o0+0xf8]%asi ! do first double of chunk 435 436 cmp %o1, 7 ! can we zero any more double words 437 bleu %ncc, .byteclr ! too small go zero bytes 438 439 andn %o1, 7, %o3 ! %o3 bytes left, double-word aligned 440 srl %o3, 1, %o2 ! using doubles, need 1 instr / 2 words 441 set .zinst, %o4 ! address of clr instructions 442 sub %o4, %o2, %o4 ! jmp address relative to instr 443 jmp %o4 444 nop 445 ! 446 ! do leftover bytes 447 ! 4483: 449 add %o0, 1, %o0 ! increment address 450.byteclr: 451 subcc %o1, 1, %o1 ! decrement count 452 bgeu,a %ncc, 3b 453 stba %g0, [%o0]%asi ! zero a byte 454 455.bzero_finished: 456 ! 457 ! We're just concerned with whether t_lofault was set 458 ! when we came in. We end up here from either kzero() 459 ! or bzero(). kzero() *always* sets a lofault handler. 460 ! It ors LOFAULT_SET into %o5 to indicate it has done 461 ! this even if the value of %o5 is otherwise zero. 462 ! bzero() sets a lofault handler *only* if one was 463 ! previously set. Accordingly we need to examine 464 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET 465 ! before resetting the error handler. 466 ! 467 tst %o5 468 bz %ncc, 1f 469 andn %o5, LOFAULT_SET, %o5 470 membar #Sync ! sync error barrier 471 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 4721: 473 retl 474 clr %o0 ! return (0) 475 476 SET_SIZE(bzero) 477#endif /* lint */ 478