1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#pragma ident "%Z%%M% %I% %E% SMI" 28 29#if !defined(lint) 30#include "assym.h" 31#endif 32 33/* 34 * General assembly language routines. 35 * It is the intent of this file to contain routines that are 36 * specific to cpu architecture. 37 */ 38 39/* 40 * WARNING: If you add a fast trap handler which can be invoked by a 41 * non-privileged user, you may have to use the FAST_TRAP_DONE macro 42 * instead of "done" instruction to return back to the user mode. See 43 * comments for the "fast_trap_done" entry point for more information. 44 */ 45#define FAST_TRAP_DONE \ 46 ba,a fast_trap_done 47 48/* 49 * Override GET_NATIVE_TIME for the cpu module code. This is not 50 * guaranteed to be exactly one instruction, be careful of using 51 * the macro in delay slots. 52 * 53 * Do not use any instruction that modifies condition codes as the 54 * caller may depend on these to remain unchanged across the macro. 55 */ 56 57#define GET_NATIVE_TIME(out, scr1, scr2) \ 58 rd STICK, out 59 60#define RD_TICKCMPR(out, scr) \ 61 rd STICK_COMPARE, out 62 63#define WR_TICKCMPR(in,scr1,scr2,label) \ 64 wr in, STICK_COMPARE 65 66 67#include <sys/clock.h> 68 69#if defined(lint) 70#include <sys/types.h> 71#include <sys/scb.h> 72#include <sys/systm.h> 73#include <sys/regset.h> 74#include <sys/sunddi.h> 75#include <sys/lockstat.h> 76#endif /* lint */ 77 78 79#include <sys/asm_linkage.h> 80#include <sys/privregs.h> 81#include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */ 82#include <sys/machthread.h> 83#include <sys/clock.h> 84#include <sys/intreg.h> 85#include <sys/psr_compat.h> 86#include <sys/isa_defs.h> 87#include <sys/dditypes.h> 88#include <sys/intr.h> 89#include <sys/hypervisor_api.h> 90 91#if !defined(lint) 92#include "assym.h" 93#endif 94 95#define ICACHE_FLUSHSZ 0x20 96 97#if defined(lint) 98/* 99 * Softint generated when counter field of tick reg matches value field 100 * of tick_cmpr reg 101 */ 102/*ARGSUSED*/ 103void 104tickcmpr_set(uint64_t clock_cycles) 105{} 106 107#else /* lint */ 108 109 ENTRY_NP(tickcmpr_set) 110 ! get 64-bit clock_cycles interval 111 mov %o0, %o2 112 mov 8, %o3 ! A reasonable initial step size 1131: 114 WR_TICKCMPR(%o2,%o4,%o5,__LINE__) ! Write to TICK_CMPR 115 116 GET_NATIVE_TIME(%o0, %o4, %o5) ! Read %tick to confirm the 117 sllx %o0, 1, %o0 ! value we wrote was in the future. 118 srlx %o0, 1, %o0 119 120 cmp %o2, %o0 ! If the value we wrote was in the 121 bg,pt %xcc, 2f ! future, then blow out of here. 122 sllx %o3, 1, %o3 ! If not, then double our step size, 123 ba,pt %xcc, 1b ! and take another lap. 124 add %o0, %o3, %o2 ! 1252: 126 retl 127 nop 128 SET_SIZE(tickcmpr_set) 129 130#endif /* lint */ 131 132#if defined(lint) 133 134void 135tickcmpr_disable(void) 136{} 137 138#else 139 140 ENTRY_NP(tickcmpr_disable) 141 mov 1, %g1 142 sllx %g1, TICKINT_DIS_SHFT, %o0 143 WR_TICKCMPR(%o0,%o4,%o5,__LINE__) ! Write to TICK_CMPR 144 retl 145 nop 146 SET_SIZE(tickcmpr_disable) 147 148#endif 149 150#if defined(lint) 151 152/* 153 * tick_write_delta() increments %tick by the specified delta. This should 154 * only be called after a CPR event to assure that gethrtime() continues to 155 * increase monotonically. Obviously, writing %tick needs to de done very 156 * carefully to avoid introducing unnecessary %tick skew across CPUs. For 157 * this reason, we make sure we're i-cache hot before actually writing to 158 * %tick. 159 * 160 * NOTE: No provision for this on sun4v right now. 161 */ 162 163/*ARGSUSED*/ 164void 165tick_write_delta(uint64_t delta) 166{} 167 168#else /* lint */ 169 170 .seg ".text" 171tick_write_delta_panic: 172 .asciz "tick_write_delta: not supported" 173 174 ENTRY_NP(tick_write_delta) 175 sethi %hi(tick_write_delta_panic), %o1 176 save %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller 177 call panic 178 or %i1, %lo(tick_write_delta_panic), %o0 179 /*NOTREACHED*/ 180 retl 181 nop 182#endif 183 184#if defined(lint) 185/* 186 * return 1 if disabled 187 */ 188 189int 190tickcmpr_disabled(void) 191{ return (0); } 192 193#else /* lint */ 194 195 ENTRY_NP(tickcmpr_disabled) 196 RD_TICKCMPR(%g1, %o0) 197 retl 198 srlx %g1, TICKINT_DIS_SHFT, %o0 199 SET_SIZE(tickcmpr_disabled) 200 201#endif /* lint */ 202 203/* 204 * Get current tick 205 */ 206#if defined(lint) 207 208u_longlong_t 209gettick(void) 210{ return (0); } 211 212#else /* lint */ 213 214 ENTRY(gettick) 215 GET_NATIVE_TIME(%o0, %o2, %o3) 216 retl 217 nop 218 SET_SIZE(gettick) 219 220#endif /* lint */ 221 222 223/* 224 * Return the counter portion of the tick register. 225 */ 226 227#if defined(lint) 228 229uint64_t 230gettick_counter(void) 231{ return(0); } 232 233#else /* lint */ 234 235 ENTRY_NP(gettick_counter) 236 rdpr %tick, %o0 237 sllx %o0, 1, %o0 238 retl 239 srlx %o0, 1, %o0 ! shake off npt bit 240 SET_SIZE(gettick_counter) 241#endif /* lint */ 242 243/* 244 * Provide a C callable interface to the trap that reads the hi-res timer. 245 * Returns 64-bit nanosecond timestamp in %o0 and %o1. 246 */ 247 248#if defined(lint) 249 250hrtime_t 251gethrtime(void) 252{ 253 return ((hrtime_t)0); 254} 255 256hrtime_t 257gethrtime_unscaled(void) 258{ 259 return ((hrtime_t)0); 260} 261 262hrtime_t 263gethrtime_max(void) 264{ 265 return ((hrtime_t)0); 266} 267 268void 269scalehrtime(hrtime_t *hrt) 270{ 271 *hrt = 0; 272} 273 274void 275gethrestime(timespec_t *tp) 276{ 277 tp->tv_sec = 0; 278 tp->tv_nsec = 0; 279} 280 281time_t 282gethrestime_sec(void) 283{ 284 return (0); 285} 286 287void 288gethrestime_lasttick(timespec_t *tp) 289{ 290 tp->tv_sec = 0; 291 tp->tv_nsec = 0; 292} 293 294/*ARGSUSED*/ 295void 296hres_tick(void) 297{ 298} 299 300void 301panic_hres_tick(void) 302{ 303} 304 305#else /* lint */ 306 307 ENTRY_NP(gethrtime) 308 GET_HRTIME(%g1, %o0, %o1, %o2, %o3, %o4, %o5, %g2) 309 ! %g1 = hrtime 310 retl 311 mov %g1, %o0 312 SET_SIZE(gethrtime) 313 314 ENTRY_NP(gethrtime_unscaled) 315 GET_NATIVE_TIME(%g1, %o2, %o3) ! %g1 = native time 316 retl 317 mov %g1, %o0 318 SET_SIZE(gethrtime_unscaled) 319 320 ENTRY_NP(gethrtime_waitfree) 321 ALTENTRY(dtrace_gethrtime) 322 GET_NATIVE_TIME(%g1, %o2, %o3) ! %g1 = native time 323 NATIVE_TIME_TO_NSEC(%g1, %o2, %o3) 324 retl 325 mov %g1, %o0 326 SET_SIZE(dtrace_gethrtime) 327 SET_SIZE(gethrtime_waitfree) 328 329 ENTRY(gethrtime_max) 330 NATIVE_TIME_MAX(%g1) 331 NATIVE_TIME_TO_NSEC(%g1, %o0, %o1) 332 333 ! hrtime_t's are signed, max hrtime_t must be positive 334 mov -1, %o2 335 brlz,a %g1, 1f 336 srlx %o2, 1, %g1 3371: 338 retl 339 mov %g1, %o0 340 SET_SIZE(gethrtime_max) 341 342 ENTRY(scalehrtime) 343 ldx [%o0], %o1 344 NATIVE_TIME_TO_NSEC(%o1, %o2, %o3) 345 retl 346 stx %o1, [%o0] 347 SET_SIZE(scalehrtime) 348 349/* 350 * Fast trap to return a timestamp, uses trap window, leaves traps 351 * disabled. Returns a 64-bit nanosecond timestamp in %o0 and %o1. 352 * 353 * This is the handler for the ST_GETHRTIME trap. 354 */ 355 356 ENTRY_NP(get_timestamp) 357 GET_HRTIME(%g1, %g2, %g3, %g4, %g5, %o0, %o1, %o2) ! %g1 = hrtime 358 srlx %g1, 32, %o0 ! %o0 = hi32(%g1) 359 srl %g1, 0, %o1 ! %o1 = lo32(%g1) 360 FAST_TRAP_DONE 361 SET_SIZE(get_timestamp) 362 363/* 364 * Macro to convert GET_HRESTIME() bits into a timestamp. 365 * 366 * We use two separate macros so that the platform-dependent GET_HRESTIME() 367 * can be as small as possible; CONV_HRESTIME() implements the generic part. 368 */ 369#define CONV_HRESTIME(hrestsec, hrestnsec, adj, nslt, nano) \ 370 brz,pt adj, 3f; /* no adjustments, it's easy */ \ 371 add hrestnsec, nslt, hrestnsec; /* hrest.tv_nsec += nslt */ \ 372 brlz,pn adj, 2f; /* if hrestime_adj negative */ \ 373 srl nslt, ADJ_SHIFT, nslt; /* delay: nslt >>= 4 */ \ 374 subcc adj, nslt, %g0; /* hrestime_adj - nslt/16 */ \ 375 movg %xcc, nslt, adj; /* adj by min(adj, nslt/16) */ \ 376 ba 3f; /* go convert to sec/nsec */ \ 377 add hrestnsec, adj, hrestnsec; /* delay: apply adjustment */ \ 3782: addcc adj, nslt, %g0; /* hrestime_adj + nslt/16 */ \ 379 bge,a,pt %xcc, 3f; /* is adj less negative? */ \ 380 add hrestnsec, adj, hrestnsec; /* yes: hrest.nsec += adj */ \ 381 sub hrestnsec, nslt, hrestnsec; /* no: hrest.nsec -= nslt/16 */ \ 3823: cmp hrestnsec, nano; /* more than a billion? */ \ 383 bl,pt %xcc, 4f; /* if not, we're done */ \ 384 nop; /* delay: do nothing :( */ \ 385 add hrestsec, 1, hrestsec; /* hrest.tv_sec++; */ \ 386 sub hrestnsec, nano, hrestnsec; /* hrest.tv_nsec -= NANOSEC; */ \ 3874: 388 389 ENTRY_NP(gethrestime) 390 GET_HRESTIME(%o1, %o2, %o3, %o4, %o5, %g1, %g2, %g3, %g4) 391 CONV_HRESTIME(%o1, %o2, %o3, %o4, %o5) 392 stn %o1, [%o0] 393 retl 394 stn %o2, [%o0 + CLONGSIZE] 395 SET_SIZE(gethrestime) 396 397/* 398 * Similar to gethrestime(), but gethrestime_sec() returns current hrestime 399 * seconds. 400 */ 401 ENTRY_NP(gethrestime_sec) 402 GET_HRESTIME(%o0, %o2, %o3, %o4, %o5, %g1, %g2, %g3, %g4) 403 CONV_HRESTIME(%o0, %o2, %o3, %o4, %o5) 404 retl ! %o0 current hrestime seconds 405 nop 406 SET_SIZE(gethrestime_sec) 407 408/* 409 * Returns the hrestime on the last tick. This is simpler than gethrestime() 410 * and gethrestime_sec(): no conversion is required. gethrestime_lasttick() 411 * follows the same locking algorithm as GET_HRESTIME and GET_HRTIME, 412 * outlined in detail in clock.h. (Unlike GET_HRESTIME/GET_HRTIME, we don't 413 * rely on load dependencies to effect the membar #LoadLoad, instead declaring 414 * it explicitly.) 415 */ 416 ENTRY_NP(gethrestime_lasttick) 417 sethi %hi(hres_lock), %o1 4180: 419 lduw [%o1 + %lo(hres_lock)], %o2 ! Load lock value 420 membar #LoadLoad ! Load of lock must complete 421 andn %o2, 1, %o2 ! Mask off lowest bit 422 ldn [%o1 + %lo(hrestime)], %g1 ! Seconds. 423 add %o1, %lo(hrestime), %o4 424 ldn [%o4 + CLONGSIZE], %g2 ! Nanoseconds. 425 membar #LoadLoad ! All loads must complete 426 lduw [%o1 + %lo(hres_lock)], %o3 ! Reload lock value 427 cmp %o3, %o2 ! If lock is locked or has 428 bne 0b ! changed, retry. 429 stn %g1, [%o0] ! Delay: store seconds 430 retl 431 stn %g2, [%o0 + CLONGSIZE] ! Delay: store nanoseconds 432 SET_SIZE(gethrestime_lasttick) 433 434/* 435 * Fast trap for gettimeofday(). Returns a timestruc_t in %o0 and %o1. 436 * 437 * This is the handler for the ST_GETHRESTIME trap. 438 */ 439 440 ENTRY_NP(get_hrestime) 441 GET_HRESTIME(%o0, %o1, %g1, %g2, %g3, %g4, %g5, %o2, %o3) 442 CONV_HRESTIME(%o0, %o1, %g1, %g2, %g3) 443 FAST_TRAP_DONE 444 SET_SIZE(get_hrestime) 445 446/* 447 * Fast trap to return lwp virtual time, uses trap window, leaves traps 448 * disabled. Returns a 64-bit number in %o0:%o1, which is the number 449 * of nanoseconds consumed. 450 * 451 * This is the handler for the ST_GETHRVTIME trap. 452 * 453 * Register usage: 454 * %o0, %o1 = return lwp virtual time 455 * %o2 = CPU/thread 456 * %o3 = lwp 457 * %g1 = scratch 458 * %g5 = scratch 459 */ 460 ENTRY_NP(get_virtime) 461 GET_NATIVE_TIME(%g5, %g1, %g2) ! %g5 = native time in ticks 462 CPU_ADDR(%g2, %g3) ! CPU struct ptr to %g2 463 ldn [%g2 + CPU_THREAD], %g2 ! thread pointer to %g2 464 ldn [%g2 + T_LWP], %g3 ! lwp pointer to %g3 465 466 /* 467 * Subtract start time of current microstate from time 468 * of day to get increment for lwp virtual time. 469 */ 470 ldx [%g3 + LWP_STATE_START], %g1 ! ms_state_start 471 sub %g5, %g1, %g5 472 473 /* 474 * Add current value of ms_acct[LMS_USER] 475 */ 476 ldx [%g3 + LWP_ACCT_USER], %g1 ! ms_acct[LMS_USER] 477 add %g5, %g1, %g5 478 NATIVE_TIME_TO_NSEC(%g5, %g1, %o0) 479 480 srl %g5, 0, %o1 ! %o1 = lo32(%g5) 481 srlx %g5, 32, %o0 ! %o0 = hi32(%g5) 482 483 FAST_TRAP_DONE 484 SET_SIZE(get_virtime) 485 486 487 488 .seg ".text" 489hrtime_base_panic: 490 .asciz "hrtime_base stepping back" 491 492 493 ENTRY_NP(hres_tick) 494 save %sp, -SA(MINFRAME), %sp ! get a new window 495 496 sethi %hi(hrestime), %l4 497 ldstub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 ! try locking 4987: tst %l5 499 bz,pt %xcc, 8f ! if we got it, drive on 500 ld [%l4 + %lo(nsec_scale)], %l5 ! delay: %l5 = scaling factor 501 ldub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 5029: tst %l5 503 bz,a,pn %xcc, 7b 504 ldstub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 505 ba,pt %xcc, 9b 506 ldub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 5078: 508 membar #StoreLoad|#StoreStore 509 510 ! 511 ! update hres_last_tick. %l5 has the scaling factor (nsec_scale). 512 ! 513 ldx [%l4 + %lo(hrtime_base)], %g1 ! load current hrtime_base 514 GET_NATIVE_TIME(%l0, %l3, %l6) ! current native time 515 stx %l0, [%l4 + %lo(hres_last_tick)]! prev = current 516 ! convert native time to nsecs 517 NATIVE_TIME_TO_NSEC_SCALE(%l0, %l5, %l2, NSEC_SHIFT) 518 519 sub %l0, %g1, %i1 ! get accurate nsec delta 520 521 ldx [%l4 + %lo(hrtime_base)], %l1 522 cmp %l1, %l0 523 bg,pn %xcc, 9f 524 nop 525 526 stx %l0, [%l4 + %lo(hrtime_base)] ! update hrtime_base 527 528 ! 529 ! apply adjustment, if any 530 ! 531 ldx [%l4 + %lo(hrestime_adj)], %l0 ! %l0 = hrestime_adj 532 brz %l0, 2f 533 ! hrestime_adj == 0 ? 534 ! yes, skip adjustments 535 clr %l5 ! delay: set adj to zero 536 tst %l0 ! is hrestime_adj >= 0 ? 537 bge,pt %xcc, 1f ! yes, go handle positive case 538 srl %i1, ADJ_SHIFT, %l5 ! delay: %l5 = adj 539 540 addcc %l0, %l5, %g0 ! hrestime_adj < -adj ? 541 bl,pt %xcc, 2f ! yes, use current adj 542 neg %l5 ! delay: %l5 = -adj 543 ba,pt %xcc, 2f 544 mov %l0, %l5 ! no, so set adj = hrestime_adj 5451: 546 subcc %l0, %l5, %g0 ! hrestime_adj < adj ? 547 bl,a,pt %xcc, 2f ! yes, set adj = hrestime_adj 548 mov %l0, %l5 ! delay: adj = hrestime_adj 5492: 550 ldx [%l4 + %lo(timedelta)], %l0 ! %l0 = timedelta 551 sub %l0, %l5, %l0 ! timedelta -= adj 552 553 stx %l0, [%l4 + %lo(timedelta)] ! store new timedelta 554 stx %l0, [%l4 + %lo(hrestime_adj)] ! hrestime_adj = timedelta 555 556 or %l4, %lo(hrestime), %l2 557 ldn [%l2], %i2 ! %i2:%i3 = hrestime sec:nsec 558 ldn [%l2 + CLONGSIZE], %i3 559 add %i3, %l5, %i3 ! hrestime.nsec += adj 560 add %i3, %i1, %i3 ! hrestime.nsec += nslt 561 562 set NANOSEC, %l5 ! %l5 = NANOSEC 563 cmp %i3, %l5 564 bl,pt %xcc, 5f ! if hrestime.tv_nsec < NANOSEC 565 sethi %hi(one_sec), %i1 ! delay 566 add %i2, 0x1, %i2 ! hrestime.tv_sec++ 567 sub %i3, %l5, %i3 ! hrestime.tv_nsec - NANOSEC 568 mov 0x1, %l5 569 st %l5, [%i1 + %lo(one_sec)] 5705: 571 stn %i2, [%l2] 572 stn %i3, [%l2 + CLONGSIZE] ! store the new hrestime 573 574 membar #StoreStore 575 576 ld [%l4 + %lo(hres_lock)], %i1 577 inc %i1 ! release lock 578 st %i1, [%l4 + %lo(hres_lock)] ! clear hres_lock 579 580 ret 581 restore 582 5839: 584 ! 585 ! release hres_lock 586 ! 587 ld [%l4 + %lo(hres_lock)], %i1 588 inc %i1 589 st %i1, [%l4 + %lo(hres_lock)] 590 591 sethi %hi(hrtime_base_panic), %o0 592 call panic 593 or %o0, %lo(hrtime_base_panic), %o0 594 595 SET_SIZE(hres_tick) 596 597#endif /* lint */ 598 599#if !defined(lint) && !defined(__lint) 600 601 .seg ".text" 602kstat_q_panic_msg: 603 .asciz "kstat_q_exit: qlen == 0" 604 605 ENTRY(kstat_q_panic) 606 save %sp, -SA(MINFRAME), %sp 607 sethi %hi(kstat_q_panic_msg), %o0 608 call panic 609 or %o0, %lo(kstat_q_panic_msg), %o0 610 /*NOTREACHED*/ 611 SET_SIZE(kstat_q_panic) 612 613#define BRZPN brz,pn 614#define BRZPT brz,pt 615 616#define KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \ 617 ld [%o0 + QTYPE/**/CNT], %o1; /* %o1 = old qlen */ \ 618 QOP %o1, 1, %o2; /* %o2 = new qlen */ \ 619 QBR %o1, QZERO; /* done if qlen == 0 */ \ 620 st %o2, [%o0 + QTYPE/**/CNT]; /* delay: save qlen */ \ 621 ldx [%o0 + QTYPE/**/LASTUPDATE], %o3; \ 622 ldx [%o0 + QTYPE/**/TIME], %o4; /* %o4 = old time */ \ 623 ldx [%o0 + QTYPE/**/LENTIME], %o5; /* %o5 = old lentime */ \ 624 sub %g1, %o3, %o2; /* %o2 = time delta */ \ 625 mulx %o1, %o2, %o3; /* %o3 = cur lentime */ \ 626 add %o4, %o2, %o4; /* %o4 = new time */ \ 627 add %o5, %o3, %o5; /* %o5 = new lentime */ \ 628 stx %o4, [%o0 + QTYPE/**/TIME]; /* save time */ \ 629 stx %o5, [%o0 + QTYPE/**/LENTIME]; /* save lentime */ \ 630QRETURN; \ 631 stx %g1, [%o0 + QTYPE/**/LASTUPDATE]; /* lastupdate = now */ 632 633 .align 16 634 ENTRY(kstat_waitq_enter) 635 GET_NATIVE_TIME(%g1, %g2, %g3) 636 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W) 637 SET_SIZE(kstat_waitq_enter) 638 639 .align 16 640 ENTRY(kstat_waitq_exit) 641 GET_NATIVE_TIME(%g1, %g2, %g3) 642 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, retl, KSTAT_IO_W) 643 SET_SIZE(kstat_waitq_exit) 644 645 .align 16 646 ENTRY(kstat_runq_enter) 647 GET_NATIVE_TIME(%g1, %g2, %g3) 648 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R) 649 SET_SIZE(kstat_runq_enter) 650 651 .align 16 652 ENTRY(kstat_runq_exit) 653 GET_NATIVE_TIME(%g1, %g2, %g3) 654 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, retl, KSTAT_IO_R) 655 SET_SIZE(kstat_runq_exit) 656 657 .align 16 658 ENTRY(kstat_waitq_to_runq) 659 GET_NATIVE_TIME(%g1, %g2, %g3) 660 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W) 661 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R) 662 SET_SIZE(kstat_waitq_to_runq) 663 664 .align 16 665 ENTRY(kstat_runq_back_to_waitq) 666 GET_NATIVE_TIME(%g1, %g2, %g3) 667 KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R) 668 KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W) 669 SET_SIZE(kstat_runq_back_to_waitq) 670 671#endif /* lint */ 672 673#ifdef lint 674 675int64_t timedelta; 676hrtime_t hres_last_tick; 677timestruc_t hrestime; 678int64_t hrestime_adj; 679int hres_lock; 680uint_t nsec_scale; 681hrtime_t hrtime_base; 682int traptrace_use_stick; 683 684#else 685 /* 686 * -- WARNING -- 687 * 688 * The following variables MUST be together on a 128-byte boundary. 689 * In addition to the primary performance motivation (having them all 690 * on the same cache line(s)), code here and in the GET*TIME() macros 691 * assumes that they all have the same high 22 address bits (so 692 * there's only one sethi). 693 */ 694 .seg ".data" 695 .global timedelta, hres_last_tick, hrestime, hrestime_adj 696 .global hres_lock, nsec_scale, hrtime_base, traptrace_use_stick 697 .global nsec_shift, adj_shift 698 699 /* XXX - above comment claims 128-bytes is necessary */ 700 .align 64 701timedelta: 702 .word 0, 0 /* int64_t */ 703hres_last_tick: 704 .word 0, 0 /* hrtime_t */ 705hrestime: 706 .nword 0, 0 /* 2 longs */ 707hrestime_adj: 708 .word 0, 0 /* int64_t */ 709hres_lock: 710 .word 0 711nsec_scale: 712 .word 0 713hrtime_base: 714 .word 0, 0 715traptrace_use_stick: 716 .word 0 717nsec_shift: 718 .word NSEC_SHIFT 719adj_shift: 720 .word ADJ_SHIFT 721 722#endif 723 724 725/* 726 * drv_usecwait(clock_t n) [DDI/DKI - section 9F] 727 * usec_delay(int n) [compatibility - should go one day] 728 * Delay by spinning. 729 * 730 * delay for n microseconds. numbers <= 0 delay 1 usec 731 * 732 * With UltraSPARC-III the combination of supporting mixed-speed CPUs 733 * and variable clock rate for power management requires that we 734 * use %stick to implement this routine. 735 */ 736 737#if defined(lint) 738 739/*ARGSUSED*/ 740void 741drv_usecwait(clock_t n) 742{} 743 744/*ARGSUSED*/ 745void 746usec_delay(int n) 747{} 748 749#else /* lint */ 750 751 ENTRY(drv_usecwait) 752 ALTENTRY(usec_delay) 753 brlez,a,pn %o0, 0f 754 mov 1, %o0 7550: 756 sethi %hi(sticks_per_usec), %o1 757 lduw [%o1 + %lo(sticks_per_usec)], %o1 758 mulx %o1, %o0, %o1 ! Scale usec to ticks 759 inc %o1 ! We don't start on a tick edge 760 GET_NATIVE_TIME(%o2, %o3, %o4) 761 add %o1, %o2, %o1 762 7631: cmp %o1, %o2 764 GET_NATIVE_TIME(%o2, %o3, %o4) 765 bgeu,pt %xcc, 1b 766 nop 767 retl 768 nop 769 SET_SIZE(usec_delay) 770 SET_SIZE(drv_usecwait) 771#endif /* lint */ 772 773#if defined(lint) 774 775/* ARGSUSED */ 776void 777pil14_interrupt(int level) 778{} 779 780#else 781 782/* 783 * Level-14 interrupt prologue. 784 */ 785 ENTRY_NP(pil14_interrupt) 786 CPU_ADDR(%g1, %g2) 787 rdpr %pil, %g6 ! %g6 = interrupted PIL 788 stn %g6, [%g1 + CPU_PROFILE_PIL] ! record interrupted PIL 789 rdpr %tstate, %g6 790 rdpr %tpc, %g5 791 btst TSTATE_PRIV, %g6 ! trap from supervisor mode? 792 bnz,a,pt %xcc, 1f 793 stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC 794 stn %g5, [%g1 + CPU_PROFILE_UPC] ! if not, record user PC 795 ba pil_interrupt_common ! must be large-disp branch 796 stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC 7971: ba pil_interrupt_common ! must be large-disp branch 798 stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC 799 SET_SIZE(pil14_interrupt) 800 801 ENTRY_NP(tick_rtt) 802 ! 803 ! Load TICK_COMPARE into %o5; if bit 63 is set, then TICK_COMPARE is 804 ! disabled. If TICK_COMPARE is enabled, we know that we need to 805 ! reenqueue the interrupt request structure. We'll then check TICKINT 806 ! in SOFTINT; if it's set, then we know that we were in a TICK_COMPARE 807 ! interrupt. In this case, TICK_COMPARE may have been rewritten 808 ! recently; we'll compare %o5 to the current time to verify that it's 809 ! in the future. 810 ! 811 ! Note that %o5 is live until after 1f. 812 ! XXX - there is a subroutine call while %o5 is live! 813 ! 814 RD_TICKCMPR(%o5, %g1) 815 srlx %o5, TICKINT_DIS_SHFT, %g1 816 brnz,pt %g1, 2f 817 nop 818 819 rdpr %pstate, %g5 820 andn %g5, PSTATE_IE, %g1 821 wrpr %g0, %g1, %pstate ! Disable vec interrupts 822 823 sethi %hi(cbe_level14_inum), %o1 824 ld [%o1 + %lo(cbe_level14_inum)], %o1 825 call intr_enqueue_req ! preserves %o5 and %g5 826 mov PIL_14, %o0 827 828 ! Check SOFTINT for TICKINT/STICKINT 829 rd SOFTINT, %o4 830 set (TICK_INT_MASK | STICK_INT_MASK), %o0 831 andcc %o4, %o0, %g0 832 bz,a,pn %icc, 2f 833 wrpr %g0, %g5, %pstate ! Enable vec interrupts 834 835 ! clear TICKINT/STICKINT 836 wr %o0, CLEAR_SOFTINT 837 838 ! 839 ! Now that we've cleared TICKINT, we can reread %tick and confirm 840 ! that the value we programmed is still in the future. If it isn't, 841 ! we need to reprogram TICK_COMPARE to fire as soon as possible. 842 ! 843 GET_NATIVE_TIME(%o0, %g1, %g2) ! %o0 = tick 844 sllx %o0, 1, %o0 ! Clear the DIS bit 845 srlx %o0, 1, %o0 846 cmp %o5, %o0 ! In the future? 847 bg,a,pt %xcc, 2f ! Yes, drive on. 848 wrpr %g0, %g5, %pstate ! delay: enable vec intr 849 850 ! 851 ! If we're here, then we have programmed TICK_COMPARE with a %tick 852 ! which is in the past; we'll now load an initial step size, and loop 853 ! until we've managed to program TICK_COMPARE to fire in the future. 854 ! 855 mov 8, %o4 ! 8 = arbitrary inital step 8561: add %o0, %o4, %o5 ! Add the step 857 WR_TICKCMPR(%o5,%g1,%g2,__LINE__) ! Write to TICK_CMPR 858 GET_NATIVE_TIME(%o0, %g1, %g2) ! %o0 = tick 859 sllx %o0, 1, %o0 ! Clear the DIS bit 860 srlx %o0, 1, %o0 861 cmp %o5, %o0 ! In the future? 862 bg,a,pt %xcc, 2f ! Yes, drive on. 863 wrpr %g0, %g5, %pstate ! delay: enable vec intr 864 ba 1b ! No, try again. 865 sllx %o4, 1, %o4 ! delay: double step size 866 8672: ba current_thread_complete 868 nop 869 SET_SIZE(tick_rtt) 870 871#endif /* lint */ 872 873#if defined(lint) 874/* 875 * Prefetch a page_t for write or read, this assumes a linear 876 * scan of sequential page_t's. 877 */ 878/*ARGSUSED*/ 879void 880prefetch_page_w(void *pp) 881{} 882 883/*ARGSUSED*/ 884void 885prefetch_page_r(void *pp) 886{} 887#else /* lint */ 888 889/* XXXQ These should be inline templates, not functions */ 890 ENTRY(prefetch_page_w) 891 retl 892 nop 893 SET_SIZE(prefetch_page_w) 894 895 ENTRY(prefetch_page_r) 896 retl 897 nop 898 SET_SIZE(prefetch_page_r) 899 900#endif /* lint */ 901 902#if defined(lint) 903/* 904 * Prefetch struct smap for write. 905 */ 906/*ARGSUSED*/ 907void 908prefetch_smap_w(void *smp) 909{} 910#else /* lint */ 911 912/* XXXQ These should be inline templates, not functions */ 913 ENTRY(prefetch_smap_w) 914 retl 915 nop 916 SET_SIZE(prefetch_smap_w) 917 918#endif /* lint */ 919 920/* 921 * Generic sun4v MMU and Cache operations. 922 */ 923 924#if defined(lint) 925 926/*ARGSUSED*/ 927void 928vtag_flushpage(caddr_t vaddr, uint_t ctxnum) 929{} 930 931/*ARGSUSED*/ 932void 933vtag_flushctx(uint_t ctxnum) 934{} 935 936/*ARGSUSED*/ 937void 938vtag_flushall(void) 939{} 940 941/*ARGSUSED*/ 942void 943vtag_unmap_perm_tl1(uint64_t vaddr, uint64_t ctxnum) 944{} 945 946/*ARGSUSED*/ 947void 948vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) 949{} 950 951/*ARGSUSED*/ 952void 953vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) 954{} 955 956/*ARGSUSED*/ 957void 958vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy) 959{} 960 961/*ARGSUSED*/ 962void 963vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2) 964{} 965 966/*ARGSUSED*/ 967void 968vac_flushpage(pfn_t pfnum, int vcolor) 969{} 970 971/*ARGSUSED*/ 972void 973vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor) 974{} 975 976/*ARGSUSED*/ 977void 978flush_instr_mem(caddr_t vaddr, size_t len) 979{} 980 981#else /* lint */ 982 983 ENTRY_NP(vtag_flushpage) 984 /* 985 * flush page from the tlb 986 * 987 * %o0 = vaddr 988 * %o1 = ctxnum 989 */ 990 mov MAP_ITLB | MAP_DTLB, %o2 991 ta MMU_UNMAP_ADDR 992 brz,pt %o0, 1f 993 nop 994 ba panic_bad_hcall 995 mov MMU_UNMAP_ADDR, %o1 9961: 997 retl 998 nop 999 SET_SIZE(vtag_flushpage) 1000 1001 ENTRY_NP(vtag_flushctx) 1002 /* 1003 * flush context from the tlb 1004 * 1005 * %o0 = ctxnum 1006 */ 1007 mov %o0, %o2 1008 mov %g0, %o0 ! XXXQ no cpu list yet 1009 mov %g0, %o1 ! XXXQ no cpu list yet 1010 mov MAP_ITLB | MAP_DTLB, %o3 1011 mov MMU_DEMAP_CTX, %o5 1012 ta FAST_TRAP 1013 brz,pt %o0, 1f 1014 nop 1015 ba panic_bad_hcall 1016 mov MMU_DEMAP_CTX, %o1 10171: 1018 retl 1019 nop 1020 SET_SIZE(vtag_flushctx) 1021 1022 ENTRY_NP(vtag_flushall) 1023 mov %g0, %o0 ! XXX no cpu list yet 1024 mov %g0, %o1 ! XXX no cpu list yet 1025 mov MAP_ITLB | MAP_DTLB, %o2 1026 mov MMU_DEMAP_ALL, %o5 1027 ta FAST_TRAP 1028 brz,pt %o0, 1f 1029 nop 1030 ba panic_bad_hcall 1031 mov MMU_DEMAP_ALL, %o1 10321: 1033 retl 1034 nop 1035 SET_SIZE(vtag_flushall) 1036 1037 ENTRY_NP(vtag_unmap_perm_tl1) 1038 /* 1039 * x-trap to unmap perm map entry 1040 * %g1 = vaddr 1041 * %g2 = ctxnum 1042 */ 1043 mov %o0, %g3 1044 mov %o1, %g4 1045 mov %o2, %g5 1046 mov %o5, %g6 1047 mov %g1, %o0 1048 mov %g2, %o1 1049 mov MAP_ITLB | MAP_DTLB, %o2 1050 mov UNMAP_PERM_ADDR, %o5 1051 ta FAST_TRAP 1052 brz,pt %o0, 1f 1053 nop 1054 ba ptl1_panic 1055 mov PTL1_BAD_HCALL, %g1 10561: 1057 mov %g6, %o5 1058 mov %g5, %o2 1059 mov %g4, %o1 1060 mov %g3, %o0 1061 retry 1062 SET_SIZE(vtag_unmap_perm_tl1) 1063 1064 ENTRY_NP(vtag_flushpage_tl1) 1065 /* 1066 * x-trap to flush page from tlb and tsb 1067 * 1068 * %g1 = vaddr, zero-extended on 32-bit kernel 1069 * %g2 = ctxnum 1070 * 1071 * assumes TSBE_TAG = 0 1072 */ 1073 srln %g1, MMU_PAGESHIFT, %g1 1074 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ 1075 mov %o0, %g3 1076 mov %o1, %g4 1077 mov %o2, %g5 1078 mov %g1, %o0 ! vaddr 1079 mov %g2, %o1 ! ctx 1080 mov MAP_ITLB | MAP_DTLB, %o2 1081 ta MMU_UNMAP_ADDR 1082 brz,pt %o0, 1f 1083 nop 1084 ba ptl1_panic 1085 mov PTL1_BAD_HCALL, %g1 10861: 1087 mov %g5, %o2 1088 mov %g4, %o1 1089 mov %g3, %o0 1090 membar #Sync 1091 retry 1092 SET_SIZE(vtag_flushpage_tl1) 1093 1094 ENTRY_NP(vtag_flush_pgcnt_tl1) 1095 /* 1096 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb 1097 * 1098 * %g1 = vaddr, zero-extended on 32-bit kernel 1099 * %g2 = <zero32|ctx16|pgcnt16> 1100 * 1101 * NOTE: this handler relies on the fact that no 1102 * interrupts or traps can occur during the loop 1103 * issuing the TLB_DEMAP operations. It is assumed 1104 * that interrupts are disabled and this code is 1105 * fetching from the kernel locked text address. 1106 * 1107 * assumes TSBE_TAG = 0 1108 */ 1109 srln %g1, MMU_PAGESHIFT, %g1 1110 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ 1111 mov %o0, %g3 1112 mov %o1, %g4 1113 mov %o2, %g5 1114 1115 set 0xffff, %g6 1116 and %g6, %g2, %g7 /* g7 = pgcnt */ 1117 srln %g2, 16, %g2 /* g2 = ctxnum */ 1118 1119 set MMU_PAGESIZE, %g6 /* g2 = pgsize */ 11201: 1121 mov %g1, %o0 ! vaddr 1122 mov %g2, %o1 ! ctx 1123 mov MAP_ITLB | MAP_DTLB, %o2 1124 ta MMU_UNMAP_ADDR 1125 brz,pt %o0, 2f 1126 nop 1127 ba ptl1_panic 1128 mov PTL1_BAD_HCALL, %g1 11292: 1130 deccc %g7 /* decr pgcnt */ 1131 bnz,pt %icc,1b 1132 add %g1, %g6, %g1 /* go to nextpage */ 1133 1134 mov %g5, %o2 1135 mov %g4, %o1 1136 mov %g3, %o0 1137 membar #Sync 1138 retry 1139 SET_SIZE(vtag_flush_pgcnt_tl1) 1140 1141 ENTRY_NP(vtag_flushctx_tl1) 1142 /* 1143 * x-trap to flush context from tlb 1144 * 1145 * %g1 = ctxnum 1146 */ 1147 mov %o0, %g3 1148 mov %o1, %g4 1149 mov %o2, %g5 1150 mov %o3, %g6 1151 mov %o5, %g7 1152 mov %g1, %o2 1153 mov %g0, %o0 ! XXXQ no cpu list yet 1154 mov %g0, %o1 ! XXXQ no cpu list yet 1155 mov MAP_ITLB | MAP_DTLB, %o3 1156 mov MMU_DEMAP_CTX, %o5 1157 ta FAST_TRAP 1158 brz,pt %o0, 1f 1159 nop 1160 ba ptl1_panic 1161 mov PTL1_BAD_HCALL, %g1 11621: 1163 mov %g7, %o5 1164 mov %g6, %o3 1165 mov %g5, %o2 1166 mov %g4, %o1 1167 mov %g3, %o0 1168 membar #Sync 1169 retry 1170 SET_SIZE(vtag_flushctx_tl1) 1171 1172 ! Not implemented on US1/US2 1173 ENTRY_NP(vtag_flushall_tl1) 1174 mov %o0, %g3 1175 mov %o1, %g4 1176 mov %o2, %g5 1177 mov %o3, %g6 ! XXXQ not used? 1178 mov %o5, %g7 1179 mov %g0, %o0 ! XXX no cpu list yet 1180 mov %g0, %o1 ! XXX no cpu list yet 1181 mov MAP_ITLB | MAP_DTLB, %o2 1182 mov MMU_DEMAP_ALL, %o5 1183 ta FAST_TRAP 1184 brz,pt %o0, 1f 1185 nop 1186 ba ptl1_panic 1187 mov PTL1_BAD_HCALL, %g1 11881: 1189 mov %g7, %o5 1190 mov %g6, %o3 ! XXXQ not used? 1191 mov %g5, %o2 1192 mov %g4, %o1 1193 mov %g3, %o0 1194 retry 1195 SET_SIZE(vtag_flushall_tl1) 1196 1197/* 1198 * vac_flushpage(pfnum, color) 1199 * Flush 1 8k page of the D-$ with physical page = pfnum 1200 * Algorithm: 1201 * The spitfire dcache is a 16k direct mapped virtual indexed, 1202 * physically tagged cache. Given the pfnum we read all cache 1203 * lines for the corresponding page in the cache (determined by 1204 * the color). Each cache line is compared with 1205 * the tag created from the pfnum. If the tags match we flush 1206 * the line. 1207 */ 1208 ENTRY(vac_flushpage) 1209 /* 1210 * flush page from the d$ 1211 * 1212 * %o0 = pfnum, %o1 = color 1213 */ 1214 ! XXXQ 1215 retl 1216 nop 1217 SET_SIZE(vac_flushpage) 1218 1219 ENTRY_NP(vac_flushpage_tl1) 1220 /* 1221 * x-trap to flush page from the d$ 1222 * 1223 * %g1 = pfnum, %g2 = color 1224 */ 1225 ! XXXQ 1226 retry 1227 SET_SIZE(vac_flushpage_tl1) 1228 1229 ENTRY(vac_flushcolor) 1230 /* 1231 * %o0 = vcolor 1232 */ 1233 ! XXXQ 1234 retl 1235 nop 1236 SET_SIZE(vac_flushcolor) 1237 1238 ENTRY(vac_flushcolor_tl1) 1239 /* 1240 * %g1 = vcolor 1241 */ 1242 ! XXXQ 1243 retry 1244 SET_SIZE(vac_flushcolor_tl1) 1245 1246/* 1247 * flush_instr_mem: 1248 * Flush a portion of the I-$ starting at vaddr 1249 * %o0 vaddr 1250 * %o1 bytes to be flushed 1251 */ 1252 1253 ENTRY(flush_instr_mem) 1254 membar #StoreStore ! Ensure the stores 1255 ! are globally visible 12561: 1257 flush %o0 1258 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20 1259 bgu,pt %ncc, 1b 1260 add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20 1261 1262 retl 1263 nop 1264 SET_SIZE(flush_instr_mem) 1265 1266#endif /* !lint */ 1267