/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#if defined(lint)
#include <sys/types.h>
#include <sys/thread.h>
#include <sys/cpuvar.h>
#else	/* lint */
#include "assym.h"
#endif	/* lint */

#include <sys/t_lock.h>
#include <sys/mutex.h>
#include <sys/mutex_impl.h>
#include <sys/rwlock_impl.h>
#include <sys/asm_linkage.h>
#include <sys/machlock.h>
#include <sys/machthread.h>
#include <sys/lockstat.h>

/* #define DEBUG */

#ifdef DEBUG
#include <sys/machparam.h>
#endif /* DEBUG */

/************************************************************************
 *		ATOMIC OPERATIONS
 */

/*
 * uint8_t	ldstub(uint8_t *cp)
 *
 * Store 0xFF at the specified location, and return its previous content.
 */

#if defined(lint)
uint8_t
ldstub(uint8_t *cp)
{
	uint8_t	rv;
	rv = *cp;
	*cp = 0xFF;
	return rv;
}
#else	/* lint */

	ENTRY(ldstub)
	retl
	ldstub	[%o0], %o0
	SET_SIZE(ldstub)

#endif	/* lint */

/************************************************************************
 *		MEMORY BARRIERS -- see atomic.h for full descriptions.
 */

#if defined(lint)

void
membar_enter(void)
{}

void
membar_exit(void)
{}

void
membar_producer(void)
{}

void
membar_consumer(void)
{}

#else	/* lint */

#ifdef SF_ERRATA_51
	.align 32
	ENTRY(membar_return)
	retl
	nop
	SET_SIZE(membar_return)
#define	MEMBAR_RETURN	ba,pt %icc, membar_return
#else
#define	MEMBAR_RETURN	retl
#endif

	ENTRY(membar_enter)
	MEMBAR_RETURN
	membar	#StoreLoad|#StoreStore
	SET_SIZE(membar_enter)

	ENTRY(membar_exit)
	MEMBAR_RETURN
	membar	#LoadStore|#StoreStore
	SET_SIZE(membar_exit)

	ENTRY(membar_producer)
	MEMBAR_RETURN
	membar	#StoreStore
	SET_SIZE(membar_producer)

	ENTRY(membar_consumer)
	MEMBAR_RETURN
	membar	#LoadLoad
	SET_SIZE(membar_consumer)

#endif	/* lint */

/************************************************************************
 *		MINIMUM LOCKS
 */

#if defined(lint)

/*
 * lock_try(lp), ulock_try(lp)
 *	- returns non-zero on success.
 *	- doesn't block interrupts so don't use this to spin on a lock.
 *	- uses "0xFF is busy, anything else is free" model.
 *
 *      ulock_try() is for a lock in the user address space.
 *      For all V7/V8 sparc systems they are same since the kernel and
 *      user are mapped in a user' context.
 *      For V9 platforms the lock_try and ulock_try are different impl.
 */

int
lock_try(lock_t *lp)
{
	return (0xFF ^ ldstub(lp));
}

int
lock_spin_try(lock_t *lp)
{
	return (0xFF ^ ldstub(lp));
}

void
lock_set(lock_t *lp)
{
	extern void lock_set_spin(lock_t *);

	if (!lock_try(lp))
		lock_set_spin(lp);
	membar_enter();
}

void
lock_clear(lock_t *lp)
{
	membar_exit();
	*lp = 0;
}

int
ulock_try(lock_t *lp)
{
	return (0xFF ^ ldstub(lp));
}

void
ulock_clear(lock_t *lp)
{
	membar_exit();
	*lp = 0;
}

#else	/* lint */

	.align	32
	ENTRY(lock_try)
	ldstub	[%o0], %o1		! try to set lock, get value in %o1
	brnz,pn	%o1, 1f
	membar	#LoadLoad
.lock_try_lockstat_patch_point:
	retl
	or	%o0, 1, %o0		! ensure lo32 != 0
1:
	retl
	clr	%o0
	SET_SIZE(lock_try)

	.align	32
	ENTRY(lock_spin_try)
	ldstub	[%o0], %o1		! try to set lock, get value in %o1
	brnz,pn	%o1, 1f
	membar	#LoadLoad
	retl
	or	%o0, 1, %o0		! ensure lo32 != 0
1:
	retl
	clr	%o0
	SET_SIZE(lock_spin_try)

	.align	32
	ENTRY(lock_set)
	ldstub	[%o0], %o1
	brnz,pn	%o1, 1f			! go to C for the hard case
	membar	#LoadLoad
.lock_set_lockstat_patch_point:
	retl
	nop
1:
	sethi	%hi(lock_set_spin), %o2	! load up for jump to C
	jmp	%o2 + %lo(lock_set_spin)
	nop				! delay: do nothing
	SET_SIZE(lock_set)

	ENTRY(lock_clear)
	membar	#LoadStore|#StoreStore
.lock_clear_lockstat_patch_point:
	retl
	clrb	[%o0]
	SET_SIZE(lock_clear)

	.align	32
	ENTRY(ulock_try)
	ldstuba	[%o0]ASI_USER, %o1	! try to set lock, get value in %o1
	xor	%o1, 0xff, %o0		! delay - return non-zero if success
	retl
	  membar	#LoadLoad
	SET_SIZE(ulock_try)

	ENTRY(ulock_clear)
	membar  #LoadStore|#StoreStore
	retl
	  stba	%g0, [%o0]ASI_USER	! clear lock
	SET_SIZE(ulock_clear)

#endif	/* lint */


/*
 * lock_set_spl(lp, new_pil, *old_pil_addr)
 * 	Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr.
 */

#if defined(lint)

/* ARGSUSED */
void
lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil_addr)
{
	extern int splr(int);
	extern void lock_set_spl_spin(lock_t *, int, u_short *, int);
	int old_pil;

	old_pil = splr(new_pil);
	if (!lock_try(lp)) {
		lock_set_spl_spin(lp, new_pil, old_pil_addr, old_pil);
	} else {
		*old_pil_addr = (u_short)old_pil;
		membar_enter();
	}
}

#else	/* lint */

	ENTRY(lock_set_spl)
	rdpr	%pil, %o3			! %o3 = current pil
	cmp	%o3, %o1			! is current pil high enough?
	bl,a,pt %icc, 1f			! if not, write %pil in delay
	wrpr	%g0, %o1, %pil
1:
	ldstub	[%o0], %o4			! try the lock
	brnz,pn	%o4, 2f				! go to C for the miss case
	membar	#LoadLoad
.lock_set_spl_lockstat_patch_point:
	retl
	sth	%o3, [%o2]			! delay - save original pil
2:
	sethi	%hi(lock_set_spl_spin), %o5	! load up jmp to C
	jmp	%o5 + %lo(lock_set_spl_spin)	! jmp to lock_set_spl_spin
	nop					! delay: do nothing
	SET_SIZE(lock_set_spl)

#endif	/* lint */

/*
 * lock_clear_splx(lp, s)
 */

#if defined(lint)

void
lock_clear_splx(lock_t *lp, int s)
{
	extern void splx(int);

	lock_clear(lp);
	splx(s);
}

#else	/* lint */

	ENTRY(lock_clear_splx)
	ldn	[THREAD_REG + T_CPU], %o2	! get CPU pointer
	membar	#LoadStore|#StoreStore
	ld	[%o2 + CPU_BASE_SPL], %o2
	clrb	[%o0]				! clear lock
	cmp	%o2, %o1			! compare new to base
	movl	%xcc, %o1, %o2			! use new pri if base is less
.lock_clear_splx_lockstat_patch_point:
	retl
	wrpr	%g0, %o2, %pil
	SET_SIZE(lock_clear_splx)

#endif	/* lint */

/*
 * mutex_enter() and mutex_exit().
 * 
 * These routines handle the simple cases of mutex_enter() (adaptive
 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
 * If anything complicated is going on we punt to mutex_vector_enter().
 *
 * mutex_tryenter() is similar to mutex_enter() but returns zero if
 * the lock cannot be acquired, nonzero on success.
 *
 * If mutex_exit() gets preempted in the window between checking waiters
 * and clearing the lock, we can miss wakeups.  Disabling preemption
 * in the mutex code is prohibitively expensive, so instead we detect
 * mutex preemption by examining the trapped PC in the interrupt path.
 * If we interrupt a thread in mutex_exit() that has not yet cleared
 * the lock, pil_interrupt() resets its PC back to the beginning of
 * mutex_exit() so it will check again for waiters when it resumes.
 *
 * The lockstat code below is activated when the lockstat driver
 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
 * Note that we don't need to test lockstat_event_mask here -- we won't
 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
 */

#if defined (lint)

/* ARGSUSED */
void
mutex_enter(kmutex_t *lp)
{}

/* ARGSUSED */
int
mutex_tryenter(kmutex_t *lp)
{ return (0); }

/* ARGSUSED */
void
mutex_exit(kmutex_t *lp)
{}

#else
	.align	32
	ENTRY(mutex_enter)
	mov	THREAD_REG, %o1
	casx	[%o0], %g0, %o1			! try to acquire as adaptive
	brnz,pn	%o1, 1f				! locked or wrong type
	membar	#LoadLoad
.mutex_enter_lockstat_patch_point:
	retl
	nop
1:
	sethi	%hi(mutex_vector_enter), %o2	! load up for jump to C
	jmp	%o2 + %lo(mutex_vector_enter)
	nop
	SET_SIZE(mutex_enter)

	ENTRY(mutex_tryenter)
	mov	THREAD_REG, %o1
	casx	[%o0], %g0, %o1			! try to acquire as adaptive
	brnz,pn	%o1, 1f				! locked or wrong type continue
	membar	#LoadLoad
.mutex_tryenter_lockstat_patch_point:
	retl
	or	%o0, 1, %o0			! ensure lo32 != 0
1:
	sethi	%hi(mutex_vector_tryenter), %o2		! hi bits
	jmp	%o2 + %lo(mutex_vector_tryenter)	! go to C
	nop
	SET_SIZE(mutex_tryenter)

	ENTRY(mutex_adaptive_tryenter)
	mov	THREAD_REG, %o1
	casx	[%o0], %g0, %o1			! try to acquire as adaptive
	brnz,pn	%o1, 0f				! locked or wrong type
	membar	#LoadLoad
	retl
	or	%o0, 1, %o0			! ensure lo32 != 0
0:
	retl
	mov	%g0, %o0
	SET_SIZE(mutex_adaptive_tryenter)

	.global	mutex_exit_critical_size
	.global	mutex_exit_critical_start

mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start

	.align	32

	ENTRY(mutex_exit)
mutex_exit_critical_start:		! If we are interrupted, restart here
	ldn	[%o0], %o1		! get the owner field
	membar	#LoadStore|#StoreStore
	cmp	THREAD_REG, %o1		! do we own lock with no waiters?
	be,a,pt	%ncc, 1f		! if so, drive on ...
	stn	%g0, [%o0]		! delay: clear lock if we owned it
.mutex_exit_critical_end:		! for pil_interrupt() hook
	ba,a,pt	%xcc, mutex_vector_exit	! go to C for the hard cases
1:
.mutex_exit_lockstat_patch_point:
	retl
	nop
	SET_SIZE(mutex_exit)

#endif	/* lint */

/*
 * rw_enter() and rw_exit().
 * 
 * These routines handle the simple cases of rw_enter (write-locking an unheld
 * lock or read-locking a lock that's neither write-locked nor write-wanted)
 * and rw_exit (no waiters or not the last reader).  If anything complicated
 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
 */
#if defined(lint)

/* ARGSUSED */
void
rw_enter(krwlock_t *lp, krw_t rw)
{}

/* ARGSUSED */
void
rw_exit(krwlock_t *lp)
{}

#else

	.align	16
	ENTRY(rw_enter)
	cmp	%o1, RW_WRITER			! entering as writer?
	be,a,pn	%icc, 2f			! if so, go do it ...
	or	THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
	ld	[THREAD_REG + T_KPRI_REQ], %o3	! begin THREAD_KPRI_REQUEST()
	ldn	[%o0], %o4			! %o4 = old lock value
	inc	%o3				! bump kpri
	st	%o3, [THREAD_REG + T_KPRI_REQ]	! store new kpri
1:
	andcc	%o4, RW_WRITE_CLAIMED, %g0	! write-locked or write-wanted?
	bz,pt	%xcc, 3f	 		! if so, prepare to block
	add	%o4, RW_READ_LOCK, %o5		! delay: increment hold count
	sethi	%hi(rw_enter_sleep), %o2	! load up jump
	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
	nop					! delay: do nothing
3:
	casx	[%o0], %o4, %o5			! try to grab read lock
	cmp	%o4, %o5			! did we get it?
	bne,pn	%xcc, 1b			! if not, try again
	mov	%o5, %o4			! delay: %o4 = old lock value
	membar	#LoadLoad
.rw_read_enter_lockstat_patch_point:
	retl
	nop
2:
	casx	[%o0], %g0, %o5			! try to grab write lock
	brz,pt %o5, 4f				! branch around if we got it
	membar	#LoadLoad			! done regardless of where we go
	sethi	%hi(rw_enter_sleep), %o2
	jmp	%o2 + %lo(rw_enter_sleep)	! jump to rw_enter_sleep if not
	nop					! delay: do nothing
4:
.rw_write_enter_lockstat_patch_point:
	retl
	nop
	SET_SIZE(rw_enter)

	.align	16
	ENTRY(rw_exit)
	ldn	[%o0], %o4			! %o4 = old lock value
	membar	#LoadStore|#StoreStore		! membar_exit()
	subcc	%o4, RW_READ_LOCK, %o5		! %o5 = new lock value if reader
	bnz,pn	%xcc, 2f			! single reader, no waiters?
	clr	%o1
1:
	ld	[THREAD_REG + T_KPRI_REQ], %g1	! begin THREAD_KPRI_RELEASE()
	srl	%o4, RW_HOLD_COUNT_SHIFT, %o3	! %o3 = hold count (lockstat)
	casx	[%o0], %o4, %o5			! try to drop lock
	cmp	%o4, %o5			! did we succeed?
	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
	dec	%g1				! delay: drop kpri
.rw_read_exit_lockstat_patch_point:
	retl
	st	%g1, [THREAD_REG + T_KPRI_REQ]	! delay: store new kpri
2:
	andcc	%o4, RW_WRITE_LOCKED, %g0	! are we a writer?
	bnz,a,pt %xcc, 3f
	or	THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
	cmp	%o5, RW_READ_LOCK		! would lock still be held?
	bge,pt	%xcc, 1b			! if so, go ahead and drop it
	nop
	ba,pt	%xcc, rw_exit_wakeup		! otherwise, wake waiters
	nop
3:
	casx	[%o0], %o4, %o1			! try to drop write lock
	cmp	%o4, %o1			! did we succeed?
	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
	nop
.rw_write_exit_lockstat_patch_point:
	retl
	nop
	SET_SIZE(rw_exit)

#endif

#if defined(lint)

void
lockstat_hot_patch(void)
{}

#else

#define	RETL			0x81c3e008
#define	NOP			0x01000000
#define BA			0x10800000

#define	DISP22			((1 << 22) - 1)
#define	ANNUL			0x20000000

#define	HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs)		\
	ba	1f;							\
	rd	%pc, %o0;						\
	save	%sp, -SA(MINFRAME), %sp;				\
	set	lockstat_probemap, %l1;					\
	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
	brz,pn	%o0, 0f;						\
	ldub	[THREAD_REG + T_LOCKSTAT], %l0;				\
	add	%l0, 1, %l2;						\
	stub	%l2, [THREAD_REG + T_LOCKSTAT];				\
	set	lockstat_probe, %g1;					\
	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
	brz,a,pn %o0, 0f;						\
	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
	ldn	[%g1], %g2;						\
	mov	rs, %o2;						\
	jmpl	%g2, %o7;						\
	mov	%i0, %o1;						\
	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
0:	ret;								\
	restore	%g0, 1, %o0;	/* for mutex_tryenter / lock_try */	\
1:	set	addr, %o1;						\
	sub	%o0, %o1, %o0;						\
	srl	%o0, 2, %o0;						\
	inc	%o0;							\
	set	DISP22, %o1;						\
	and	%o1, %o0, %o0;						\
	set	BA, %o1;						\
	or	%o1, %o0, %o0;						\
	sethi	%hi(annul), %o2;					\
	add	%o0, %o2, %o2;						\
	set	addr, %o0;						\
	set	normal_instr, %o1;					\
	ld	[%i0 + (event * DTRACE_IDSIZE)], %o3;			\
	tst	%o3;							\
	movnz	%icc, %o2, %o1;						\
	call	hot_patch_kernel_text;					\
	mov	4, %o2;							\
	membar	#Sync

#define	HOT_PATCH(addr, event, normal_instr)	\
	HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1)

#define	HOT_PATCH_ARG(addr, event, normal_instr, arg)	\
	HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg)

#define HOT_PATCH_ANNULLED(addr, event, normal_instr)	\
	HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1)

	ENTRY(lockstat_hot_patch)
	save	%sp, -SA(MINFRAME), %sp
	set	lockstat_probemap, %i0
	HOT_PATCH(.mutex_enter_lockstat_patch_point,
		LS_MUTEX_ENTER_ACQUIRE, RETL)
	HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point,
		LS_MUTEX_TRYENTER_ACQUIRE, RETL)
	HOT_PATCH(.mutex_exit_lockstat_patch_point,
		LS_MUTEX_EXIT_RELEASE, RETL)
	HOT_PATCH(.rw_write_enter_lockstat_patch_point,
		LS_RW_ENTER_ACQUIRE, RETL)
	HOT_PATCH(.rw_read_enter_lockstat_patch_point,
		LS_RW_ENTER_ACQUIRE, RETL)
	HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point,
		LS_RW_EXIT_RELEASE, RETL, RW_WRITER)
	HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point,
		LS_RW_EXIT_RELEASE, RETL, RW_READER)
	HOT_PATCH(.lock_set_lockstat_patch_point,
		LS_LOCK_SET_ACQUIRE, RETL)
	HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point,
		LS_LOCK_TRY_ACQUIRE, RETL)
	HOT_PATCH(.lock_clear_lockstat_patch_point,
		LS_LOCK_CLEAR_RELEASE, RETL)
	HOT_PATCH(.lock_set_spl_lockstat_patch_point,
		LS_LOCK_SET_SPL_ACQUIRE, RETL)
	HOT_PATCH(.lock_clear_splx_lockstat_patch_point,
		LS_LOCK_CLEAR_SPLX_RELEASE, RETL)
	ret
	restore
	SET_SIZE(lockstat_hot_patch)

#endif	/* lint */

/*
 * asm_mutex_spin_enter(mutex_t *)
 *
 * For use by assembly interrupt handler only.
 * Does not change spl, since the interrupt handler is assumed to be
 * running at high level already.
 * Traps may be off, so cannot panic.
 * Does not keep statistics on the lock.
 *
 * Entry:	%l6 - points to mutex
 * 		%l7 - address of call (returns to %l7+8)
 * Uses:	%l6, %l5
 */
#ifndef lint
	.align 16
	ENTRY_NP(asm_mutex_spin_enter)
	ldstub	[%l6 + M_SPINLOCK], %l5	! try to set lock, get value in %l5
1:
	tst	%l5
	bnz	3f			! lock already held - go spin
	nop
2:	
	jmp	%l7 + 8			! return
	membar	#LoadLoad
	!
	! Spin on lock without using an atomic operation to prevent the caches
	! from unnecessarily moving ownership of the line around.
	!
3:
	ldub	[%l6 + M_SPINLOCK], %l5
4:
	tst	%l5
	bz,a	1b			! lock appears to be free, try again
	ldstub	[%l6 + M_SPINLOCK], %l5	! delay slot - try to set lock

	sethi	%hi(panicstr) , %l5
	ldn	[%l5 + %lo(panicstr)], %l5
	tst 	%l5
	bnz	2b			! after panic, feign success
	nop
	b	4b
	ldub	[%l6 + M_SPINLOCK], %l5	! delay - reload lock
	SET_SIZE(asm_mutex_spin_enter)
#endif /* lint */

/*
 * asm_mutex_spin_exit(mutex_t *)
 *
 * For use by assembly interrupt handler only.
 * Does not change spl, since the interrupt handler is assumed to be
 * running at high level already.
 *
 * Entry:	%l6 - points to mutex
 * 		%l7 - address of call (returns to %l7+8)
 * Uses:	none
 */
#ifndef lint
	ENTRY_NP(asm_mutex_spin_exit)
	membar	#LoadStore|#StoreStore
	jmp	%l7 + 8			! return
	clrb	[%l6 + M_SPINLOCK]	! delay - clear lock
	SET_SIZE(asm_mutex_spin_exit)
#endif /* lint */

/*
 * thread_onproc()
 * Set thread in onproc state for the specified CPU.
 * Also set the thread lock pointer to the CPU's onproc lock.
 * Since the new lock isn't held, the store ordering is important.
 * If not done in assembler, the compiler could reorder the stores.
 */
#if defined(lint)

void
thread_onproc(kthread_id_t t, cpu_t *cp)
{
	t->t_state = TS_ONPROC;
	t->t_lockp = &cp->cpu_thread_lock;
}

#else	/* lint */

	ENTRY(thread_onproc)
	set	TS_ONPROC, %o2		! TS_ONPROC state
	st	%o2, [%o0 + T_STATE]	! store state
	add	%o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running
	retl				! return
	stn	%o3, [%o0 + T_LOCKP]	! delay - store new lock pointer
	SET_SIZE(thread_onproc)

#endif	/* lint */