1 // SPDX-License-Identifier: GPL-2.0-or-later
61  * 		- 1 CB & a few DSRs that are reserved for each cpu on the blade.
69  *		- Additional resources can be reserved long term & used directly
73  *		  	- these resources must be explicitly locked/unlocked
74  *		  	- locked resources prevent (obviously) the kernel
76  *			- drivers using these resource directly issue their own
87 #define ASYNC_HAN_TO_BID(h)	((h) - 1)
99 /* GRU cacheline size is always 64 bytes - even on arches with 128 byte lines */
111 #define MQS_NOOP		2
113 /*----------------- RESOURCE MANAGEMENT -------------------------------------*/
118 	long 			hstatus[2];
123 	char			data ____cacheline_aligned;	/* CL 2 */
126 /* First word in every message - used by mesq interface */
147 	up_read(&bs->bs_kgts_sema);  in gru_load_kernel_context()
148 	down_write(&bs->bs_kgts_sema);  in gru_load_kernel_context()
150 	if (!bs->bs_kgts) {  in gru_load_kernel_context()
152 			bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0, 0);  in gru_load_kernel_context()
153 			if (!IS_ERR(bs->bs_kgts))  in gru_load_kernel_context()
157 		bs->bs_kgts->ts_user_blade_id = blade_id;  in gru_load_kernel_context()
159 	kgts = bs->bs_kgts;  in gru_load_kernel_context()
161 	if (!kgts->ts_gru) {  in gru_load_kernel_context()
164 		kgts->ts_cbr_au_count = GRU_CB_COUNT_TO_AU(  in gru_load_kernel_context()
165 			GRU_NUM_KERNEL_CBR * ncpus + bs->bs_async_cbrs);  in gru_load_kernel_context()
166 		kgts->ts_dsr_au_count = GRU_DS_BYTES_TO_AU(  in gru_load_kernel_context()
168 				bs->bs_async_dsr_bytes);  in gru_load_kernel_context()
174 		gru = bs->bs_kgts->ts_gru;  in gru_load_kernel_context()
175 		vaddr = gru->gs_gru_base_vaddr;  in gru_load_kernel_context()
176 		ctxnum = kgts->ts_ctxnum;  in gru_load_kernel_context()
177 		bs->kernel_cb = get_gseg_base_address_cb(vaddr, ctxnum, 0);  in gru_load_kernel_context()
178 		bs->kernel_dsr = get_gseg_base_address_ds(vaddr, ctxnum, 0);  in gru_load_kernel_context()
180 	downgrade_write(&bs->bs_kgts_sema);  in gru_load_kernel_context()
199 		if (down_write_trylock(&bs->bs_kgts_sema)) {  in gru_free_kernel_contexts()
200 			kgts = bs->bs_kgts;  in gru_free_kernel_contexts()
201 			if (kgts && kgts->ts_gru)  in gru_free_kernel_contexts()
203 			bs->bs_kgts = NULL;  in gru_free_kernel_contexts()
204 			up_write(&bs->bs_kgts_sema);  in gru_free_kernel_contexts()
227 	down_read(&bs->bs_kgts_sema);  in gru_lock_kernel_context()
229 		up_read(&bs->bs_kgts_sema);  in gru_lock_kernel_context()
232 	if (!bs->bs_kgts || !bs->bs_kgts->ts_gru)  in gru_lock_kernel_context()
247 	up_read(&bs->bs_kgts_sema);  in gru_unlock_kernel_context()
253  * 	- returns with preemption disabled
261 	bs = gru_lock_kernel_context(-1);  in gru_get_cpu_resources()
263 	*cb = bs->kernel_cb + lcpu * GRU_HANDLE_STRIDE;  in gru_get_cpu_resources()
264 	*dsr = bs->kernel_dsr + lcpu * GRU_NUM_KERNEL_DSR_BYTES;  in gru_get_cpu_resources()
281  * 		blade_id  - blade on which resources should be reserved
282  * 		cbrs	  - number of CBRs
283  * 		dsr_bytes - number of DSR bytes needed
297 	down_write(&bs->bs_kgts_sema);  in gru_reserve_async_resources()
300 	if (bs->bs_async_dsr_bytes + bs->bs_async_cbrs)  in gru_reserve_async_resources()
302 	bs->bs_async_dsr_bytes = dsr_bytes;  in gru_reserve_async_resources()
303 	bs->bs_async_cbrs = cbrs;  in gru_reserve_async_resources()
304 	bs->bs_async_wq = cmp;  in gru_reserve_async_resources()
305 	kgts = bs->bs_kgts;  in gru_reserve_async_resources()
308 	if (kgts && kgts->ts_gru)  in gru_reserve_async_resources()
313 	up_write(&bs->bs_kgts_sema);  in gru_reserve_async_resources()
321  *		han - handle to identify resources
327 	down_write(&bs->bs_kgts_sema);  in gru_release_async_resources()
328 	bs->bs_async_dsr_bytes = 0;  in gru_release_async_resources()
329 	bs->bs_async_cbrs = 0;  in gru_release_async_resources()
330 	bs->bs_async_wq = NULL;  in gru_release_async_resources()
331 	up_write(&bs->bs_kgts_sema);  in gru_release_async_resources()
338  *		han - handle to identify resources
344 	wait_for_completion(bs->bs_async_wq);  in gru_wait_async_cbr()
352  *		han - handle to identify resources
354  *		cb  - pointer to first CBR
355  *		dsr - pointer to first DSR
366 		*cb = bs->kernel_cb + ncpus * GRU_HANDLE_STRIDE;  in gru_lock_async_resource()
368 		*dsr = bs->kernel_dsr + ncpus * GRU_NUM_KERNEL_DSR_BYTES;  in gru_lock_async_resource()
375  *		han - handle to identify resources
384 /*----------------------------------------------------------------------*/
401 		kgts = gru_base[bid]->bs_kgts;  in gru_get_cb_exception_detail()
402 		if (!kgts || !kgts->ts_gru)  in gru_get_cb_exception_detail()
404 		off = cb - kgts->ts_gru->gs_gru_base_vaddr;  in gru_get_cb_exception_detail()
414 	excdet->opc = cbe->opccpy;  in gru_get_cb_exception_detail()
415 	excdet->exopc = cbe->exopccpy;  in gru_get_cb_exception_detail()
416 	excdet->ecause = cbe->ecause;  in gru_get_cb_exception_detail()
417 	excdet->exceptdet0 = cbe->idef1upd;  in gru_get_cb_exception_detail()
418 	excdet->exceptdet1 = cbe->idef3upd;  in gru_get_cb_exception_detail()
426 	struct gru_control_block_status *gen = cb;  in gru_get_cb_exception_detail_str()  local
429 	if (ret > 0 && gen->istatus == CBS_EXCEPTION) {  in gru_get_cb_exception_detail_str()
434 			gen, excdet.opc, excdet.exopc, excdet.ecause,  in gru_get_cb_exception_detail_str()
442 static int gru_wait_idle_or_exception(struct gru_control_block_status *gen)  in gru_wait_idle_or_exception()  argument
444 	while (gen->istatus >= CBS_ACTIVE) {  in gru_wait_idle_or_exception()
448 	return gen->istatus;  in gru_wait_idle_or_exception()
453 	struct gru_control_block_status *gen = cb;  in gru_retry_exception()  local
458 		if (gru_wait_idle_or_exception(gen) == CBS_IDLE)  in gru_retry_exception()
466 		if (retry-- == 0)  in gru_retry_exception()
468 		gen->icmd = 1;  in gru_retry_exception()
469 		gru_flush_cache(gen);  in gru_retry_exception()
476 	struct gru_control_block_status *gen = cb;  in gru_check_status_proc()  local
479 	ret = gen->istatus;  in gru_check_status_proc()
489 	struct gru_control_block_status *gen = cb;  in gru_wait_proc()  local
492 	ret = gru_wait_idle_or_exception(gen);  in gru_wait_proc()
503 	panic("GRU FATAL ERROR: %s - %s\n", str,  in gru_abort()
517 /*------------------------------ MESSAGE QUEUES -----------------------------*/
520 #define MQIE_AGAIN		-1	/* try again */
524  * Save/restore the "present" flag that is in the second line of 2-line
530 	return mhdr->present;  in get_present2()
536 	mhdr->present = val;  in restore_present2()
541  * 	qlines - message queue size in cache lines. Includes 2-line header.
549 	qlines = bytes / GRU_CACHE_LINE_BYTES - 2;  in gru_create_message_queue()
551 	mq->start = &mq->data;  in gru_create_message_queue()
552 	mq->start2 = &mq->data + (qlines / 2 - 1) * GRU_CACHE_LINE_BYTES;  in gru_create_message_queue()
553 	mq->next = &mq->data;  in gru_create_message_queue()
554 	mq->limit = &mq->data + (qlines - 2) * GRU_CACHE_LINE_BYTES;  in gru_create_message_queue()
555 	mq->qlines = qlines;  in gru_create_message_queue()
556 	mq->hstatus[0] = 0;  in gru_create_message_queue()
557 	mq->hstatus[1] = 1;  in gru_create_message_queue()
558 	mq->head = gru_mesq_head(2, qlines / 2 + 1);  in gru_create_message_queue()
559 	mqd->mq = mq;  in gru_create_message_queue()
560 	mqd->mq_gpa = uv_gpa(mq);  in gru_create_message_queue()
561 	mqd->qlines = qlines;  in gru_create_message_queue()
562 	mqd->interrupt_pnode = nasid >> 1;  in gru_create_message_queue()
563 	mqd->interrupt_vector = vector;  in gru_create_message_queue()
564 	mqd->interrupt_apicid = apicid;  in gru_create_message_queue()
572  * 		 0 - if queue is full after the send. This is the normal case
574  *		-1 - if mesq sent successfully but queue not full
575  *		>0 - unexpected error. MQE_xxx returned
589 	gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);  in send_noop_message()
613 			m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);  in send_noop_message()
648 	qlines = mqd->qlines;  in send_message_queue_full()
652 		mqh = gru_mesq_head(qlines / 2 + 1, qlines);  in send_message_queue_full()
654 		mqh = gru_mesq_head(2, qlines / 2 + 1);  in send_message_queue_full()
657 	gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);  in send_message_queue_full()
668 			gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),  in send_message_queue_full()
679 	gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,  in send_message_queue_full()
687 		gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,  in send_message_queue_full()
699  * Handle a PUT failure. Note: if message was a 2-line message, one of the
710 	m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);  in send_message_put_nacked()
711 	if (lines == 2) {  in send_message_put_nacked()
720 	if (!mqd->interrupt_vector)  in send_message_put_nacked()
724 	 * Send a noop message in order to deliver a cross-partition interrupt  in send_message_put_nacked()
733 	} while ((ret == MQIE_AGAIN || ret == MQE_CONGESTION) && (loops-- > 0));  in send_message_put_nacked()
792  * 	bytes	message size (<= 2 CL)
803 	BUG_ON(bytes < sizeof(int) || bytes > 2 * GRU_CACHE_LINE_BYTES);  in gru_send_message_gpa()
810 	mhdr->present = MQS_FULL;  in gru_send_message_gpa()
811 	mhdr->lines = clines;  in gru_send_message_gpa()
812 	if (clines == 2) {  in gru_send_message_gpa()
813 		mhdr->present2 = get_present2(mhdr);  in gru_send_message_gpa()
819 		gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), clines, IMA);  in gru_send_message_gpa()
837 	struct message_queue *mq = mqd->mq;  in gru_free_message()
838 	struct message_header *mhdr = mq->next;  in gru_free_message()
840 	int half = -1;  in gru_free_message()
841 	int lines = mhdr->lines;  in gru_free_message()
843 	if (lines == 2)  in gru_free_message()
845 	mhdr->present = MQS_EMPTY;  in gru_free_message()
847 	pnext = mq->next;  in gru_free_message()
849 	if (next == mq->limit) {  in gru_free_message()
850 		next = mq->start;  in gru_free_message()
852 	} else if (pnext < mq->start2 && next >= mq->start2) {  in gru_free_message()
857 		mq->hstatus[half] = 1;  in gru_free_message()
858 	mq->next = next;  in gru_free_message()
869 	struct message_queue *mq = mqd->mq;  in gru_get_next_message()
870 	struct message_header *mhdr = mq->next;  in gru_get_next_message()
871 	int present = mhdr->present;  in gru_get_next_message()
876 		mhdr = mq->next;  in gru_get_next_message()
877 		present = mhdr->present;  in gru_get_next_message()
880 	/* Wait for both halves of 2 line messages */  in gru_get_next_message()
881 	if (present == MQS_FULL && mhdr->lines == 2 &&  in gru_get_next_message()
890 	if (mhdr->lines == 2)  in gru_get_next_message()
891 		restore_present2(mhdr, mhdr->present2);  in gru_get_next_message()
898 /* ---------------------- GRU DATA COPY FUNCTIONS ---------------------------*/
944 /* ------------------- KERNEL QUICKTESTS RUN AT STARTUP ----------------*/
945 /* 	Temp - will delete after we gain confidence in the GRU		*/
954 	int ret = -EIO;  in quicktest0()
974 		printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 2\n", smp_processor_id());  in quicktest0()
991 #define ALIGNUP(p, q)	((void *)(((unsigned long)(p) + (q) - 1) & ~(q - 1)))
997 	int i, ret = -EIO;  in quicktest1()
1003 		return -ENOMEM;  in quicktest1()
1030 			smp_processor_id(), i, m, m ? m[8] : -1);  in quicktest1()
1049 	struct gru_control_block_status *gen;  in quicktest2()  local
1055 		return -ENOMEM;  in quicktest2()
1057 	ret = -EBUSY;  in quicktest2()
1082 			ret = -EFAULT;  in quicktest2()
1083 		} else if (buf[4 * i] || buf[4 * i + 1] || buf[4 * i + 2] ||  in quicktest2()
1086 			       smp_processor_id(), i, buf[4 * i], buf[4 * i + 1], buf[4 * i + 2], buf[4 * i + 3]);  in quicktest2()
1087 			ret = -EIO;  in quicktest2()
1089 		k--;  in quicktest2()
1090 		gen = cb;  in quicktest2()
1091 		gen->istatus = CBS_CALL_OS; /* don't handle this CBR again */  in quicktest2()
1113 		ret = -EIO;  in quicktest3()
1124 	int ret = -EINVAL;  in gru_ktest()
1133 	case 2:  in gru_ktest()