13 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
51 			   struct hfi1_ctxtdata *uctxt)  in hfi1_user_exp_rcv_init()  argument
55 	fd->entry_to_rb = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
61 	if (!HFI1_CAP_UGET_MASK(uctxt->flags, TID_UNMAP)) {  in hfi1_user_exp_rcv_init()
63 		fd->invalid_tids = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
88 	if (uctxt->subctxt_cnt && fd->use_mn) {  in hfi1_user_exp_rcv_init()
91 		fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
92 		remainder = uctxt->expected_count % uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
96 		fd->tid_limit = uctxt->expected_count;  in hfi1_user_exp_rcv_init()
105 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_free()  local
107 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_free()
108 	if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list))  in hfi1_user_exp_rcv_free()
109 		unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd);  in hfi1_user_exp_rcv_free()
110 	if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list))  in hfi1_user_exp_rcv_free()
111 		unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd);  in hfi1_user_exp_rcv_free()
112 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_free()
140 	struct hfi1_devdata *dd = fd->uctxt->dd;  in unpin_rcv_pages()
165 	struct hfi1_devdata *dd = fd->uctxt->dd;  in pin_rcv_pages()
167 	if (npages > fd->uctxt->expected_count) {  in pin_rcv_pages()
250 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_setup()  local
251 	struct hfi1_devdata *dd = uctxt->dd;  in hfi1_user_exp_rcv_setup()
271 	tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),  in hfi1_user_exp_rcv_setup()
324 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_setup()
329 	while (ngroups && uctxt->tid_group_list.count) {  in hfi1_user_exp_rcv_setup()
331 			tid_group_pop(&uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
342 			tid_group_add_tail(grp, &uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
348 		tid_group_add_tail(grp, &uctxt->tid_full_list);  in hfi1_user_exp_rcv_setup()
360 		if (!uctxt->tid_used_list.count || need_group) {  in hfi1_user_exp_rcv_setup()
361 			if (!uctxt->tid_group_list.count)  in hfi1_user_exp_rcv_setup()
364 			grp = tid_group_pop(&uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
365 			tid_group_add_tail(grp, &uctxt->tid_used_list);  in hfi1_user_exp_rcv_setup()
373 		list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list,  in hfi1_user_exp_rcv_setup()
389 						       &uctxt->tid_used_list,  in hfi1_user_exp_rcv_setup()
390 						       &uctxt->tid_full_list);  in hfi1_user_exp_rcv_setup()
408 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_setup()
487 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_clear()  local
499 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_clear()
512 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_clear()
521 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_invalid()  local
522 	unsigned long *ev = uctxt->dd->events +  in hfi1_user_exp_rcv_invalid()
523 		(uctxt_offset(uctxt) + fd->subctxt);  in hfi1_user_exp_rcv_invalid()
533 	array = kcalloc(uctxt->expected_count, sizeof(*array), GFP_KERNEL);  in hfi1_user_exp_rcv_invalid()
661 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in program_rcvarray()  local
662 	struct hfi1_devdata *dd = uctxt->dd;  in program_rcvarray()
709 		tidinfo = create_tid(rcventry - uctxt->expected_base, npages);  in program_rcvarray()
729 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in set_rcvarray_entry()  local
731 	struct hfi1_devdata *dd = uctxt->dd;  in set_rcvarray_entry()
770 	fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node;  in set_rcvarray_entry()
773 	trace_hfi1_exp_tid_reg(uctxt->ctxt, fd->subctxt, rcventry, npages,  in set_rcvarray_entry()
790 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in unprogram_rcvarray()  local
791 	struct hfi1_devdata *dd = uctxt->dd;  in unprogram_rcvarray()
801 	if (rcventry >= uctxt->expected_count) {  in unprogram_rcvarray()
803 			   rcventry, uctxt->ctxt);  in unprogram_rcvarray()
808 	if (!node || node->rcventry != (uctxt->expected_base + rcventry))  in unprogram_rcvarray()
820 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in __clear_tid_node()  local
821 	struct hfi1_devdata *dd = uctxt->dd;  in __clear_tid_node()
828 	trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,  in __clear_tid_node()
843 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in clear_tid_node()  local
851 		tid_group_move(node->grp, &uctxt->tid_full_list,  in clear_tid_node()
852 			       &uctxt->tid_used_list);  in clear_tid_node()
854 		tid_group_move(node->grp, &uctxt->tid_used_list,  in clear_tid_node()
855 			       &uctxt->tid_group_list);  in clear_tid_node()
863 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,  in unlock_exp_tids()  argument
879 							  uctxt->expected_base];  in unlock_exp_tids()
899 	struct hfi1_ctxtdata *uctxt = fdata->uctxt;  in tid_rb_invalidate()  local
908 	trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt,  in tid_rb_invalidate()
916 	if (fdata->invalid_tid_idx < uctxt->expected_count) {  in tid_rb_invalidate()
918 			create_tid(node->rcventry - uctxt->expected_base,  in tid_rb_invalidate()
932 			ev = uctxt->dd->events +  in tid_rb_invalidate()
933 				(uctxt_offset(uctxt) + fdata->subctxt);  in tid_rb_invalidate()
962 	u32 base = fdata->uctxt->expected_base;  in cacheless_tid_rb_remove()