96 					    struct kfd_event *ev,  in allocate_event_notification_slot()  argument
110 		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,  in allocate_event_notification_slot()
119 		id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,  in allocate_event_notification_slot()
125 	ev->event_id = id;  in allocate_event_notification_slot()
160 	struct kfd_event *ev;  in lookup_signaled_event_by_partial_id()  local
178 	for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) {  in lookup_signaled_event_by_partial_id()
182 		ev = idr_find(&p->event_idr, id);  in lookup_signaled_event_by_partial_id()
185 	return ev;  in lookup_signaled_event_by_partial_id()
189 				struct kfd_event *ev, const int *restore_id)  in create_signal_event()  argument
202 	ret = allocate_event_notification_slot(p, ev, restore_id);  in create_signal_event()
210 	ev->user_signal_address = &p->signal_page->user_address[ev->event_id];  in create_signal_event()
212 			p->signal_event_count, ev->event_id,  in create_signal_event()
213 			ev->user_signal_address);  in create_signal_event()
218 static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id)  in create_other_event()  argument
223 		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,  in create_other_event()
231 		id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,  in create_other_event()
237 	ev->event_id = id;  in create_other_event()
262 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)  in destroy_event()  argument
267 	spin_lock(&ev->lock);  in destroy_event()
268 	list_for_each_entry(waiter, &ev->wq.head, wait.entry)  in destroy_event()
270 	wake_up_all(&ev->wq);  in destroy_event()
271 	spin_unlock(&ev->lock);  in destroy_event()
273 	if (ev->type == KFD_EVENT_TYPE_SIGNAL ||  in destroy_event()
274 	    ev->type == KFD_EVENT_TYPE_DEBUG)  in destroy_event()
277 	idr_remove(&p->event_idr, ev->event_id);  in destroy_event()
278 	kfree_rcu(ev, rcu);  in destroy_event()
283 	struct kfd_event *ev;  in destroy_events()  local
286 	idr_for_each_entry(&p->event_idr, ev, id)  in destroy_events()
287 		if (ev)  in destroy_events()
288 			destroy_event(p, ev);  in destroy_events()
315 static bool event_can_be_gpu_signaled(const struct kfd_event *ev)  in event_can_be_gpu_signaled()  argument
317 	return ev->type == KFD_EVENT_TYPE_SIGNAL ||  in event_can_be_gpu_signaled()
318 					ev->type == KFD_EVENT_TYPE_DEBUG;  in event_can_be_gpu_signaled()
321 static bool event_can_be_cpu_signaled(const struct kfd_event *ev)  in event_can_be_cpu_signaled()  argument
323 	return ev->type == KFD_EVENT_TYPE_SIGNAL;  in event_can_be_cpu_signaled()
402 	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);  in kfd_event_create()  local
404 	if (!ev)  in kfd_event_create()
407 	ev->type = event_type;  in kfd_event_create()
408 	ev->auto_reset = auto_reset;  in kfd_event_create()
409 	ev->signaled = false;  in kfd_event_create()
411 	spin_lock_init(&ev->lock);  in kfd_event_create()
412 	init_waitqueue_head(&ev->wq);  in kfd_event_create()
421 		ret = create_signal_event(devkfd, p, ev, NULL);  in kfd_event_create()
424 			*event_slot_index = ev->event_id;  in kfd_event_create()
428 		ret = create_other_event(p, ev, NULL);  in kfd_event_create()
433 		*event_id = ev->event_id;  in kfd_event_create()
434 		*event_trigger_data = ev->event_id;  in kfd_event_create()
435 		ev->event_age = 1;  in kfd_event_create()
437 		kfree(ev);  in kfd_event_create()
452 	struct kfd_event *ev = NULL;  in kfd_criu_restore_event()  local
459 	ev = kzalloc(sizeof(*ev), GFP_KERNEL);  in kfd_criu_restore_event()
460 	if (!ev) {  in kfd_criu_restore_event()
483 	ev->type = ev_priv->type;  in kfd_criu_restore_event()
484 	ev->auto_reset = ev_priv->auto_reset;  in kfd_criu_restore_event()
485 	ev->signaled = ev_priv->signaled;  in kfd_criu_restore_event()
487 	spin_lock_init(&ev->lock);  in kfd_criu_restore_event()
488 	init_waitqueue_head(&ev->wq);  in kfd_criu_restore_event()
491 	switch (ev->type) {  in kfd_criu_restore_event()
494 		ret = create_signal_event(devkfd, p, ev, &ev_priv->event_id);  in kfd_criu_restore_event()
497 		memcpy(&ev->memory_exception_data,  in kfd_criu_restore_event()
501 		ret = create_other_event(p, ev, &ev_priv->event_id);  in kfd_criu_restore_event()
504 		memcpy(&ev->hw_exception_data,  in kfd_criu_restore_event()
508 		ret = create_other_event(p, ev, &ev_priv->event_id);  in kfd_criu_restore_event()
515 		kfree(ev);  in kfd_criu_restore_event()
529 	struct kfd_event *ev;  in kfd_criu_checkpoint_events()  local
542 	idr_for_each_entry(&p->event_idr, ev, ev_id) {  in kfd_criu_checkpoint_events()
557 		ev_priv->event_id = ev->event_id;  in kfd_criu_checkpoint_events()
558 		ev_priv->auto_reset = ev->auto_reset;  in kfd_criu_checkpoint_events()
559 		ev_priv->type = ev->type;  in kfd_criu_checkpoint_events()
560 		ev_priv->signaled = ev->signaled;  in kfd_criu_checkpoint_events()
564 				&ev->memory_exception_data,  in kfd_criu_checkpoint_events()
568 				&ev->hw_exception_data,  in kfd_criu_checkpoint_events()
595 	struct kfd_event *ev;  in kfd_get_num_events()  local
599 	idr_for_each_entry(&p->event_idr, ev, id)  in kfd_get_num_events()
608 	struct kfd_event *ev;  in kfd_event_destroy()  local
613 	ev = lookup_event_by_id(p, event_id);  in kfd_event_destroy()
615 	if (ev)  in kfd_event_destroy()
616 		destroy_event(p, ev);  in kfd_event_destroy()
624 static void set_event(struct kfd_event *ev)  in set_event()  argument
633 	ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq);  in set_event()
634 	if (!(++ev->event_age)) {  in set_event()
636 		ev->event_age = 2;  in set_event()
640 	list_for_each_entry(waiter, &ev->wq.head, wait.entry)  in set_event()
643 	wake_up_all(&ev->wq);  in set_event()
650 	struct kfd_event *ev;  in kfd_set_event()  local
654 	ev = lookup_event_by_id(p, event_id);  in kfd_set_event()
655 	if (!ev) {  in kfd_set_event()
659 	spin_lock(&ev->lock);  in kfd_set_event()
661 	if (event_can_be_cpu_signaled(ev))  in kfd_set_event()
662 		set_event(ev);  in kfd_set_event()
666 	spin_unlock(&ev->lock);  in kfd_set_event()
672 static void reset_event(struct kfd_event *ev)  in reset_event()  argument
674 	ev->signaled = false;  in reset_event()
681 	struct kfd_event *ev;  in kfd_reset_event()  local
685 	ev = lookup_event_by_id(p, event_id);  in kfd_reset_event()
686 	if (!ev) {  in kfd_reset_event()
690 	spin_lock(&ev->lock);  in kfd_reset_event()
692 	if (event_can_be_cpu_signaled(ev))  in kfd_reset_event()
693 		reset_event(ev);  in kfd_reset_event()
697 	spin_unlock(&ev->lock);  in kfd_reset_event()
704 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)  in acknowledge_signal()  argument
706 	WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT);  in acknowledge_signal()
710 					struct kfd_event *ev)  in set_event_from_interrupt()  argument
712 	if (ev && event_can_be_gpu_signaled(ev)) {  in set_event_from_interrupt()
713 		acknowledge_signal(p, ev);  in set_event_from_interrupt()
714 		spin_lock(&ev->lock);  in set_event_from_interrupt()
715 		set_event(ev);  in set_event_from_interrupt()
716 		spin_unlock(&ev->lock);  in set_event_from_interrupt()
723 	struct kfd_event *ev = NULL;  in kfd_signal_event_interrupt()  local
738 		ev = lookup_signaled_event_by_partial_id(p, partial_id,  in kfd_signal_event_interrupt()
740 	if (ev) {  in kfd_signal_event_interrupt()
741 		set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
769 			idr_for_each_entry(&p->event_idr, ev, id) {  in kfd_signal_event_interrupt()
774 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
783 					ev = lookup_event_by_id(p, id);  in kfd_signal_event_interrupt()
784 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
814 	struct kfd_event *ev = lookup_event_by_id(p, event_data->event_id);  in init_event_waiter()  local
816 	if (!ev)  in init_event_waiter()
819 	spin_lock(&ev->lock);  in init_event_waiter()
820 	waiter->event = ev;  in init_event_waiter()
821 	waiter->activated = ev->signaled;  in init_event_waiter()
822 	ev->signaled = ev->signaled && !ev->auto_reset;  in init_event_waiter()
828 		if (ev->event_age != event_data->signal_event_data.last_event_age)  in init_event_waiter()
833 		add_wait_queue(&ev->wq, &waiter->wait);  in init_event_waiter()
834 	spin_unlock(&ev->lock);  in init_event_waiter()
1118 	struct kfd_event *ev;  in lookup_events_by_type_and_signal()  local
1127 	idr_for_each_entry_continue(&p->event_idr, ev, id)  in lookup_events_by_type_and_signal()
1128 		if (ev->type == type) {  in lookup_events_by_type_and_signal()
1132 					ev->event_id, ev->type);  in lookup_events_by_type_and_signal()
1133 			spin_lock(&ev->lock);  in lookup_events_by_type_and_signal()
1134 			set_event(ev);  in lookup_events_by_type_and_signal()
1135 			if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data)  in lookup_events_by_type_and_signal()
1136 				ev->memory_exception_data = *ev_data;  in lookup_events_by_type_and_signal()
1137 			spin_unlock(&ev->lock);  in lookup_events_by_type_and_signal()
1184 	struct kfd_event *ev;  in kfd_signal_vm_fault_event()  local
1222 	idr_for_each_entry_continue(&p->event_idr, ev, id)  in kfd_signal_vm_fault_event()
1223 		if (ev->type == KFD_EVENT_TYPE_MEMORY) {  in kfd_signal_vm_fault_event()
1224 			spin_lock(&ev->lock);  in kfd_signal_vm_fault_event()
1225 			ev->memory_exception_data = data ? *data :  in kfd_signal_vm_fault_event()
1227 			set_event(ev);  in kfd_signal_vm_fault_event()
1228 			spin_unlock(&ev->lock);  in kfd_signal_vm_fault_event()
1240 	struct kfd_event *ev;  in kfd_signal_reset_event()  local
1289 		idr_for_each_entry_continue(&p->event_idr, ev, id) {  in kfd_signal_reset_event()
1290 			if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {  in kfd_signal_reset_event()
1291 				spin_lock(&ev->lock);  in kfd_signal_reset_event()
1292 				ev->hw_exception_data = hw_exception_data;  in kfd_signal_reset_event()
1293 				ev->hw_exception_data.gpu_id = user_gpu_id;  in kfd_signal_reset_event()
1294 				set_event(ev);  in kfd_signal_reset_event()
1295 				spin_unlock(&ev->lock);  in kfd_signal_reset_event()
1297 			if (ev->type == KFD_EVENT_TYPE_MEMORY &&  in kfd_signal_reset_event()
1299 				spin_lock(&ev->lock);  in kfd_signal_reset_event()
1300 				ev->memory_exception_data = memory_exception_data;  in kfd_signal_reset_event()
1301 				ev->memory_exception_data.gpu_id = user_gpu_id;  in kfd_signal_reset_event()
1302 				set_event(ev);  in kfd_signal_reset_event()
1303 				spin_unlock(&ev->lock);  in kfd_signal_reset_event()
1317 	struct kfd_event *ev;  in kfd_signal_poison_consumed_event()  local
1344 	idr_for_each_entry_continue(&p->event_idr, ev, id) {  in kfd_signal_poison_consumed_event()
1345 		if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {  in kfd_signal_poison_consumed_event()
1346 			spin_lock(&ev->lock);  in kfd_signal_poison_consumed_event()
1347 			ev->hw_exception_data = hw_exception_data;  in kfd_signal_poison_consumed_event()
1348 			set_event(ev);  in kfd_signal_poison_consumed_event()
1349 			spin_unlock(&ev->lock);  in kfd_signal_poison_consumed_event()
1352 		if (ev->type == KFD_EVENT_TYPE_MEMORY) {  in kfd_signal_poison_consumed_event()
1353 			spin_lock(&ev->lock);  in kfd_signal_poison_consumed_event()
1354 			ev->memory_exception_data = memory_exception_data;  in kfd_signal_poison_consumed_event()
1355 			set_event(ev);  in kfd_signal_poison_consumed_event()
1356 			spin_unlock(&ev->lock);  in kfd_signal_poison_consumed_event()