Lines Matching +full:ch +full:- +full:func
6 * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
10 * Cross Partition Communication (XPC) uv-based functions.
29 #include "../sgi-gru/gru.h"
30 #include "../sgi-gru/grukservices.h"
59 mutex_init(&part_uv->cached_activate_gru_mq_desc_mutex); in xpc_setup_partitions_uv()
60 spin_lock_init(&part_uv->flags_lock); in xpc_setup_partitions_uv()
61 part_uv->remote_act_state = XPC_P_AS_INACTIVE; in xpc_setup_partitions_uv()
76 if (part_uv->cached_activate_gru_mq_desc != NULL) { in xpc_teardown_partitions_uv()
77 mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex); in xpc_teardown_partitions_uv()
78 spin_lock_irqsave(&part_uv->flags_lock, irq_flags); in xpc_teardown_partitions_uv()
79 part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV; in xpc_teardown_partitions_uv()
80 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); in xpc_teardown_partitions_uv()
81 kfree(part_uv->cached_activate_gru_mq_desc); in xpc_teardown_partitions_uv()
82 part_uv->cached_activate_gru_mq_desc = NULL; in xpc_teardown_partitions_uv()
83 mutex_unlock(&part_uv-> in xpc_teardown_partitions_uv()
92 int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); in xpc_get_gru_mq_irq_uv()
94 mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset, in xpc_get_gru_mq_irq_uv()
96 if (mq->irq < 0) in xpc_get_gru_mq_irq_uv()
97 return mq->irq; in xpc_get_gru_mq_irq_uv()
99 mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset); in xpc_get_gru_mq_irq_uv()
107 uv_teardown_irq(mq->irq); in xpc_release_gru_mq_irq_uv()
115 ret = uv_bios_mq_watchlist_alloc(uv_gpa(mq->address), in xpc_gru_mq_watchlist_alloc_uv()
116 mq->order, &mq->mmr_offset); in xpc_gru_mq_watchlist_alloc_uv()
123 mq->watchlist_num = ret; in xpc_gru_mq_watchlist_alloc_uv()
131 int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); in xpc_gru_mq_watchlist_free_uv()
133 ret = uv_bios_mq_watchlist_free(mmr_pnode, mq->watchlist_num); in xpc_gru_mq_watchlist_free_uv()
154 ret = -ENOMEM; in xpc_create_gru_mq_uv()
158 mq->gru_mq_desc = kzalloc(sizeof(struct gru_message_queue_desc), in xpc_create_gru_mq_uv()
160 if (mq->gru_mq_desc == NULL) { in xpc_create_gru_mq_uv()
163 ret = -ENOMEM; in xpc_create_gru_mq_uv()
168 mq->order = pg_order + PAGE_SHIFT; in xpc_create_gru_mq_uv()
169 mq_size = 1UL << mq->order; in xpc_create_gru_mq_uv()
171 mq->mmr_blade = uv_cpu_to_blade_id(cpu); in xpc_create_gru_mq_uv()
180 ret = -ENOMEM; in xpc_create_gru_mq_uv()
183 mq->address = page_address(page); in xpc_create_gru_mq_uv()
194 ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL); in xpc_create_gru_mq_uv()
197 mq->irq, -ret); in xpc_create_gru_mq_uv()
203 mmr_value = (struct uv_IO_APIC_route_entry *)&mq->mmr_value; in xpc_create_gru_mq_uv()
204 ret = gru_create_message_queue(mq->gru_mq_desc, mq->address, mq_size, in xpc_create_gru_mq_uv()
205 nasid, mmr_value->vector, mmr_value->dest); in xpc_create_gru_mq_uv()
209 ret = -EINVAL; in xpc_create_gru_mq_uv()
214 xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size); in xpc_create_gru_mq_uv()
216 ret = -EACCES; in xpc_create_gru_mq_uv()
224 free_irq(mq->irq, NULL); in xpc_create_gru_mq_uv()
230 free_pages((unsigned long)mq->address, pg_order); in xpc_create_gru_mq_uv()
232 kfree(mq->gru_mq_desc); in xpc_create_gru_mq_uv()
247 mq_size = 1UL << mq->order; in xpc_destroy_gru_mq_uv()
248 ret = xp_restrict_memprotect(xp_pa(mq->address), mq_size); in xpc_destroy_gru_mq_uv()
252 free_irq(mq->irq, NULL); in xpc_destroy_gru_mq_uv()
258 pg_order = mq->order - PAGE_SHIFT; in xpc_destroy_gru_mq_uv()
259 free_pages((unsigned long)mq->address, pg_order); in xpc_destroy_gru_mq_uv()
314 if (part->sn.uv.act_state_req == 0) in xpc_process_activate_IRQ_rcvd_uv()
317 xpc_activate_IRQ_rcvd--; in xpc_process_activate_IRQ_rcvd_uv()
320 act_state_req = part->sn.uv.act_state_req; in xpc_process_activate_IRQ_rcvd_uv()
321 part->sn.uv.act_state_req = 0; in xpc_process_activate_IRQ_rcvd_uv()
325 if (part->act_state == XPC_P_AS_INACTIVE) in xpc_process_activate_IRQ_rcvd_uv()
327 else if (part->act_state == XPC_P_AS_DEACTIVATING) in xpc_process_activate_IRQ_rcvd_uv()
331 if (part->act_state == XPC_P_AS_INACTIVE) in xpc_process_activate_IRQ_rcvd_uv()
337 XPC_DEACTIVATE_PARTITION(part, part->sn.uv.reason); in xpc_process_activate_IRQ_rcvd_uv()
358 struct xpc_partition_uv *part_uv = &part->sn.uv; in xpc_handle_activate_mq_msg_uv()
361 part_uv->remote_act_state = msg_hdr->act_state; in xpc_handle_activate_mq_msg_uv()
363 switch (msg_hdr->type) { in xpc_handle_activate_mq_msg_uv()
380 if (part_uv->act_state_req == 0) in xpc_handle_activate_mq_msg_uv()
382 part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV; in xpc_handle_activate_mq_msg_uv()
383 part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */ in xpc_handle_activate_mq_msg_uv()
384 part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies; in xpc_handle_activate_mq_msg_uv()
385 part_uv->heartbeat_gpa = msg->heartbeat_gpa; in xpc_handle_activate_mq_msg_uv()
387 if (msg->activate_gru_mq_desc_gpa != in xpc_handle_activate_mq_msg_uv()
388 part_uv->activate_gru_mq_desc_gpa) { in xpc_handle_activate_mq_msg_uv()
389 spin_lock(&part_uv->flags_lock); in xpc_handle_activate_mq_msg_uv()
390 part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV; in xpc_handle_activate_mq_msg_uv()
391 spin_unlock(&part_uv->flags_lock); in xpc_handle_activate_mq_msg_uv()
392 part_uv->activate_gru_mq_desc_gpa = in xpc_handle_activate_mq_msg_uv()
393 msg->activate_gru_mq_desc_gpa; in xpc_handle_activate_mq_msg_uv()
407 if (part_uv->act_state_req == 0) in xpc_handle_activate_mq_msg_uv()
409 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV; in xpc_handle_activate_mq_msg_uv()
410 part_uv->reason = msg->reason; in xpc_handle_activate_mq_msg_uv()
425 args = &part->remote_openclose_args[msg->ch_number]; in xpc_handle_activate_mq_msg_uv()
426 args->reason = msg->reason; in xpc_handle_activate_mq_msg_uv()
428 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
429 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREQUEST; in xpc_handle_activate_mq_msg_uv()
430 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
445 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
446 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREPLY; in xpc_handle_activate_mq_msg_uv()
447 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
461 args = &part->remote_openclose_args[msg->ch_number]; in xpc_handle_activate_mq_msg_uv()
462 args->entry_size = msg->entry_size; in xpc_handle_activate_mq_msg_uv()
463 args->local_nentries = msg->local_nentries; in xpc_handle_activate_mq_msg_uv()
465 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
466 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREQUEST; in xpc_handle_activate_mq_msg_uv()
467 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
480 args = &part->remote_openclose_args[msg->ch_number]; in xpc_handle_activate_mq_msg_uv()
481 args->remote_nentries = msg->remote_nentries; in xpc_handle_activate_mq_msg_uv()
482 args->local_nentries = msg->local_nentries; in xpc_handle_activate_mq_msg_uv()
483 args->local_msgqueue_pa = msg->notify_gru_mq_desc_gpa; in xpc_handle_activate_mq_msg_uv()
485 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
486 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY; in xpc_handle_activate_mq_msg_uv()
487 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
500 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
501 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENCOMPLETE; in xpc_handle_activate_mq_msg_uv()
502 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
508 spin_lock_irqsave(&part_uv->flags_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
509 part_uv->flags |= XPC_P_ENGAGED_UV; in xpc_handle_activate_mq_msg_uv()
510 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
514 spin_lock_irqsave(&part_uv->flags_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
515 part_uv->flags &= ~XPC_P_ENGAGED_UV; in xpc_handle_activate_mq_msg_uv()
516 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); in xpc_handle_activate_mq_msg_uv()
521 "from partition=%d\n", msg_hdr->type, XPC_PARTID(part)); in xpc_handle_activate_mq_msg_uv()
525 if (part_uv->act_state_req == 0) in xpc_handle_activate_mq_msg_uv()
527 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV; in xpc_handle_activate_mq_msg_uv()
528 part_uv->reason = xpBadMsgType; in xpc_handle_activate_mq_msg_uv()
535 if (msg_hdr->rp_ts_jiffies != part->remote_rp_ts_jiffies && in xpc_handle_activate_mq_msg_uv()
536 part->remote_rp_ts_jiffies != 0) { in xpc_handle_activate_mq_msg_uv()
542 if (part_uv->act_state_req == 0) in xpc_handle_activate_mq_msg_uv()
544 part_uv->act_state_req = XPC_P_ASR_REACTIVATE_UV; in xpc_handle_activate_mq_msg_uv()
561 msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc); in xpc_handle_activate_IRQ_uv()
565 partid = msg_hdr->partid; in xpc_handle_activate_IRQ_uv()
581 gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr); in xpc_handle_activate_IRQ_uv()
599 gru_mq_desc->mq = NULL; in xpc_cache_remote_gru_mq_desc_uv()
609 struct xpc_partition_uv *part_uv = &part->sn.uv; in xpc_send_activate_IRQ_uv()
616 msg_hdr->type = msg_type; in xpc_send_activate_IRQ_uv()
617 msg_hdr->partid = xp_partition_id; in xpc_send_activate_IRQ_uv()
618 msg_hdr->act_state = part->act_state; in xpc_send_activate_IRQ_uv()
619 msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies; in xpc_send_activate_IRQ_uv()
621 mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex); in xpc_send_activate_IRQ_uv()
623 if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) { in xpc_send_activate_IRQ_uv()
624 gru_mq_desc = part_uv->cached_activate_gru_mq_desc; in xpc_send_activate_IRQ_uv()
633 part_uv->cached_activate_gru_mq_desc = gru_mq_desc; in xpc_send_activate_IRQ_uv()
637 part_uv-> in xpc_send_activate_IRQ_uv()
642 spin_lock_irqsave(&part_uv->flags_lock, irq_flags); in xpc_send_activate_IRQ_uv()
643 part_uv->flags |= XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV; in xpc_send_activate_IRQ_uv()
644 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); in xpc_send_activate_IRQ_uv()
647 /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */ in xpc_send_activate_IRQ_uv()
648 ret = xpc_send_gru_msg(part_uv->cached_activate_gru_mq_desc, msg, in xpc_send_activate_IRQ_uv()
651 smp_rmb(); /* ensure a fresh copy of part_uv->flags */ in xpc_send_activate_IRQ_uv()
652 if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) in xpc_send_activate_IRQ_uv()
656 mutex_unlock(&part_uv->cached_activate_gru_mq_desc_mutex); in xpc_send_activate_IRQ_uv()
672 xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags, in xpc_send_activate_IRQ_ch_uv() argument
675 struct xpc_partition *part = &xpc_partitions[ch->partid]; in xpc_send_activate_IRQ_ch_uv()
681 spin_unlock_irqrestore(&ch->lock, *irq_flags); in xpc_send_activate_IRQ_ch_uv()
686 spin_lock_irqsave(&ch->lock, *irq_flags); in xpc_send_activate_IRQ_ch_uv()
694 struct xpc_partition_uv *part_uv = &part->sn.uv; in xpc_send_local_activate_IRQ_uv()
703 if (part_uv->act_state_req == 0) in xpc_send_local_activate_IRQ_uv()
705 part_uv->act_state_req = act_state_req; in xpc_send_local_activate_IRQ_uv()
735 rp->sn.uv.heartbeat_gpa = uv_gpa(xpc_heartbeat_uv); in xpc_setup_rsvd_page_uv()
736 rp->sn.uv.activate_gru_mq_desc_gpa = in xpc_setup_rsvd_page_uv()
737 uv_gpa(xpc_activate_mq_uv->gru_mq_desc); in xpc_setup_rsvd_page_uv()
759 xpc_heartbeat_uv->value++; in xpc_increment_heartbeat_uv()
766 xpc_heartbeat_uv->offline = 1; in xpc_offline_heartbeat_uv()
773 xpc_heartbeat_uv->offline = 0; in xpc_online_heartbeat_uv()
779 xpc_heartbeat_uv->value = 1; in xpc_heartbeat_init_uv()
780 xpc_heartbeat_uv->offline = 0; in xpc_heartbeat_init_uv()
792 struct xpc_partition_uv *part_uv = &part->sn.uv; in xpc_get_remote_heartbeat_uv()
795 ret = xp_remote_memcpy(uv_gpa(&part_uv->cached_heartbeat), in xpc_get_remote_heartbeat_uv()
796 part_uv->heartbeat_gpa, in xpc_get_remote_heartbeat_uv()
801 if (part_uv->cached_heartbeat.value == part->last_heartbeat && in xpc_get_remote_heartbeat_uv()
802 !part_uv->cached_heartbeat.offline) { in xpc_get_remote_heartbeat_uv()
806 part->last_heartbeat = part_uv->cached_heartbeat.value; in xpc_get_remote_heartbeat_uv()
815 short partid = remote_rp->SAL_partid; in xpc_request_partition_activation_uv()
819 part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */ in xpc_request_partition_activation_uv()
820 part->remote_rp_ts_jiffies = remote_rp->ts_jiffies; in xpc_request_partition_activation_uv()
821 part->sn.uv.heartbeat_gpa = remote_rp->sn.uv.heartbeat_gpa; in xpc_request_partition_activation_uv()
822 part->sn.uv.activate_gru_mq_desc_gpa = in xpc_request_partition_activation_uv()
823 remote_rp->sn.uv.activate_gru_mq_desc_gpa; in xpc_request_partition_activation_uv()
829 if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) { in xpc_request_partition_activation_uv()
831 msg.heartbeat_gpa = xpc_rsvd_page->sn.uv.heartbeat_gpa; in xpc_request_partition_activation_uv()
833 xpc_rsvd_page->sn.uv.activate_gru_mq_desc_gpa; in xpc_request_partition_activation_uv()
838 if (part->act_state == XPC_P_AS_INACTIVE) in xpc_request_partition_activation_uv()
857 if (part->sn.uv.remote_act_state != XPC_P_AS_DEACTIVATING && in xpc_request_partition_deactivation_uv()
858 part->sn.uv.remote_act_state != XPC_P_AS_INACTIVE) { in xpc_request_partition_deactivation_uv()
860 msg.reason = part->reason; in xpc_request_partition_deactivation_uv()
876 head->first = NULL; in xpc_init_fifo_uv()
877 head->last = NULL; in xpc_init_fifo_uv()
878 spin_lock_init(&head->lock); in xpc_init_fifo_uv()
879 head->n_entries = 0; in xpc_init_fifo_uv()
888 spin_lock_irqsave(&head->lock, irq_flags); in xpc_get_fifo_entry_uv()
889 first = head->first; in xpc_get_fifo_entry_uv()
890 if (head->first != NULL) { in xpc_get_fifo_entry_uv()
891 head->first = first->next; in xpc_get_fifo_entry_uv()
892 if (head->first == NULL) in xpc_get_fifo_entry_uv()
893 head->last = NULL; in xpc_get_fifo_entry_uv()
895 head->n_entries--; in xpc_get_fifo_entry_uv()
896 BUG_ON(head->n_entries < 0); in xpc_get_fifo_entry_uv()
898 first->next = NULL; in xpc_get_fifo_entry_uv()
900 spin_unlock_irqrestore(&head->lock, irq_flags); in xpc_get_fifo_entry_uv()
910 last->next = NULL; in xpc_put_fifo_entry_uv()
911 spin_lock_irqsave(&head->lock, irq_flags); in xpc_put_fifo_entry_uv()
912 if (head->last != NULL) in xpc_put_fifo_entry_uv()
913 head->last->next = last; in xpc_put_fifo_entry_uv()
915 head->first = last; in xpc_put_fifo_entry_uv()
916 head->last = last; in xpc_put_fifo_entry_uv()
917 head->n_entries++; in xpc_put_fifo_entry_uv()
918 spin_unlock_irqrestore(&head->lock, irq_flags); in xpc_put_fifo_entry_uv()
924 return head->n_entries; in xpc_n_of_fifo_entries_uv()
936 for (ch_number = 0; ch_number < part->nchannels; ch_number++) { in xpc_setup_ch_structures_uv()
937 ch_uv = &part->channels[ch_number].sn.uv; in xpc_setup_ch_structures_uv()
939 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); in xpc_setup_ch_structures_uv()
940 xpc_init_fifo_uv(&ch_uv->recv_msg_list); in xpc_setup_ch_structures_uv()
969 while (!((part->sn.uv.remote_act_state == XPC_P_AS_ACTIVATING) || in xpc_make_first_contact_uv()
970 (part->sn.uv.remote_act_state == XPC_P_AS_ACTIVE))) { in xpc_make_first_contact_uv()
978 if (part->act_state == XPC_P_AS_DEACTIVATING) in xpc_make_first_contact_uv()
979 return part->reason; in xpc_make_first_contact_uv()
991 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_get_chctl_all_flags_uv()
992 chctl = part->chctl; in xpc_get_chctl_all_flags_uv()
994 part->chctl.all_flags = 0; in xpc_get_chctl_all_flags_uv()
996 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_get_chctl_all_flags_uv()
1001 xpc_allocate_send_msg_slot_uv(struct xpc_channel *ch) in xpc_allocate_send_msg_slot_uv() argument
1003 struct xpc_channel_uv *ch_uv = &ch->sn.uv; in xpc_allocate_send_msg_slot_uv()
1010 for (nentries = ch->local_nentries; nentries > 0; nentries--) { in xpc_allocate_send_msg_slot_uv()
1012 ch_uv->send_msg_slots = kzalloc(nbytes, GFP_KERNEL); in xpc_allocate_send_msg_slot_uv()
1013 if (ch_uv->send_msg_slots == NULL) in xpc_allocate_send_msg_slot_uv()
1017 msg_slot = &ch_uv->send_msg_slots[entry]; in xpc_allocate_send_msg_slot_uv()
1019 msg_slot->msg_slot_number = entry; in xpc_allocate_send_msg_slot_uv()
1020 xpc_put_fifo_entry_uv(&ch_uv->msg_slot_free_list, in xpc_allocate_send_msg_slot_uv()
1021 &msg_slot->next); in xpc_allocate_send_msg_slot_uv()
1024 spin_lock_irqsave(&ch->lock, irq_flags); in xpc_allocate_send_msg_slot_uv()
1025 if (nentries < ch->local_nentries) in xpc_allocate_send_msg_slot_uv()
1026 ch->local_nentries = nentries; in xpc_allocate_send_msg_slot_uv()
1027 spin_unlock_irqrestore(&ch->lock, irq_flags); in xpc_allocate_send_msg_slot_uv()
1035 xpc_allocate_recv_msg_slot_uv(struct xpc_channel *ch) in xpc_allocate_recv_msg_slot_uv() argument
1037 struct xpc_channel_uv *ch_uv = &ch->sn.uv; in xpc_allocate_recv_msg_slot_uv()
1044 for (nentries = ch->remote_nentries; nentries > 0; nentries--) { in xpc_allocate_recv_msg_slot_uv()
1045 nbytes = nentries * ch->entry_size; in xpc_allocate_recv_msg_slot_uv()
1046 ch_uv->recv_msg_slots = kzalloc(nbytes, GFP_KERNEL); in xpc_allocate_recv_msg_slot_uv()
1047 if (ch_uv->recv_msg_slots == NULL) in xpc_allocate_recv_msg_slot_uv()
1051 msg_slot = ch_uv->recv_msg_slots + in xpc_allocate_recv_msg_slot_uv()
1052 entry * ch->entry_size; in xpc_allocate_recv_msg_slot_uv()
1054 msg_slot->hdr.msg_slot_number = entry; in xpc_allocate_recv_msg_slot_uv()
1057 spin_lock_irqsave(&ch->lock, irq_flags); in xpc_allocate_recv_msg_slot_uv()
1058 if (nentries < ch->remote_nentries) in xpc_allocate_recv_msg_slot_uv()
1059 ch->remote_nentries = nentries; in xpc_allocate_recv_msg_slot_uv()
1060 spin_unlock_irqrestore(&ch->lock, irq_flags); in xpc_allocate_recv_msg_slot_uv()
1071 xpc_setup_msg_structures_uv(struct xpc_channel *ch) in xpc_setup_msg_structures_uv() argument
1074 struct xpc_channel_uv *ch_uv = &ch->sn.uv; in xpc_setup_msg_structures_uv()
1076 DBUG_ON(ch->flags & XPC_C_SETUP); in xpc_setup_msg_structures_uv()
1078 ch_uv->cached_notify_gru_mq_desc = kmalloc(sizeof(struct in xpc_setup_msg_structures_uv()
1081 if (ch_uv->cached_notify_gru_mq_desc == NULL) in xpc_setup_msg_structures_uv()
1084 ret = xpc_allocate_send_msg_slot_uv(ch); in xpc_setup_msg_structures_uv()
1087 ret = xpc_allocate_recv_msg_slot_uv(ch); in xpc_setup_msg_structures_uv()
1089 kfree(ch_uv->send_msg_slots); in xpc_setup_msg_structures_uv()
1090 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); in xpc_setup_msg_structures_uv()
1101 xpc_teardown_msg_structures_uv(struct xpc_channel *ch) in xpc_teardown_msg_structures_uv() argument
1103 struct xpc_channel_uv *ch_uv = &ch->sn.uv; in xpc_teardown_msg_structures_uv()
1105 lockdep_assert_held(&ch->lock); in xpc_teardown_msg_structures_uv()
1107 kfree(ch_uv->cached_notify_gru_mq_desc); in xpc_teardown_msg_structures_uv()
1108 ch_uv->cached_notify_gru_mq_desc = NULL; in xpc_teardown_msg_structures_uv()
1110 if (ch->flags & XPC_C_SETUP) { in xpc_teardown_msg_structures_uv()
1111 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); in xpc_teardown_msg_structures_uv()
1112 kfree(ch_uv->send_msg_slots); in xpc_teardown_msg_structures_uv()
1113 xpc_init_fifo_uv(&ch_uv->recv_msg_list); in xpc_teardown_msg_structures_uv()
1114 kfree(ch_uv->recv_msg_slots); in xpc_teardown_msg_structures_uv()
1119 xpc_send_chctl_closerequest_uv(struct xpc_channel *ch, unsigned long *irq_flags) in xpc_send_chctl_closerequest_uv() argument
1123 msg.ch_number = ch->number; in xpc_send_chctl_closerequest_uv()
1124 msg.reason = ch->reason; in xpc_send_chctl_closerequest_uv()
1125 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), in xpc_send_chctl_closerequest_uv()
1130 xpc_send_chctl_closereply_uv(struct xpc_channel *ch, unsigned long *irq_flags) in xpc_send_chctl_closereply_uv() argument
1134 msg.ch_number = ch->number; in xpc_send_chctl_closereply_uv()
1135 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), in xpc_send_chctl_closereply_uv()
1140 xpc_send_chctl_openrequest_uv(struct xpc_channel *ch, unsigned long *irq_flags) in xpc_send_chctl_openrequest_uv() argument
1144 msg.ch_number = ch->number; in xpc_send_chctl_openrequest_uv()
1145 msg.entry_size = ch->entry_size; in xpc_send_chctl_openrequest_uv()
1146 msg.local_nentries = ch->local_nentries; in xpc_send_chctl_openrequest_uv()
1147 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), in xpc_send_chctl_openrequest_uv()
1152 xpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags) in xpc_send_chctl_openreply_uv() argument
1156 msg.ch_number = ch->number; in xpc_send_chctl_openreply_uv()
1157 msg.local_nentries = ch->local_nentries; in xpc_send_chctl_openreply_uv()
1158 msg.remote_nentries = ch->remote_nentries; in xpc_send_chctl_openreply_uv()
1159 msg.notify_gru_mq_desc_gpa = uv_gpa(xpc_notify_mq_uv->gru_mq_desc); in xpc_send_chctl_openreply_uv()
1160 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), in xpc_send_chctl_openreply_uv()
1165 xpc_send_chctl_opencomplete_uv(struct xpc_channel *ch, unsigned long *irq_flags) in xpc_send_chctl_opencomplete_uv() argument
1169 msg.ch_number = ch->number; in xpc_send_chctl_opencomplete_uv()
1170 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), in xpc_send_chctl_opencomplete_uv()
1179 spin_lock_irqsave(&part->chctl_lock, irq_flags); in xpc_send_chctl_local_msgrequest_uv()
1180 part->chctl.flags[ch_number] |= XPC_CHCTL_MSGREQUEST; in xpc_send_chctl_local_msgrequest_uv()
1181 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); in xpc_send_chctl_local_msgrequest_uv()
1187 xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch, in xpc_save_remote_msgqueue_pa_uv() argument
1190 struct xpc_channel_uv *ch_uv = &ch->sn.uv; in xpc_save_remote_msgqueue_pa_uv()
1192 DBUG_ON(ch_uv->cached_notify_gru_mq_desc == NULL); in xpc_save_remote_msgqueue_pa_uv()
1193 return xpc_cache_remote_gru_mq_desc_uv(ch_uv->cached_notify_gru_mq_desc, in xpc_save_remote_msgqueue_pa_uv()
1221 spin_lock_irqsave(&part_uv->flags_lock, irq_flags); in xpc_assume_partition_disengaged_uv()
1222 part_uv->flags &= ~XPC_P_ENGAGED_UV; in xpc_assume_partition_disengaged_uv()
1223 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); in xpc_assume_partition_disengaged_uv()
1240 if ((part_uv->flags & XPC_P_ENGAGED_UV) != 0) in xpc_any_partition_engaged_uv()
1247 xpc_allocate_msg_slot_uv(struct xpc_channel *ch, u32 flags, in xpc_allocate_msg_slot_uv() argument
1255 entry = xpc_get_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list); in xpc_allocate_msg_slot_uv()
1262 ret = xpc_allocate_msg_wait(ch); in xpc_allocate_msg_slot_uv()
1273 xpc_free_msg_slot_uv(struct xpc_channel *ch, in xpc_free_msg_slot_uv() argument
1276 xpc_put_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list, &msg_slot->next); in xpc_free_msg_slot_uv()
1279 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) in xpc_free_msg_slot_uv()
1280 wake_up(&ch->msg_allocate_wq); in xpc_free_msg_slot_uv()
1284 xpc_notify_sender_uv(struct xpc_channel *ch, in xpc_notify_sender_uv() argument
1288 xpc_notify_func func = msg_slot->func; in xpc_notify_sender_uv() local
1290 if (func != NULL && cmpxchg(&msg_slot->func, func, NULL) == func) { in xpc_notify_sender_uv()
1292 atomic_dec(&ch->n_to_notify); in xpc_notify_sender_uv()
1294 dev_dbg(xpc_chan, "msg_slot->func() called, msg_slot=0x%p " in xpc_notify_sender_uv()
1296 msg_slot->msg_slot_number, ch->partid, ch->number); in xpc_notify_sender_uv()
1298 func(reason, ch->partid, ch->number, msg_slot->key); in xpc_notify_sender_uv()
1300 dev_dbg(xpc_chan, "msg_slot->func() returned, msg_slot=0x%p " in xpc_notify_sender_uv()
1302 msg_slot->msg_slot_number, ch->partid, ch->number); in xpc_notify_sender_uv()
1307 xpc_handle_notify_mq_ack_uv(struct xpc_channel *ch, in xpc_handle_notify_mq_ack_uv() argument
1311 int entry = msg->hdr.msg_slot_number % ch->local_nentries; in xpc_handle_notify_mq_ack_uv()
1313 msg_slot = &ch->sn.uv.send_msg_slots[entry]; in xpc_handle_notify_mq_ack_uv()
1315 BUG_ON(msg_slot->msg_slot_number != msg->hdr.msg_slot_number); in xpc_handle_notify_mq_ack_uv()
1316 msg_slot->msg_slot_number += ch->local_nentries; in xpc_handle_notify_mq_ack_uv()
1318 if (msg_slot->func != NULL) in xpc_handle_notify_mq_ack_uv()
1319 xpc_notify_sender_uv(ch, msg_slot, xpMsgDelivered); in xpc_handle_notify_mq_ack_uv()
1321 xpc_free_msg_slot_uv(ch, msg_slot); in xpc_handle_notify_mq_ack_uv()
1328 struct xpc_partition_uv *part_uv = &part->sn.uv; in xpc_handle_notify_mq_msg_uv()
1329 struct xpc_channel *ch; in xpc_handle_notify_mq_msg_uv() local
1333 int ch_number = msg->hdr.ch_number; in xpc_handle_notify_mq_msg_uv()
1335 if (unlikely(ch_number >= part->nchannels)) { in xpc_handle_notify_mq_msg_uv()
1342 if (part_uv->act_state_req == 0) in xpc_handle_notify_mq_msg_uv()
1344 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV; in xpc_handle_notify_mq_msg_uv()
1345 part_uv->reason = xpBadChannelNumber; in xpc_handle_notify_mq_msg_uv()
1352 ch = &part->channels[ch_number]; in xpc_handle_notify_mq_msg_uv()
1353 xpc_msgqueue_ref(ch); in xpc_handle_notify_mq_msg_uv()
1355 if (!(ch->flags & XPC_C_CONNECTED)) { in xpc_handle_notify_mq_msg_uv()
1356 xpc_msgqueue_deref(ch); in xpc_handle_notify_mq_msg_uv()
1361 if (msg->hdr.size == 0) { in xpc_handle_notify_mq_msg_uv()
1362 xpc_handle_notify_mq_ack_uv(ch, msg); in xpc_handle_notify_mq_msg_uv()
1363 xpc_msgqueue_deref(ch); in xpc_handle_notify_mq_msg_uv()
1368 ch_uv = &ch->sn.uv; in xpc_handle_notify_mq_msg_uv()
1370 msg_slot = ch_uv->recv_msg_slots + in xpc_handle_notify_mq_msg_uv()
1371 (msg->hdr.msg_slot_number % ch->remote_nentries) * ch->entry_size; in xpc_handle_notify_mq_msg_uv()
1373 BUG_ON(msg_slot->hdr.size != 0); in xpc_handle_notify_mq_msg_uv()
1375 memcpy(msg_slot, msg, msg->hdr.size); in xpc_handle_notify_mq_msg_uv()
1377 xpc_put_fifo_entry_uv(&ch_uv->recv_msg_list, &msg_slot->hdr.u.next); in xpc_handle_notify_mq_msg_uv()
1379 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) { in xpc_handle_notify_mq_msg_uv()
1385 if (atomic_read(&ch->kthreads_idle) > 0) in xpc_handle_notify_mq_msg_uv()
1386 wake_up_nr(&ch->idle_wq, 1); in xpc_handle_notify_mq_msg_uv()
1388 xpc_send_chctl_local_msgrequest_uv(part, ch->number); in xpc_handle_notify_mq_msg_uv()
1390 xpc_msgqueue_deref(ch); in xpc_handle_notify_mq_msg_uv()
1400 while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) != in xpc_handle_notify_IRQ_uv()
1403 partid = msg->hdr.partid; in xpc_handle_notify_IRQ_uv()
1416 gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg); in xpc_handle_notify_IRQ_uv()
1423 xpc_n_of_deliverable_payloads_uv(struct xpc_channel *ch) in xpc_n_of_deliverable_payloads_uv() argument
1425 return xpc_n_of_fifo_entries_uv(&ch->sn.uv.recv_msg_list); in xpc_n_of_deliverable_payloads_uv()
1431 struct xpc_channel *ch = &part->channels[ch_number]; in xpc_process_msg_chctl_flags_uv() local
1434 xpc_msgqueue_ref(ch); in xpc_process_msg_chctl_flags_uv()
1436 ndeliverable_payloads = xpc_n_of_deliverable_payloads_uv(ch); in xpc_process_msg_chctl_flags_uv()
1439 (ch->flags & XPC_C_CONNECTED) && in xpc_process_msg_chctl_flags_uv()
1440 (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)) { in xpc_process_msg_chctl_flags_uv()
1442 xpc_activate_kthreads(ch, ndeliverable_payloads); in xpc_process_msg_chctl_flags_uv()
1445 xpc_msgqueue_deref(ch); in xpc_process_msg_chctl_flags_uv()
1449 xpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload, in xpc_send_payload_uv() argument
1450 u16 payload_size, u8 notify_type, xpc_notify_func func, in xpc_send_payload_uv() argument
1462 if (msg_size > ch->entry_size) in xpc_send_payload_uv()
1465 xpc_msgqueue_ref(ch); in xpc_send_payload_uv()
1467 if (ch->flags & XPC_C_DISCONNECTING) { in xpc_send_payload_uv()
1468 ret = ch->reason; in xpc_send_payload_uv()
1471 if (!(ch->flags & XPC_C_CONNECTED)) { in xpc_send_payload_uv()
1476 ret = xpc_allocate_msg_slot_uv(ch, flags, &msg_slot); in xpc_send_payload_uv()
1480 if (func != NULL) { in xpc_send_payload_uv()
1481 atomic_inc(&ch->n_to_notify); in xpc_send_payload_uv()
1483 msg_slot->key = key; in xpc_send_payload_uv()
1484 smp_wmb(); /* a non-NULL func must hit memory after the key */ in xpc_send_payload_uv()
1485 msg_slot->func = func; in xpc_send_payload_uv()
1487 if (ch->flags & XPC_C_DISCONNECTING) { in xpc_send_payload_uv()
1488 ret = ch->reason; in xpc_send_payload_uv()
1494 msg->hdr.partid = xp_partition_id; in xpc_send_payload_uv()
1495 msg->hdr.ch_number = ch->number; in xpc_send_payload_uv()
1496 msg->hdr.size = msg_size; in xpc_send_payload_uv()
1497 msg->hdr.msg_slot_number = msg_slot->msg_slot_number; in xpc_send_payload_uv()
1498 memcpy(&msg->payload, payload, payload_size); in xpc_send_payload_uv()
1500 ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg, in xpc_send_payload_uv()
1505 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret); in xpc_send_payload_uv()
1507 if (func != NULL) { in xpc_send_payload_uv()
1509 * Try to NULL the msg_slot's func field. If we fail, then in xpc_send_payload_uv()
1518 if (cmpxchg(&msg_slot->func, func, NULL) != func) { in xpc_send_payload_uv()
1523 msg_slot->key = NULL; in xpc_send_payload_uv()
1524 atomic_dec(&ch->n_to_notify); in xpc_send_payload_uv()
1526 xpc_free_msg_slot_uv(ch, msg_slot); in xpc_send_payload_uv()
1528 xpc_msgqueue_deref(ch); in xpc_send_payload_uv()
1540 xpc_notify_senders_of_disconnect_uv(struct xpc_channel *ch) in xpc_notify_senders_of_disconnect_uv() argument
1545 DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING)); in xpc_notify_senders_of_disconnect_uv()
1547 for (entry = 0; entry < ch->local_nentries; entry++) { in xpc_notify_senders_of_disconnect_uv()
1549 if (atomic_read(&ch->n_to_notify) == 0) in xpc_notify_senders_of_disconnect_uv()
1552 msg_slot = &ch->sn.uv.send_msg_slots[entry]; in xpc_notify_senders_of_disconnect_uv()
1553 if (msg_slot->func != NULL) in xpc_notify_senders_of_disconnect_uv()
1554 xpc_notify_sender_uv(ch, msg_slot, ch->reason); in xpc_notify_senders_of_disconnect_uv()
1562 xpc_get_deliverable_payload_uv(struct xpc_channel *ch) in xpc_get_deliverable_payload_uv() argument
1568 if (!(ch->flags & XPC_C_DISCONNECTING)) { in xpc_get_deliverable_payload_uv()
1569 entry = xpc_get_fifo_entry_uv(&ch->sn.uv.recv_msg_list); in xpc_get_deliverable_payload_uv()
1573 payload = &msg->payload; in xpc_get_deliverable_payload_uv()
1580 xpc_received_payload_uv(struct xpc_channel *ch, void *payload) in xpc_received_payload_uv() argument
1589 msg->hdr.partid = xp_partition_id; in xpc_received_payload_uv()
1590 msg->hdr.size = 0; /* size of zero indicates this is an ACK */ in xpc_received_payload_uv()
1592 ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg, in xpc_received_payload_uv()
1595 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret); in xpc_received_payload_uv()
1704 return -E2BIG; in xpc_init_uv()
1719 -ret); in xpc_init_uv()