416 	struct noise_keypair *kp;  in noise_remote_index_lookup()  local
426 				kp = (struct noise_keypair *) i;  in noise_remote_index_lookup()
427 				r = kp->kp_remote;  in noise_remote_index_lookup()
565 	struct noise_keypair *kp;  in noise_remote_keypairs_clear()  local
568 	kp = atomic_load_ptr(&r->r_next);  in noise_remote_keypairs_clear()
570 	noise_keypair_drop(kp);  in noise_remote_keypairs_clear()
572 	kp = atomic_load_ptr(&r->r_current);  in noise_remote_keypairs_clear()
574 	noise_keypair_drop(kp);  in noise_remote_keypairs_clear()
576 	kp = atomic_load_ptr(&r->r_previous);  in noise_remote_keypairs_clear()
578 	noise_keypair_drop(kp);  in noise_remote_keypairs_clear()
586 	struct noise_keypair *kp;  in noise_remote_expire_current()  local
591 	kp = atomic_load_ptr(&r->r_next);  in noise_remote_expire_current()
592 	if (kp != NULL)  in noise_remote_expire_current()
593 		atomic_store_bool(&kp->kp_can_send, false);  in noise_remote_expire_current()
594 	kp = atomic_load_ptr(&r->r_current);  in noise_remote_expire_current()
595 	if (kp != NULL)  in noise_remote_expire_current()
596 		atomic_store_bool(&kp->kp_can_send, false);  in noise_remote_expire_current()
603     struct noise_keypair *kp)  in noise_add_new_keypair()  argument
614 	if (kp->kp_is_initiator) {  in noise_add_new_keypair()
623 		atomic_store_ptr(&r->r_current, kp);  in noise_add_new_keypair()
625 		atomic_store_ptr(&r->r_next, kp);  in noise_add_new_keypair()
636 	kp->kp_index.i_is_keypair = true;  in noise_add_new_keypair()
637 	kp->kp_index.i_local_index = r_i->i_local_index;  in noise_add_new_keypair()
638 	kp->kp_index.i_remote_index = r_i->i_remote_index;  in noise_add_new_keypair()
641 	CK_LIST_INSERT_BEFORE(r_i, &kp->kp_index, i_entry);  in noise_add_new_keypair()
652 	struct noise_keypair *kp;  in noise_begin_session()  local
656 	if ((kp = malloc(sizeof(*kp), M_NOISE, M_NOWAIT | M_ZERO)) == NULL)  in noise_begin_session()
659 	refcount_init(&kp->kp_refcnt, 1);  in noise_begin_session()
660 	kp->kp_can_send = true;  in noise_begin_session()
661 	kp->kp_is_initiator = r->r_handshake_state == HANDSHAKE_INITIATOR;  in noise_begin_session()
662 	kp->kp_birthdate = getsbinuptime();  in noise_begin_session()
663 	kp->kp_remote = noise_remote_ref(r);  in noise_begin_session()
665 	if (kp->kp_is_initiator)  in noise_begin_session()
666 		noise_kdf(kp->kp_send, kp->kp_recv, NULL, NULL,  in noise_begin_session()
670 		noise_kdf(kp->kp_recv, kp->kp_send, NULL, NULL,  in noise_begin_session()
674 	rw_init(&kp->kp_nonce_lock, "noise_nonce");  in noise_begin_session()
676 	noise_add_new_keypair(r->r_local, r, kp);  in noise_begin_session()
685 	struct noise_keypair *kp, *ret = NULL;  in noise_keypair_lookup()  local
691 			kp = (struct noise_keypair *) i;  in noise_keypair_lookup()
692 			if (refcount_acquire_if_not_zero(&kp->kp_refcnt))  in noise_keypair_lookup()
693 				ret = kp;  in noise_keypair_lookup()
705 	struct noise_keypair *kp, *ret = NULL;  in noise_keypair_current()  local
708 	kp = atomic_load_ptr(&r->r_current);  in noise_keypair_current()
709 	if (kp != NULL && atomic_load_bool(&kp->kp_can_send)) {  in noise_keypair_current()
710 		if (noise_timer_expired(kp->kp_birthdate, REJECT_AFTER_TIME, 0))  in noise_keypair_current()
711 			atomic_store_bool(&kp->kp_can_send, false);  in noise_keypair_current()
712 		else if (refcount_acquire_if_not_zero(&kp->kp_refcnt))  in noise_keypair_current()
713 			ret = kp;  in noise_keypair_current()
720 noise_keypair_ref(struct noise_keypair *kp)  in noise_keypair_ref()  argument
722 	refcount_acquire(&kp->kp_refcnt);  in noise_keypair_ref()
723 	return (kp);  in noise_keypair_ref()
727 noise_keypair_received_with(struct noise_keypair *kp)  in noise_keypair_received_with()  argument
730 	struct noise_remote *r = kp->kp_remote;  in noise_keypair_received_with()
732 	if (kp != atomic_load_ptr(&r->r_next))  in noise_keypair_received_with()
736 	if (kp != atomic_load_ptr(&r->r_next)) {  in noise_keypair_received_with()
744 	atomic_store_ptr(&r->r_current, kp);  in noise_keypair_received_with()
754 	struct noise_keypair *kp;  in noise_keypair_smr_free()  local
755 	kp = __containerof(smr, struct noise_keypair, kp_smr);  in noise_keypair_smr_free()
756 	noise_remote_put(kp->kp_remote);  in noise_keypair_smr_free()
757 	rw_destroy(&kp->kp_nonce_lock);  in noise_keypair_smr_free()
758 	zfree(kp, M_NOISE);  in noise_keypair_smr_free()
762 noise_keypair_put(struct noise_keypair *kp)  in noise_keypair_put()  argument
764 	if (refcount_release(&kp->kp_refcnt))  in noise_keypair_put()
765 		NET_EPOCH_CALL(noise_keypair_smr_free, &kp->kp_smr);  in noise_keypair_put()
769 noise_keypair_drop(struct noise_keypair *kp)  in noise_keypair_drop()  argument
774 	if (kp == NULL)  in noise_keypair_drop()
777 	r = kp->kp_remote;  in noise_keypair_drop()
781 	CK_LIST_REMOVE(&kp->kp_index, i_entry);  in noise_keypair_drop()
784 	noise_keypair_put(kp);  in noise_keypair_drop()
788 noise_keypair_remote(struct noise_keypair *kp)  in noise_keypair_remote()  argument
790 	return (noise_remote_ref(kp->kp_remote));  in noise_keypair_remote()
794 noise_keypair_nonce_next(struct noise_keypair *kp, uint64_t *send)  in noise_keypair_nonce_next()  argument
796 	if (!atomic_load_bool(&kp->kp_can_send))  in noise_keypair_nonce_next()
800 	*send = atomic_fetchadd_64(&kp->kp_nonce_send, 1);  in noise_keypair_nonce_next()
802 	rw_wlock(&kp->kp_nonce_lock);  in noise_keypair_nonce_next()
803 	*send = kp->kp_nonce_send++;  in noise_keypair_nonce_next()
804 	rw_wunlock(&kp->kp_nonce_lock);  in noise_keypair_nonce_next()
808 	atomic_store_bool(&kp->kp_can_send, false);  in noise_keypair_nonce_next()
813 noise_keypair_nonce_check(struct noise_keypair *kp, uint64_t recv)  in noise_keypair_nonce_check()  argument
818 	rw_wlock(&kp->kp_nonce_lock);  in noise_keypair_nonce_check()
820 	if (__predict_false(kp->kp_nonce_recv >= REJECT_AFTER_MESSAGES + 1 ||  in noise_keypair_nonce_check()
826 	if (__predict_false(recv + COUNTER_WINDOW_SIZE < kp->kp_nonce_recv))  in noise_keypair_nonce_check()
831 	if (__predict_true(recv > kp->kp_nonce_recv)) {  in noise_keypair_nonce_check()
832 		index_current = kp->kp_nonce_recv >> COUNTER_ORDER;  in noise_keypair_nonce_check()
835 			kp->kp_backtrack[  in noise_keypair_nonce_check()
839 		atomic_store_64(&kp->kp_nonce_recv, recv);  in noise_keypair_nonce_check()
841 		kp->kp_nonce_recv = recv;  in noise_keypair_nonce_check()
847 	if (kp->kp_backtrack[index] & bit)  in noise_keypair_nonce_check()
850 	kp->kp_backtrack[index] |= bit;  in noise_keypair_nonce_check()
853 	rw_wunlock(&kp->kp_nonce_lock);  in noise_keypair_nonce_check()
905 noise_keypair_encrypt(struct noise_keypair *kp, uint32_t *r_idx, uint64_t nonce, struct mbuf *m)  in noise_keypair_encrypt()  argument
909 	ret = chacha20poly1305_encrypt_mbuf(m, nonce, kp->kp_send);  in noise_keypair_encrypt()
913 	*r_idx = kp->kp_index.i_remote_index;  in noise_keypair_encrypt()
918 noise_keypair_decrypt(struct noise_keypair *kp, uint64_t nonce, struct mbuf *m)  in noise_keypair_decrypt()  argument
924 	cur_nonce = atomic_load_64(&kp->kp_nonce_recv);  in noise_keypair_decrypt()
926 	rw_rlock(&kp->kp_nonce_lock);  in noise_keypair_decrypt()
927 	cur_nonce = kp->kp_nonce_recv;  in noise_keypair_decrypt()
928 	rw_runlock(&kp->kp_nonce_lock);  in noise_keypair_decrypt()
932 	    noise_timer_expired(kp->kp_birthdate, REJECT_AFTER_TIME, 0))  in noise_keypair_decrypt()
935 	ret = chacha20poly1305_decrypt_mbuf(m, nonce, kp->kp_recv);  in noise_keypair_decrypt()