206 	struct tipc_aead __rcu *aead[KEY_MAX + 1];  member
240 	struct tipc_aead *aead;  member
247 	struct tipc_aead *aead;  member
251 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
252 static inline void tipc_aead_put(struct tipc_aead *aead);
254 static int tipc_aead_users(struct tipc_aead __rcu *aead);
255 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
256 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
257 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
258 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
259 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
266 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
271 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
275 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
283 				  struct tipc_aead *aead, u8 pos,
296 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
383 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead)  in tipc_aead_get()  argument
388 	tmp = rcu_dereference(aead);  in tipc_aead_get()
396 static inline void tipc_aead_put(struct tipc_aead *aead)  in tipc_aead_put()  argument
398 	if (aead && refcount_dec_and_test(&aead->refcnt))  in tipc_aead_put()
399 		call_rcu(&aead->rcu, tipc_aead_free);  in tipc_aead_put()
408 	struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu);  in tipc_aead_free()  local
411 	if (aead->cloned) {  in tipc_aead_free()
412 		tipc_aead_put(aead->cloned);  in tipc_aead_free()
414 		head = *get_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
415 		put_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
426 	free_percpu(aead->tfm_entry);  in tipc_aead_free()
427 	kfree_sensitive(aead->key);  in tipc_aead_free()
428 	kfree_sensitive(aead);  in tipc_aead_free()
431 static int tipc_aead_users(struct tipc_aead __rcu *aead)  in tipc_aead_users()  argument
437 	tmp = rcu_dereference(aead);  in tipc_aead_users()
445 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_inc()  argument
450 	tmp = rcu_dereference(aead);  in tipc_aead_users_inc()
456 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_dec()  argument
461 	tmp = rcu_dereference(aead);  in tipc_aead_users_dec()
463 		atomic_add_unless(&rcu_dereference(aead)->users, -1, lim);  in tipc_aead_users_dec()
467 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val)  in tipc_aead_users_set()  argument
473 	tmp = rcu_dereference(aead);  in tipc_aead_users_set()
488 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead)  in tipc_aead_tfm_next()  argument
493 	tfm_entry = get_cpu_ptr(aead->tfm_entry);  in tipc_aead_tfm_next()
514 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,  in tipc_aead_init()  argument
523 	if (unlikely(*aead))  in tipc_aead_init()
609 	*aead = tmp;  in tipc_aead_init()
628 	struct tipc_aead *aead;  in tipc_aead_clone()  local
640 	aead = kzalloc(sizeof(*aead), GFP_ATOMIC);  in tipc_aead_clone()
641 	if (unlikely(!aead))  in tipc_aead_clone()
644 	aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC);  in tipc_aead_clone()
645 	if (unlikely(!aead->tfm_entry)) {  in tipc_aead_clone()
646 		kfree_sensitive(aead);  in tipc_aead_clone()
651 		*per_cpu_ptr(aead->tfm_entry, cpu) =  in tipc_aead_clone()
655 	memcpy(aead->hint, src->hint, sizeof(src->hint));  in tipc_aead_clone()
656 	aead->mode = src->mode;  in tipc_aead_clone()
657 	aead->salt = src->salt;  in tipc_aead_clone()
658 	aead->authsize = src->authsize;  in tipc_aead_clone()
659 	atomic_set(&aead->users, 0);  in tipc_aead_clone()
660 	atomic64_set(&aead->seqno, 0);  in tipc_aead_clone()
661 	refcount_set(&aead->refcnt, 1);  in tipc_aead_clone()
664 	aead->cloned = src;  in tipc_aead_clone()
666 	*dst = aead;  in tipc_aead_clone()
732 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,  in tipc_aead_encrypt()  argument
737 	struct crypto_aead *tfm = tipc_aead_tfm_next(aead);  in tipc_aead_encrypt()
750 	tailen = len - skb->len + aead->authsize;  in tipc_aead_encrypt()
792 	salt = aead->salt;  in tipc_aead_encrypt()
793 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_encrypt()
810 	tx_ctx->aead = aead;  in tipc_aead_encrypt()
821 	if (!maybe_get_net(aead->crypto->net)) {  in tipc_aead_encrypt()
833 	put_net(aead->crypto->net);  in tipc_aead_encrypt()
846 	struct tipc_aead *aead = tx_ctx->aead;  in tipc_aead_encrypt_done()  local
847 	struct tipc_crypto *tx = aead->crypto;  in tipc_aead_encrypt_done()
870 	tipc_aead_put(aead);  in tipc_aead_encrypt_done()
886 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,  in tipc_aead_decrypt()  argument
900 	if (unlikely(!aead))  in tipc_aead_decrypt()
910 	tfm = tipc_aead_tfm_next(aead);  in tipc_aead_decrypt()
926 	salt = aead->salt;  in tipc_aead_decrypt()
927 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_decrypt()
944 	rx_ctx->aead = aead;  in tipc_aead_decrypt()
971 	struct tipc_aead *aead = rx_ctx->aead;  in tipc_aead_decrypt_done()  local
972 	struct tipc_crypto_stats __percpu *stats = aead->crypto->stats;  in tipc_aead_decrypt_done()
973 	struct net *net = aead->crypto->net;  in tipc_aead_decrypt_done()
987 	tipc_crypto_rcv_complete(net, aead, b, &skb, err);  in tipc_aead_decrypt_done()
1039 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,  in tipc_ehdr_build()  argument
1058 	if (!__rx || aead->mode == CLUSTER_KEY)  in tipc_ehdr_build()
1059 		seqno = atomic64_inc_return(&aead->seqno);  in tipc_ehdr_build()
1078 	ehdr->master_key = aead->crypto->key_master;  in tipc_ehdr_build()
1131 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_init()  local
1135 	rc = tipc_aead_init(&aead, ukey, mode);  in tipc_crypto_key_init()
1139 		rc = tipc_crypto_key_attach(c, aead, 0, master_key);  in tipc_crypto_key_init()
1141 			tipc_aead_free(&aead->rcu);  in tipc_crypto_key_init()
1157 				  struct tipc_aead *aead, u8 pos,  in tipc_crypto_key_attach()  argument
1173 		if (tipc_aead_users(c->aead[key.pending]) > 0)  in tipc_crypto_key_attach()
1195 	aead->crypto = c;  in tipc_crypto_key_attach()
1196 	aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen;  in tipc_crypto_key_attach()
1197 	tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock);  in tipc_crypto_key_attach()
1230 			tipc_aead_users_dec(tx->aead[k], 0);  in tipc_crypto_key_flush()
1239 		tipc_crypto_key_detach(c->aead[k], &c->lock);  in tipc_crypto_key_flush()
1275 	if (tipc_aead_users(rx->aead[key.pending]) > 0)  in tipc_crypto_key_try_align()
1279 	tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock);  in tipc_crypto_key_try_align()
1282 	rcu_assign_pointer(rx->aead[key.pending], NULL);  in tipc_crypto_key_try_align()
1286 		tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock));  in tipc_crypto_key_try_align()
1293 	rcu_assign_pointer(rx->aead[new_pending], tmp1);  in tipc_crypto_key_try_align()
1295 		rcu_assign_pointer(rx->aead[new_passive], tmp2);  in tipc_crypto_key_try_align()
1325 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_pick_tx()  local
1342 		aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock);  in tipc_crypto_key_pick_tx()
1350 		aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock);  in tipc_crypto_key_pick_tx()
1351 		if (!aead)  in tipc_crypto_key_pick_tx()
1353 		if (aead->mode != CLUSTER_KEY ||  in tipc_crypto_key_pick_tx()
1354 		    aead == skb_cb->tx_clone_ctx.last) {  in tipc_crypto_key_pick_tx()
1355 			aead = NULL;  in tipc_crypto_key_pick_tx()
1359 		skb_cb->tx_clone_ctx.last = aead;  in tipc_crypto_key_pick_tx()
1368 	if (likely(aead))  in tipc_crypto_key_pick_tx()
1369 		WARN_ON(!refcount_inc_not_zero(&aead->refcnt));  in tipc_crypto_key_pick_tx()
1372 	return aead;  in tipc_crypto_key_pick_tx()
1435 			tipc_aead_users_inc(tx->aead[new], INT_MAX);  in tipc_crypto_key_synch()
1437 			tipc_aead_users_dec(tx->aead[cur], 0);  in tipc_crypto_key_synch()
1459 	tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_key_revoke()
1540 		tipc_aead_put(rcu_dereference(c->aead[k]));  in tipc_crypto_stop()
1561 	if (key.active && tipc_aead_users(tx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1563 	if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1570 		tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_timeout()
1580 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1594 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10)  in tipc_crypto_timeout()
1598 	tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock);  in tipc_crypto_timeout()
1607 	    tipc_aead_users(rx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1616 	tipc_aead_users_set(rx->aead[key.pending], 0);  in tipc_crypto_timeout()
1625 	    tipc_aead_users(rx->aead[key.passive]) > -10)  in tipc_crypto_timeout()
1629 	tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock);  in tipc_crypto_timeout()
1698 	struct tipc_aead *aead = NULL;  in tipc_crypto_xmit()  local
1760 	aead = tipc_aead_get(tx->aead[tx_key]);  in tipc_crypto_xmit()
1761 	if (unlikely(!aead))  in tipc_crypto_xmit()
1763 	rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx);  in tipc_crypto_xmit()
1765 		rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode);  in tipc_crypto_xmit()
1788 	tipc_aead_put(aead);  in tipc_crypto_xmit()
1819 	struct tipc_aead *aead = NULL;  in tipc_crypto_rcv()  local
1844 	aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key);  in tipc_crypto_rcv()
1845 	if (aead)  in tipc_crypto_rcv()
1851 	if (!aead)  in tipc_crypto_rcv()
1852 		aead = tipc_aead_get(rx->aead[tx_key]);  in tipc_crypto_rcv()
1853 	rc = tipc_aead_decrypt(net, aead, *skb, b);  in tipc_crypto_rcv()
1879 					      rcu_access_pointer(rx->aead[n]));  in tipc_crypto_rcv()
1893 	tipc_crypto_rcv_complete(net, aead, b, skb, rc);  in tipc_crypto_rcv()
1897 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,  in tipc_crypto_rcv_complete()  argument
1902 	struct tipc_crypto *rx = aead->crypto;  in tipc_crypto_rcv_complete()
1908 	if (unlikely(is_tx(aead->crypto))) {  in tipc_crypto_rcv_complete()
1911 			 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead,  in tipc_crypto_rcv_complete()
1915 			 aead->crypto->aead[1], aead->crypto->aead[2],  in tipc_crypto_rcv_complete()
1916 			 aead->crypto->aead[3]);  in tipc_crypto_rcv_complete()
1940 		if (tipc_aead_clone(&tmp, aead) < 0)  in tipc_crypto_rcv_complete()
1947 		tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
1948 		aead = tmp;  in tipc_crypto_rcv_complete()
1952 		tipc_aead_users_dec((struct tipc_aead __force __rcu *)aead, INT_MIN);  in tipc_crypto_rcv_complete()
1957 	tipc_aead_users_set((struct tipc_aead __force __rcu *)aead, 1);  in tipc_crypto_rcv_complete()
1972 	if (pskb_trim(*skb, (*skb)->len - aead->authsize))  in tipc_crypto_rcv_complete()
2002 	tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
2083 	struct tipc_aead *aead;  in tipc_crypto_key_dump()  local
2109 		aead = rcu_dereference(c->aead[k]);  in tipc_crypto_key_dump()
2110 		if (aead)  in tipc_crypto_key_dump()
2113 				       aead->hint,  in tipc_crypto_key_dump()
2114 				       (aead->mode == CLUSTER_KEY) ? "c" : "p",  in tipc_crypto_key_dump()
2115 				       atomic_read(&aead->users),  in tipc_crypto_key_dump()
2116 				       refcount_read(&aead->refcnt));  in tipc_crypto_key_dump()
2203 	struct tipc_aead *aead;  in tipc_crypto_key_distr()  local
2212 		aead = tipc_aead_get(tx->aead[key]);  in tipc_crypto_key_distr()
2213 		if (likely(aead)) {  in tipc_crypto_key_distr()
2214 			rc = tipc_crypto_key_xmit(tx->net, aead->key,  in tipc_crypto_key_distr()
2215 						  aead->gen, aead->mode,  in tipc_crypto_key_distr()
2217 			tipc_aead_put(aead);  in tipc_crypto_key_distr()
2450 	struct tipc_aead *aead;  in tipc_crypto_work_tx()  local
2458 	aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]);  in tipc_crypto_work_tx()
2459 	if (unlikely(!aead)) {  in tipc_crypto_work_tx()
2466 	skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC);  in tipc_crypto_work_tx()