1 // SPDX-License-Identifier: GPL-2.0-only
11  * 		Split up af-specific functions
36 	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
38 	rcu_dereference_check((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
56 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)  in xfrm_state_hold_rcu()  argument
58 	return refcount_inc_not_zero(&x->refcnt);  in xfrm_state_hold_rcu()
67 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_dst_hash()
69 	return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);  in xfrm_dst_hash()
77 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_src_hash()
79 	return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);  in xfrm_src_hash()
86 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_spi_hash()
88 	return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);  in xfrm_spi_hash()
93 	lockdep_assert_held(&net->xfrm.xfrm_state_lock);  in xfrm_seq_hash()
95 	return __xfrm_seq_hash(seq, net->xfrm.state_hmask);  in xfrm_seq_hash()
104 				if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
110 		if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET)        \
114 			hlist_add_before_rcu(_n, &_x->by);                 \
125 	struct xfrm_state *x;  in xfrm_hash_transfer()  local
127 	hlist_for_each_entry_safe(x, tmp, list, bydst) {  in xfrm_hash_transfer()
130 		h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,  in xfrm_hash_transfer()
131 				    x->props.reqid, x->props.family,  in xfrm_hash_transfer()
133 		XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);  in xfrm_hash_transfer()
135 		h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,  in xfrm_hash_transfer()
136 				    x->props.family,  in xfrm_hash_transfer()
138 		XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);  in xfrm_hash_transfer()
140 		if (x->id.spi) {  in xfrm_hash_transfer()
141 			h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,  in xfrm_hash_transfer()
142 					    x->id.proto, x->props.family,  in xfrm_hash_transfer()
144 			XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,  in xfrm_hash_transfer()
145 					  x->xso.type);  in xfrm_hash_transfer()
148 		if (x->km.seq) {  in xfrm_hash_transfer()
149 			h = __xfrm_seq_hash(x->km.seq, nhashmask);  in xfrm_hash_transfer()
150 			XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,  in xfrm_hash_transfer()
151 					  x->xso.type);  in xfrm_hash_transfer()
169 	nsize = xfrm_hash_new_size(net->xfrm.state_hmask);  in xfrm_hash_resize()
192 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
193 	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
195 	nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;  in xfrm_hash_resize()
196 	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);  in xfrm_hash_resize()
197 	for (i = net->xfrm.state_hmask; i >= 0; i--)  in xfrm_hash_resize()
200 	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);  in xfrm_hash_resize()
201 	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);  in xfrm_hash_resize()
202 	oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);  in xfrm_hash_resize()
203 	ohashmask = net->xfrm.state_hmask;  in xfrm_hash_resize()
205 	rcu_assign_pointer(net->xfrm.state_bydst, ndst);  in xfrm_hash_resize()
206 	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);  in xfrm_hash_resize()
207 	rcu_assign_pointer(net->xfrm.state_byspi, nspi);  in xfrm_hash_resize()
208 	rcu_assign_pointer(net->xfrm.state_byseq, nseq);  in xfrm_hash_resize()
209 	net->xfrm.state_hmask = nhashmask;  in xfrm_hash_resize()
211 	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
212 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
230 int __xfrm_state_delete(struct xfrm_state *x);
232 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
234 void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
242 		return -EAFNOSUPPORT;  in xfrm_register_type()
244 #define X(afi, T, name) do {			\  in xfrm_register_type()  macro
245 		WARN_ON((afi)->type_ ## name);	\  in xfrm_register_type()
246 		(afi)->type_ ## name = (T);	\  in xfrm_register_type()
249 	switch (type->proto) {  in xfrm_register_type()
251 		X(afinfo, type, comp);  in xfrm_register_type()
254 		X(afinfo, type, ah);  in xfrm_register_type()
257 		X(afinfo, type, esp);  in xfrm_register_type()
260 		X(afinfo, type, ipip);  in xfrm_register_type()
263 		X(afinfo, type, dstopts);  in xfrm_register_type()
266 		X(afinfo, type, routing);  in xfrm_register_type()
269 		X(afinfo, type, ipip6);  in xfrm_register_type()
273 		err = -EPROTONOSUPPORT;  in xfrm_register_type()
276 #undef X  in xfrm_register_type()
289 #define X(afi, T, name) do {				\  in xfrm_unregister_type()  macro
290 		WARN_ON((afi)->type_ ## name != (T));	\  in xfrm_unregister_type()
291 		(afi)->type_ ## name = NULL;		\  in xfrm_unregister_type()
294 	switch (type->proto) {  in xfrm_unregister_type()
296 		X(afinfo, type, comp);  in xfrm_unregister_type()
299 		X(afinfo, type, ah);  in xfrm_unregister_type()
302 		X(afinfo, type, esp);  in xfrm_unregister_type()
305 		X(afinfo, type, ipip);  in xfrm_unregister_type()
308 		X(afinfo, type, dstopts);  in xfrm_unregister_type()
311 		X(afinfo, type, routing);  in xfrm_unregister_type()
314 		X(afinfo, type, ipip6);  in xfrm_unregister_type()
320 #undef X  in xfrm_unregister_type()
338 		type = afinfo->type_comp;  in xfrm_get_type()
341 		type = afinfo->type_ah;  in xfrm_get_type()
344 		type = afinfo->type_esp;  in xfrm_get_type()
347 		type = afinfo->type_ipip;  in xfrm_get_type()
350 		type = afinfo->type_dstopts;  in xfrm_get_type()
353 		type = afinfo->type_routing;  in xfrm_get_type()
356 		type = afinfo->type_ipip6;  in xfrm_get_type()
362 	if (unlikely(type && !try_module_get(type->owner)))  in xfrm_get_type()
368 		request_module("xfrm-type-%d-%d", family, proto);  in xfrm_get_type()
378 	module_put(type->owner);  in xfrm_put_type()
388 		return -EAFNOSUPPORT;  in xfrm_register_type_offload()
390 	switch (type->proto) {  in xfrm_register_type_offload()
392 		WARN_ON(afinfo->type_offload_esp);  in xfrm_register_type_offload()
393 		afinfo->type_offload_esp = type;  in xfrm_register_type_offload()
397 		err = -EPROTONOSUPPORT;  in xfrm_register_type_offload()
414 	switch (type->proto) {  in xfrm_unregister_type_offload()
416 		WARN_ON(afinfo->type_offload_esp != type);  in xfrm_unregister_type_offload()
417 		afinfo->type_offload_esp = NULL;  in xfrm_unregister_type_offload()
427 void xfrm_set_type_offload(struct xfrm_state *x, bool try_load)  in xfrm_set_type_offload()  argument
433 	afinfo = xfrm_state_get_afinfo(x->props.family);  in xfrm_set_type_offload()
437 	switch (x->id.proto) {  in xfrm_set_type_offload()
439 		type = afinfo->type_offload_esp;  in xfrm_set_type_offload()
445 	if ((type && !try_module_get(type->owner)))  in xfrm_set_type_offload()
451 		request_module("xfrm-offload-%d-%d", x->props.family,  in xfrm_set_type_offload()
452 			       x->id.proto);  in xfrm_set_type_offload()
458 	x->type_offload = type;  in xfrm_set_type_offload()
520 		if (mode->family == family)  in xfrm_get_mode()
525 		if (mode->family == family)  in xfrm_get_mode()
541 		return -EINVAL;  in xfrm_register_mode_cbs()
575 	if (cbs && !try_module_get(cbs->owner))  in xfrm_get_mode_cbs()
581 		request_module("xfrm-iptfs");  in xfrm_get_mode_cbs()
589 void xfrm_state_free(struct xfrm_state *x)  in xfrm_state_free()  argument
591 	kmem_cache_free(xfrm_state_cache, x);  in xfrm_state_free()
595 static void xfrm_state_gc_destroy(struct xfrm_state *x)  in xfrm_state_gc_destroy()  argument
597 	if (x->mode_cbs && x->mode_cbs->destroy_state)  in xfrm_state_gc_destroy()
598 		x->mode_cbs->destroy_state(x);  in xfrm_state_gc_destroy()
599 	hrtimer_cancel(&x->mtimer);  in xfrm_state_gc_destroy()
600 	timer_delete_sync(&x->rtimer);  in xfrm_state_gc_destroy()
601 	kfree_sensitive(x->aead);  in xfrm_state_gc_destroy()
602 	kfree_sensitive(x->aalg);  in xfrm_state_gc_destroy()
603 	kfree_sensitive(x->ealg);  in xfrm_state_gc_destroy()
604 	kfree(x->calg);  in xfrm_state_gc_destroy()
605 	kfree(x->encap);  in xfrm_state_gc_destroy()
606 	kfree(x->coaddr);  in xfrm_state_gc_destroy()
607 	kfree(x->replay_esn);  in xfrm_state_gc_destroy()
608 	kfree(x->preplay_esn);  in xfrm_state_gc_destroy()
609 	xfrm_unset_type_offload(x);  in xfrm_state_gc_destroy()
610 	if (x->type) {  in xfrm_state_gc_destroy()
611 		x->type->destructor(x);  in xfrm_state_gc_destroy()
612 		xfrm_put_type(x->type);  in xfrm_state_gc_destroy()
614 	if (x->xfrag.page)  in xfrm_state_gc_destroy()
615 		put_page(x->xfrag.page);  in xfrm_state_gc_destroy()
616 	xfrm_dev_state_free(x);  in xfrm_state_gc_destroy()
617 	security_xfrm_state_free(x);  in xfrm_state_gc_destroy()
618 	xfrm_state_free(x);  in xfrm_state_gc_destroy()
623 	struct xfrm_state *x;  in xfrm_state_gc_task()  local
633 	hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)  in xfrm_state_gc_task()
634 		xfrm_state_gc_destroy(x);  in xfrm_state_gc_task()
639 	struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);  in xfrm_timer_handler()  local
646 	spin_lock(&x->lock);  in xfrm_timer_handler()
647 	xfrm_dev_state_update_stats(x);  in xfrm_timer_handler()
649 	if (x->km.state == XFRM_STATE_DEAD)  in xfrm_timer_handler()
651 	if (x->km.state == XFRM_STATE_EXPIRED)  in xfrm_timer_handler()
653 	if (x->lft.hard_add_expires_seconds) {  in xfrm_timer_handler()
654 		time64_t tmo = x->lft.hard_add_expires_seconds +  in xfrm_timer_handler()
655 			x->curlft.add_time - now;  in xfrm_timer_handler()
657 			if (x->xflags & XFRM_SOFT_EXPIRE) {  in xfrm_timer_handler()
660 				 * workaround: fix x->curflt.add_time by below:  in xfrm_timer_handler()
662 				x->curlft.add_time = now - x->saved_tmo - 1;  in xfrm_timer_handler()
663 				tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;  in xfrm_timer_handler()
670 	if (x->lft.hard_use_expires_seconds) {  in xfrm_timer_handler()
671 		time64_t tmo = x->lft.hard_use_expires_seconds +  in xfrm_timer_handler()
672 			(READ_ONCE(x->curlft.use_time) ? : now) - now;  in xfrm_timer_handler()
678 	if (x->km.dying)  in xfrm_timer_handler()
680 	if (x->lft.soft_add_expires_seconds) {  in xfrm_timer_handler()
681 		time64_t tmo = x->lft.soft_add_expires_seconds +  in xfrm_timer_handler()
682 			x->curlft.add_time - now;  in xfrm_timer_handler()
685 			x->xflags &= ~XFRM_SOFT_EXPIRE;  in xfrm_timer_handler()
688 			x->xflags |= XFRM_SOFT_EXPIRE;  in xfrm_timer_handler()
689 			x->saved_tmo = tmo;  in xfrm_timer_handler()
692 	if (x->lft.soft_use_expires_seconds) {  in xfrm_timer_handler()
693 		time64_t tmo = x->lft.soft_use_expires_seconds +  in xfrm_timer_handler()
694 			(READ_ONCE(x->curlft.use_time) ? : now) - now;  in xfrm_timer_handler()
701 	x->km.dying = warn;  in xfrm_timer_handler()
703 		km_state_expired(x, 0, 0);  in xfrm_timer_handler()
706 		hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));  in xfrm_timer_handler()
713 	if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)  in xfrm_timer_handler()
714 		x->km.state = XFRM_STATE_EXPIRED;  in xfrm_timer_handler()
716 	err = __xfrm_state_delete(x);  in xfrm_timer_handler()
718 		km_state_expired(x, 1, 0);  in xfrm_timer_handler()
720 	xfrm_audit_state_delete(x, err ? 0 : 1, true);  in xfrm_timer_handler()
723 	spin_unlock(&x->lock);  in xfrm_timer_handler()
731 	struct xfrm_state *x;  in xfrm_state_alloc()  local
733 	x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);  in xfrm_state_alloc()
735 	if (x) {  in xfrm_state_alloc()
736 		write_pnet(&x->xs_net, net);  in xfrm_state_alloc()
737 		refcount_set(&x->refcnt, 1);  in xfrm_state_alloc()
738 		atomic_set(&x->tunnel_users, 0);  in xfrm_state_alloc()
739 		INIT_LIST_HEAD(&x->km.all);  in xfrm_state_alloc()
740 		INIT_HLIST_NODE(&x->state_cache);  in xfrm_state_alloc()
741 		INIT_HLIST_NODE(&x->bydst);  in xfrm_state_alloc()
742 		INIT_HLIST_NODE(&x->bysrc);  in xfrm_state_alloc()
743 		INIT_HLIST_NODE(&x->byspi);  in xfrm_state_alloc()
744 		INIT_HLIST_NODE(&x->byseq);  in xfrm_state_alloc()
745 		hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME,  in xfrm_state_alloc()
747 		timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);  in xfrm_state_alloc()
748 		x->curlft.add_time = ktime_get_real_seconds();  in xfrm_state_alloc()
749 		x->lft.soft_byte_limit = XFRM_INF;  in xfrm_state_alloc()
750 		x->lft.soft_packet_limit = XFRM_INF;  in xfrm_state_alloc()
751 		x->lft.hard_byte_limit = XFRM_INF;  in xfrm_state_alloc()
752 		x->lft.hard_packet_limit = XFRM_INF;  in xfrm_state_alloc()
753 		x->replay_maxage = 0;  in xfrm_state_alloc()
754 		x->replay_maxdiff = 0;  in xfrm_state_alloc()
755 		x->pcpu_num = UINT_MAX;  in xfrm_state_alloc()
756 		spin_lock_init(&x->lock);  in xfrm_state_alloc()
757 		x->mode_data = NULL;  in xfrm_state_alloc()
759 	return x;  in xfrm_state_alloc()
764 void xfrm_dev_state_delete(struct xfrm_state *x)  in xfrm_dev_state_delete()  argument
766 	struct xfrm_dev_offload *xso = &x->xso;  in xfrm_dev_state_delete()
767 	struct net_device *dev = READ_ONCE(xso->dev);  in xfrm_dev_state_delete()
770 		dev->xfrmdev_ops->xdo_dev_state_delete(dev, x);  in xfrm_dev_state_delete()
772 		hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);  in xfrm_dev_state_delete()
778 void xfrm_dev_state_free(struct xfrm_state *x)  in xfrm_dev_state_free()  argument
780 	struct xfrm_dev_offload *xso = &x->xso;  in xfrm_dev_state_free()
781 	struct net_device *dev = READ_ONCE(xso->dev);  in xfrm_dev_state_free()
783 	if (dev && dev->xfrmdev_ops) {  in xfrm_dev_state_free()
785 		if (!hlist_unhashed(&x->dev_gclist))  in xfrm_dev_state_free()
786 			hlist_del(&x->dev_gclist);  in xfrm_dev_state_free()
789 		if (dev->xfrmdev_ops->xdo_dev_state_free)  in xfrm_dev_state_free()
790 			dev->xfrmdev_ops->xdo_dev_state_free(dev, x);  in xfrm_dev_state_free()
791 		WRITE_ONCE(xso->dev, NULL);  in xfrm_dev_state_free()
792 		xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;  in xfrm_dev_state_free()
793 		netdev_put(dev, &xso->dev_tracker);  in xfrm_dev_state_free()
798 void __xfrm_state_destroy(struct xfrm_state *x)  in __xfrm_state_destroy()  argument
800 	WARN_ON(x->km.state != XFRM_STATE_DEAD);  in __xfrm_state_destroy()
803 	hlist_add_head(&x->gclist, &xfrm_state_gc_list);  in __xfrm_state_destroy()
809 static void xfrm_state_delete_tunnel(struct xfrm_state *x);
810 int __xfrm_state_delete(struct xfrm_state *x)  in __xfrm_state_delete()  argument
812 	struct net *net = xs_net(x);  in __xfrm_state_delete()
813 	int err = -ESRCH;  in __xfrm_state_delete()
815 	if (x->km.state != XFRM_STATE_DEAD) {  in __xfrm_state_delete()
816 		x->km.state = XFRM_STATE_DEAD;  in __xfrm_state_delete()
818 		spin_lock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
819 		list_del(&x->km.all);  in __xfrm_state_delete()
820 		hlist_del_rcu(&x->bydst);  in __xfrm_state_delete()
821 		hlist_del_rcu(&x->bysrc);  in __xfrm_state_delete()
822 		if (x->km.seq)  in __xfrm_state_delete()
823 			hlist_del_rcu(&x->byseq);  in __xfrm_state_delete()
824 		if (!hlist_unhashed(&x->state_cache))  in __xfrm_state_delete()
825 			hlist_del_rcu(&x->state_cache);  in __xfrm_state_delete()
826 		if (!hlist_unhashed(&x->state_cache_input))  in __xfrm_state_delete()
827 			hlist_del_rcu(&x->state_cache_input);  in __xfrm_state_delete()
829 		if (x->id.spi)  in __xfrm_state_delete()
830 			hlist_del_rcu(&x->byspi);  in __xfrm_state_delete()
831 		net->xfrm.state_num--;  in __xfrm_state_delete()
832 		xfrm_nat_keepalive_state_updated(x);  in __xfrm_state_delete()
833 		spin_unlock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
835 		xfrm_dev_state_delete(x);  in __xfrm_state_delete()
837 		xfrm_state_delete_tunnel(x);  in __xfrm_state_delete()
843 		xfrm_state_put(x);  in __xfrm_state_delete()
851 int xfrm_state_delete(struct xfrm_state *x)  in xfrm_state_delete()  argument
855 	spin_lock_bh(&x->lock);  in xfrm_state_delete()
856 	err = __xfrm_state_delete(x);  in xfrm_state_delete()
857 	spin_unlock_bh(&x->lock);  in xfrm_state_delete()
869 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush_secctx_check()
870 		struct xfrm_state *x;  in xfrm_state_flush_secctx_check()  local
872 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush_secctx_check()
873 			if (xfrm_id_proto_match(x->id.proto, proto) &&  in xfrm_state_flush_secctx_check()
874 			   (err = security_xfrm_state_delete(x)) != 0) {  in xfrm_state_flush_secctx_check()
875 				xfrm_audit_state_delete(x, 0, task_valid);  in xfrm_state_flush_secctx_check()
889 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush_secctx_check()
890 		struct xfrm_state *x;  in xfrm_dev_state_flush_secctx_check()  local
893 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush_secctx_check()
894 			xso = &x->xso;  in xfrm_dev_state_flush_secctx_check()
896 			if (xso->dev == dev &&  in xfrm_dev_state_flush_secctx_check()
897 			   (err = security_xfrm_state_delete(x)) != 0) {  in xfrm_dev_state_flush_secctx_check()
898 				xfrm_audit_state_delete(x, 0, task_valid);  in xfrm_dev_state_flush_secctx_check()
924 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
929 	err = -ESRCH;  in xfrm_state_flush()
930 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush()
931 		struct xfrm_state *x;  in xfrm_state_flush()  local
933 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush()
934 			if (!xfrm_state_kern(x) &&  in xfrm_state_flush()
935 			    xfrm_id_proto_match(x->id.proto, proto)) {  in xfrm_state_flush()
936 				xfrm_state_hold(x);  in xfrm_state_flush()
937 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
939 				err = xfrm_state_delete(x);  in xfrm_state_flush()
940 				xfrm_audit_state_delete(x, err ? 0 : 1,  in xfrm_state_flush()
942 				xfrm_state_put(x);  in xfrm_state_flush()
946 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
952 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
962 	struct xfrm_state *x;  in xfrm_dev_state_flush()  local
967 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
972 	err = -ESRCH;  in xfrm_dev_state_flush()
973 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush()
975 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush()
976 			xso = &x->xso;  in xfrm_dev_state_flush()
978 			if (!xfrm_state_kern(x) && xso->dev == dev) {  in xfrm_dev_state_flush()
979 				xfrm_state_hold(x);  in xfrm_dev_state_flush()
980 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
982 				err = xfrm_state_delete(x);  in xfrm_dev_state_flush()
983 				xfrm_dev_state_free(x);  in xfrm_dev_state_flush()
985 				xfrm_audit_state_delete(x, err ? 0 : 1,  in xfrm_dev_state_flush()
987 				xfrm_state_put(x);  in xfrm_dev_state_flush()
991 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
1000 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
1004 	hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) {  in xfrm_dev_state_flush()
1005 		xso = &x->xso;  in xfrm_dev_state_flush()
1007 		if (xso->dev == dev) {  in xfrm_dev_state_flush()
1009 			xfrm_dev_state_free(x);  in xfrm_dev_state_flush()
1025 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
1026 	si->sadcnt = net->xfrm.state_num;  in xfrm_sad_getinfo()
1027 	si->sadhcnt = net->xfrm.state_hmask + 1;  in xfrm_sad_getinfo()
1028 	si->sadhmcnt = xfrm_state_hashmax;  in xfrm_sad_getinfo()
1029 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
1036 	const struct flowi4 *fl4 = &fl->u.ip4;  in __xfrm4_init_tempsel()
1038 	sel->daddr.a4 = fl4->daddr;  in __xfrm4_init_tempsel()
1039 	sel->saddr.a4 = fl4->saddr;  in __xfrm4_init_tempsel()
1040 	sel->dport = xfrm_flowi_dport(fl, &fl4->uli);  in __xfrm4_init_tempsel()
1041 	sel->dport_mask = htons(0xffff);  in __xfrm4_init_tempsel()
1042 	sel->sport = xfrm_flowi_sport(fl, &fl4->uli);  in __xfrm4_init_tempsel()
1043 	sel->sport_mask = htons(0xffff);  in __xfrm4_init_tempsel()
1044 	sel->family = AF_INET;  in __xfrm4_init_tempsel()
1045 	sel->prefixlen_d = 32;  in __xfrm4_init_tempsel()
1046 	sel->prefixlen_s = 32;  in __xfrm4_init_tempsel()
1047 	sel->proto = fl4->flowi4_proto;  in __xfrm4_init_tempsel()
1048 	sel->ifindex = fl4->flowi4_oif;  in __xfrm4_init_tempsel()
1054 	const struct flowi6 *fl6 = &fl->u.ip6;  in __xfrm6_init_tempsel()
1057 	*(struct in6_addr *)&sel->daddr = fl6->daddr;  in __xfrm6_init_tempsel()
1058 	*(struct in6_addr *)&sel->saddr = fl6->saddr;  in __xfrm6_init_tempsel()
1059 	sel->dport = xfrm_flowi_dport(fl, &fl6->uli);  in __xfrm6_init_tempsel()
1060 	sel->dport_mask = htons(0xffff);  in __xfrm6_init_tempsel()
1061 	sel->sport = xfrm_flowi_sport(fl, &fl6->uli);  in __xfrm6_init_tempsel()
1062 	sel->sport_mask = htons(0xffff);  in __xfrm6_init_tempsel()
1063 	sel->family = AF_INET6;  in __xfrm6_init_tempsel()
1064 	sel->prefixlen_d = 128;  in __xfrm6_init_tempsel()
1065 	sel->prefixlen_s = 128;  in __xfrm6_init_tempsel()
1066 	sel->proto = fl6->flowi6_proto;  in __xfrm6_init_tempsel()
1067 	sel->ifindex = fl6->flowi6_oif;  in __xfrm6_init_tempsel()
1071 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,  in xfrm_init_tempstate()  argument
1078 		__xfrm4_init_tempsel(&x->sel, fl);  in xfrm_init_tempstate()
1081 		__xfrm6_init_tempsel(&x->sel, fl);  in xfrm_init_tempstate()
1085 	x->id = tmpl->id;  in xfrm_init_tempstate()
1087 	switch (tmpl->encap_family) {  in xfrm_init_tempstate()
1089 		if (x->id.daddr.a4 == 0)  in xfrm_init_tempstate()
1090 			x->id.daddr.a4 = daddr->a4;  in xfrm_init_tempstate()
1091 		x->props.saddr = tmpl->saddr;  in xfrm_init_tempstate()
1092 		if (x->props.saddr.a4 == 0)  in xfrm_init_tempstate()
1093 			x->props.saddr.a4 = saddr->a4;  in xfrm_init_tempstate()
1096 		if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))  in xfrm_init_tempstate()
1097 			memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));  in xfrm_init_tempstate()
1098 		memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));  in xfrm_init_tempstate()
1099 		if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))  in xfrm_init_tempstate()
1100 			memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));  in xfrm_init_tempstate()
1104 	x->props.mode = tmpl->mode;  in xfrm_init_tempstate()
1105 	x->props.reqid = tmpl->reqid;  in xfrm_init_tempstate()
1106 	x->props.family = tmpl->encap_family;  in xfrm_init_tempstate()
1121 		sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_ptrs_get()
1123 		ptrs->bydst = xfrm_state_deref_check(net->xfrm.state_bydst, net);  in xfrm_hash_ptrs_get()
1124 		ptrs->bysrc = xfrm_state_deref_check(net->xfrm.state_bysrc, net);  in xfrm_hash_ptrs_get()
1125 		ptrs->byspi = xfrm_state_deref_check(net->xfrm.state_byspi, net);  in xfrm_hash_ptrs_get()
1126 		ptrs->hmask = net->xfrm.state_hmask;  in xfrm_hash_ptrs_get()
1127 	} while (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence));  in xfrm_hash_ptrs_get()
1137 	unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);  in __xfrm_state_lookup_all()
1138 	struct xfrm_state *x;  in __xfrm_state_lookup_all()  local
1140 	hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {  in __xfrm_state_lookup_all()
1142 		if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {  in __xfrm_state_lookup_all()
1143 			if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)  in __xfrm_state_lookup_all()
1152 			if (xdo->dev != x->xso.dev)  in __xfrm_state_lookup_all()
1154 		} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in __xfrm_state_lookup_all()
1158 		if (x->props.family != family ||  in __xfrm_state_lookup_all()
1159 		    x->id.spi       != spi ||  in __xfrm_state_lookup_all()
1160 		    x->id.proto     != proto ||  in __xfrm_state_lookup_all()
1161 		    !xfrm_addr_equal(&x->id.daddr, daddr, family))  in __xfrm_state_lookup_all()
1164 		if ((mark & x->mark.m) != x->mark.v)  in __xfrm_state_lookup_all()
1166 		if (!xfrm_state_hold_rcu(x))  in __xfrm_state_lookup_all()
1168 		return x;  in __xfrm_state_lookup_all()
1180 	unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);  in __xfrm_state_lookup()
1181 	struct xfrm_state *x;  in __xfrm_state_lookup()  local
1183 	hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {  in __xfrm_state_lookup()
1184 		if (x->props.family != family ||  in __xfrm_state_lookup()
1185 		    x->id.spi       != spi ||  in __xfrm_state_lookup()
1186 		    x->id.proto     != proto ||  in __xfrm_state_lookup()
1187 		    !xfrm_addr_equal(&x->id.daddr, daddr, family))  in __xfrm_state_lookup()
1190 		if ((mark & x->mark.m) != x->mark.v)  in __xfrm_state_lookup()
1192 		if (!xfrm_state_hold_rcu(x))  in __xfrm_state_lookup()
1194 		return x;  in __xfrm_state_lookup()
1207 	struct xfrm_state *x = NULL;  in xfrm_input_state_lookup()  local
1209 	state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input);  in xfrm_input_state_lookup()
1212 	hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) {  in xfrm_input_state_lookup()
1213 		if (x->props.family != family ||  in xfrm_input_state_lookup()
1214 		    x->id.spi       != spi ||  in xfrm_input_state_lookup()
1215 		    x->id.proto     != proto ||  in xfrm_input_state_lookup()
1216 		    !xfrm_addr_equal(&x->id.daddr, daddr, family))  in xfrm_input_state_lookup()
1219 		if ((mark & x->mark.m) != x->mark.v)  in xfrm_input_state_lookup()
1221 		if (!xfrm_state_hold_rcu(x))  in xfrm_input_state_lookup()
1228 	x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);  in xfrm_input_state_lookup()
1230 	if (x && x->km.state == XFRM_STATE_VALID) {  in xfrm_input_state_lookup()
1231 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_input_state_lookup()
1232 		if (hlist_unhashed(&x->state_cache_input)) {  in xfrm_input_state_lookup()
1233 			hlist_add_head_rcu(&x->state_cache_input, state_cache_input);  in xfrm_input_state_lookup()
1235 			hlist_del_rcu(&x->state_cache_input);  in xfrm_input_state_lookup()
1236 			hlist_add_head_rcu(&x->state_cache_input, state_cache_input);  in xfrm_input_state_lookup()
1238 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_input_state_lookup()
1243 	return x;  in xfrm_input_state_lookup()
1253 	unsigned int h = __xfrm_src_hash(daddr, saddr, family, state_ptrs->hmask);  in __xfrm_state_lookup_byaddr()
1254 	struct xfrm_state *x;  in __xfrm_state_lookup_byaddr()  local
1256 	hlist_for_each_entry_rcu(x, state_ptrs->bysrc + h, bysrc) {  in __xfrm_state_lookup_byaddr()
1257 		if (x->props.family != family ||  in __xfrm_state_lookup_byaddr()
1258 		    x->id.proto     != proto ||  in __xfrm_state_lookup_byaddr()
1259 		    !xfrm_addr_equal(&x->id.daddr, daddr, family) ||  in __xfrm_state_lookup_byaddr()
1260 		    !xfrm_addr_equal(&x->props.saddr, saddr, family))  in __xfrm_state_lookup_byaddr()
1263 		if ((mark & x->mark.m) != x->mark.v)  in __xfrm_state_lookup_byaddr()
1265 		if (!xfrm_state_hold_rcu(x))  in __xfrm_state_lookup_byaddr()
1267 		return x;  in __xfrm_state_lookup_byaddr()
1274 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)  in __xfrm_state_locate()  argument
1277 	struct net *net = xs_net(x);  in __xfrm_state_locate()
1278 	u32 mark = x->mark.v & x->mark.m;  in __xfrm_state_locate()
1283 		return __xfrm_state_lookup(&state_ptrs, mark, &x->id.daddr,  in __xfrm_state_locate()
1284 					   x->id.spi, x->id.proto, family);  in __xfrm_state_locate()
1287 						  &x->id.daddr,  in __xfrm_state_locate()
1288 						  &x->props.saddr,  in __xfrm_state_locate()
1289 						  x->id.proto, family);  in __xfrm_state_locate()
1295 	    (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&  in xfrm_hash_grow_check()
1296 	    net->xfrm.state_num > net->xfrm.state_hmask)  in xfrm_hash_grow_check()
1297 		schedule_work(&net->xfrm.state_hash_work);  in xfrm_hash_grow_check()
1300 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,  in xfrm_state_look_at()  argument
1316 	if (x->km.state == XFRM_STATE_VALID) {  in xfrm_state_look_at()
1317 		if ((x->sel.family &&  in xfrm_state_look_at()
1318 		     (x->sel.family != family ||  in xfrm_state_look_at()
1319 		      !xfrm_selector_match(&x->sel, fl, family))) ||  in xfrm_state_look_at()
1320 		    !security_xfrm_state_pol_flow_match(x, pol,  in xfrm_state_look_at()
1321 							&fl->u.__fl_common))  in xfrm_state_look_at()
1324 		if (x->pcpu_num != UINT_MAX && x->pcpu_num != pcpu_id)  in xfrm_state_look_at()
1328 		    ((*best)->pcpu_num == UINT_MAX && x->pcpu_num == pcpu_id) ||  in xfrm_state_look_at()
1329 		    (*best)->km.dying > x->km.dying ||  in xfrm_state_look_at()
1330 		    ((*best)->km.dying == x->km.dying &&  in xfrm_state_look_at()
1331 		     (*best)->curlft.add_time < x->curlft.add_time))  in xfrm_state_look_at()
1332 			*best = x;  in xfrm_state_look_at()
1333 	} else if (x->km.state == XFRM_STATE_ACQ) {  in xfrm_state_look_at()
1334 		if (!*best || x->pcpu_num == pcpu_id)  in xfrm_state_look_at()
1336 	} else if (x->km.state == XFRM_STATE_ERROR ||  in xfrm_state_look_at()
1337 		   x->km.state == XFRM_STATE_EXPIRED) {  in xfrm_state_look_at()
1338 		if ((!x->sel.family ||  in xfrm_state_look_at()
1339 		     (x->sel.family == family &&  in xfrm_state_look_at()
1340 		      xfrm_selector_match(&x->sel, fl, family))) &&  in xfrm_state_look_at()
1341 		    security_xfrm_state_pol_flow_match(x, pol,  in xfrm_state_look_at()
1342 						       &fl->u.__fl_common))  in xfrm_state_look_at()
1343 			*error = -ESRCH;  in xfrm_state_look_at()
1357 	struct xfrm_state *x, *x0, *to_put;  in xfrm_state_find()  local
1361 	u32 mark = pol->mark.v & pol->mark.m;  in xfrm_state_find()
1362 	unsigned short encap_family = tmpl->encap_family;  in xfrm_state_find()
1375 	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_state_find()
1380 	hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {  in xfrm_state_find()
1381 		if (x->props.family == encap_family &&  in xfrm_state_find()
1382 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1383 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1384 		    x->if_id == if_id &&  in xfrm_state_find()
1385 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1386 		    xfrm_state_addr_check(x, daddr, saddr, encap_family) &&  in xfrm_state_find()
1387 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1388 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1389 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1390 			xfrm_state_look_at(pol, x, fl, encap_family,  in xfrm_state_find()
1397 	hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {  in xfrm_state_find()
1398 		if (x->props.family == encap_family &&  in xfrm_state_find()
1399 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1400 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1401 		    x->if_id == if_id &&  in xfrm_state_find()
1402 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1403 		    xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&  in xfrm_state_find()
1404 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1405 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1406 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1407 			xfrm_state_look_at(pol, x, fl, family,  in xfrm_state_find()
1420 	h = __xfrm_dst_hash(daddr, saddr, tmpl->reqid, encap_family, state_ptrs.hmask);  in xfrm_state_find()
1421 	hlist_for_each_entry_rcu(x, state_ptrs.bydst + h, bydst) {  in xfrm_state_find()
1423 		if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1424 			if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1433 			if (pol->xdo.dev != x->xso.dev)  in xfrm_state_find()
1435 		} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1439 		if (x->props.family == encap_family &&  in xfrm_state_find()
1440 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1441 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1442 		    x->if_id == if_id &&  in xfrm_state_find()
1443 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1444 		    xfrm_state_addr_check(x, daddr, saddr, encap_family) &&  in xfrm_state_find()
1445 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1446 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1447 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1448 			xfrm_state_look_at(pol, x, fl, family,  in xfrm_state_find()
1454 	h_wildcard = __xfrm_dst_hash(daddr, &saddr_wildcard, tmpl->reqid,  in xfrm_state_find()
1456 	hlist_for_each_entry_rcu(x, state_ptrs.bydst + h_wildcard, bydst) {  in xfrm_state_find()
1458 		if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1459 			if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1468 			if (pol->xdo.dev != x->xso.dev)  in xfrm_state_find()
1470 		} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_find()
1474 		if (x->props.family == encap_family &&  in xfrm_state_find()
1475 		    x->props.reqid == tmpl->reqid &&  in xfrm_state_find()
1476 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_state_find()
1477 		    x->if_id == if_id &&  in xfrm_state_find()
1478 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_state_find()
1479 		    xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&  in xfrm_state_find()
1480 		    tmpl->mode == x->props.mode &&  in xfrm_state_find()
1481 		    tmpl->id.proto == x->id.proto &&  in xfrm_state_find()
1482 		    (tmpl->id.spi == x->id.spi || !tmpl->id.spi))  in xfrm_state_find()
1483 			xfrm_state_look_at(pol, x, fl, family,  in xfrm_state_find()
1488 	if (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) ||  in xfrm_state_find()
1489 	    (best && (best->pcpu_num == pcpu_id)))  in xfrm_state_find()
1490 		x = best;  in xfrm_state_find()
1492 	if (!x && !error && !acquire_in_progress) {  in xfrm_state_find()
1493 		if (tmpl->id.spi &&  in xfrm_state_find()
1495 						  tmpl->id.spi, tmpl->id.proto,  in xfrm_state_find()
1497 						  &pol->xdo)) != NULL) {  in xfrm_state_find()
1499 			error = -EEXIST;  in xfrm_state_find()
1505 		 * for each and every packet - garbage collection might not  in xfrm_state_find()
1509 			error = -ESRCH;  in xfrm_state_find()
1513 		x = xfrm_state_alloc(net);  in xfrm_state_find()
1514 		if (x == NULL) {  in xfrm_state_find()
1515 			error = -ENOMEM;  in xfrm_state_find()
1520 		xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);  in xfrm_state_find()
1521 		memcpy(&x->mark, &pol->mark, sizeof(x->mark));  in xfrm_state_find()
1522 		x->if_id = if_id;  in xfrm_state_find()
1523 		if ((pol->flags & XFRM_POLICY_CPU_ACQUIRE) && best)  in xfrm_state_find()
1524 			x->pcpu_num = pcpu_id;  in xfrm_state_find()
1526 		error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);  in xfrm_state_find()
1528 			x->km.state = XFRM_STATE_DEAD;  in xfrm_state_find()
1529 			to_put = x;  in xfrm_state_find()
1530 			x = NULL;  in xfrm_state_find()
1534 		if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1535 			struct xfrm_dev_offload *xdo = &pol->xdo;  in xfrm_state_find()
1536 			struct xfrm_dev_offload *xso = &x->xso;  in xfrm_state_find()
1537 			struct net_device *dev = xdo->dev;  in xfrm_state_find()
1539 			xso->type = XFRM_DEV_OFFLOAD_PACKET;  in xfrm_state_find()
1540 			xso->dir = xdo->dir;  in xfrm_state_find()
1541 			xso->dev = dev;  in xfrm_state_find()
1542 			xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;  in xfrm_state_find()
1543 			netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC);  in xfrm_state_find()
1544 			error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x,  in xfrm_state_find()
1547 				xso->dir = 0;  in xfrm_state_find()
1548 				netdev_put(dev, &xso->dev_tracker);  in xfrm_state_find()
1549 				xso->dev = NULL;  in xfrm_state_find()
1550 				xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;  in xfrm_state_find()
1551 				x->km.state = XFRM_STATE_DEAD;  in xfrm_state_find()
1552 				to_put = x;  in xfrm_state_find()
1553 				x = NULL;  in xfrm_state_find()
1558 		if (km_query(x, tmpl, pol) == 0) {  in xfrm_state_find()
1559 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1560 			x->km.state = XFRM_STATE_ACQ;  in xfrm_state_find()
1561 			x->dir = XFRM_SA_DIR_OUT;  in xfrm_state_find()
1562 			list_add(&x->km.all, &net->xfrm.state_all);  in xfrm_state_find()
1563 			h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);  in xfrm_state_find()
1564 			XFRM_STATE_INSERT(bydst, &x->bydst,  in xfrm_state_find()
1565 					  net->xfrm.state_bydst + h,  in xfrm_state_find()
1566 					  x->xso.type);  in xfrm_state_find()
1568 			XFRM_STATE_INSERT(bysrc, &x->bysrc,  in xfrm_state_find()
1569 					  net->xfrm.state_bysrc + h,  in xfrm_state_find()
1570 					  x->xso.type);  in xfrm_state_find()
1571 			INIT_HLIST_NODE(&x->state_cache);  in xfrm_state_find()
1572 			if (x->id.spi) {  in xfrm_state_find()
1573 				h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);  in xfrm_state_find()
1574 				XFRM_STATE_INSERT(byspi, &x->byspi,  in xfrm_state_find()
1575 						  net->xfrm.state_byspi + h,  in xfrm_state_find()
1576 						  x->xso.type);  in xfrm_state_find()
1578 			if (x->km.seq) {  in xfrm_state_find()
1579 				h = xfrm_seq_hash(net, x->km.seq);  in xfrm_state_find()
1580 				XFRM_STATE_INSERT(byseq, &x->byseq,  in xfrm_state_find()
1581 						  net->xfrm.state_byseq + h,  in xfrm_state_find()
1582 						  x->xso.type);  in xfrm_state_find()
1584 			x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in xfrm_state_find()
1585 			hrtimer_start(&x->mtimer,  in xfrm_state_find()
1586 				      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in xfrm_state_find()
1588 			net->xfrm.state_num++;  in xfrm_state_find()
1589 			xfrm_hash_grow_check(net, x->bydst.next != NULL);  in xfrm_state_find()
1590 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1593 			struct xfrm_dev_offload *xso = &x->xso;  in xfrm_state_find()
1595 			if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {  in xfrm_state_find()
1596 				xfrm_dev_state_delete(x);  in xfrm_state_find()
1597 				xfrm_dev_state_free(x);  in xfrm_state_find()
1600 			x->km.state = XFRM_STATE_DEAD;  in xfrm_state_find()
1601 			to_put = x;  in xfrm_state_find()
1602 			x = NULL;  in xfrm_state_find()
1603 			error = -ESRCH;  in xfrm_state_find()
1606 		/* Use the already installed 'fallback' while the CPU-specific  in xfrm_state_find()
1609 			x = best;  in xfrm_state_find()
1612 	if (x) {  in xfrm_state_find()
1613 		if (!xfrm_state_hold_rcu(x)) {  in xfrm_state_find()
1614 			*err = -EAGAIN;  in xfrm_state_find()
1615 			x = NULL;  in xfrm_state_find()
1618 		*err = acquire_in_progress ? -EAGAIN : error;  in xfrm_state_find()
1621 	if (x && x->km.state == XFRM_STATE_VALID && !cached &&  in xfrm_state_find()
1622 	    (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) || x->pcpu_num == pcpu_id)) {  in xfrm_state_find()
1623 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1624 		if (hlist_unhashed(&x->state_cache))  in xfrm_state_find()
1625 			hlist_add_head_rcu(&x->state_cache, &pol->state_cache_list);  in xfrm_state_find()
1626 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1633 	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {  in xfrm_state_find()
1634 		*err = -EAGAIN;  in xfrm_state_find()
1635 		if (x) {  in xfrm_state_find()
1636 			xfrm_state_put(x);  in xfrm_state_find()
1637 			x = NULL;  in xfrm_state_find()
1641 	return x;  in xfrm_state_find()
1650 	struct xfrm_state *rx = NULL, *x = NULL;  in xfrm_stateonly_find()  local
1652 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1654 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_stateonly_find()
1655 		if (x->props.family == family &&  in xfrm_stateonly_find()
1656 		    x->props.reqid == reqid &&  in xfrm_stateonly_find()
1657 		    (mark & x->mark.m) == x->mark.v &&  in xfrm_stateonly_find()
1658 		    x->if_id == if_id &&  in xfrm_stateonly_find()
1659 		    !(x->props.flags & XFRM_STATE_WILDRECV) &&  in xfrm_stateonly_find()
1660 		    xfrm_state_addr_check(x, daddr, saddr, family) &&  in xfrm_stateonly_find()
1661 		    mode == x->props.mode &&  in xfrm_stateonly_find()
1662 		    proto == x->id.proto &&  in xfrm_stateonly_find()
1663 		    x->km.state == XFRM_STATE_VALID) {  in xfrm_stateonly_find()
1664 			rx = x;  in xfrm_stateonly_find()
1671 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1681 	struct xfrm_state *x;  in xfrm_state_lookup_byspi()  local
1684 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1685 	list_for_each_entry(w, &net->xfrm.state_all, all) {  in xfrm_state_lookup_byspi()
1686 		x = container_of(w, struct xfrm_state, km);  in xfrm_state_lookup_byspi()
1687 		if (x->props.family != family ||  in xfrm_state_lookup_byspi()
1688 			x->id.spi != spi)  in xfrm_state_lookup_byspi()
1691 		xfrm_state_hold(x);  in xfrm_state_lookup_byspi()
1692 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1693 		return x;  in xfrm_state_lookup_byspi()
1695 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1702 	struct xfrm_state *x;  in xfrm_state_lookup_spi_proto()  local
1706 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_lookup_spi_proto()
1707 		hlist_for_each_entry_rcu(x, &net->xfrm.state_byspi[i], byspi) {  in xfrm_state_lookup_spi_proto()
1708 			if (x->id.spi == spi && x->id.proto == proto) {  in xfrm_state_lookup_spi_proto()
1709 				if (!xfrm_state_hold_rcu(x))  in xfrm_state_lookup_spi_proto()
1712 				return x;  in xfrm_state_lookup_spi_proto()
1720 static void __xfrm_state_insert(struct xfrm_state *x)  in __xfrm_state_insert()  argument
1722 	struct net *net = xs_net(x);  in __xfrm_state_insert()
1725 	list_add(&x->km.all, &net->xfrm.state_all);  in __xfrm_state_insert()
1728 	x->mark.v &= x->mark.m;  in __xfrm_state_insert()
1730 	h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,  in __xfrm_state_insert()
1731 			  x->props.reqid, x->props.family);  in __xfrm_state_insert()
1732 	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,  in __xfrm_state_insert()
1733 			  x->xso.type);  in __xfrm_state_insert()
1735 	h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);  in __xfrm_state_insert()
1736 	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,  in __xfrm_state_insert()
1737 			  x->xso.type);  in __xfrm_state_insert()
1739 	if (x->id.spi) {  in __xfrm_state_insert()
1740 		h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,  in __xfrm_state_insert()
1741 				  x->props.family);  in __xfrm_state_insert()
1743 		XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,  in __xfrm_state_insert()
1744 				  x->xso.type);  in __xfrm_state_insert()
1747 	if (x->km.seq) {  in __xfrm_state_insert()
1748 		h = xfrm_seq_hash(net, x->km.seq);  in __xfrm_state_insert()
1750 		XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,  in __xfrm_state_insert()
1751 				  x->xso.type);  in __xfrm_state_insert()
1754 	hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);  in __xfrm_state_insert()
1755 	if (x->replay_maxage)  in __xfrm_state_insert()
1756 		mod_timer(&x->rtimer, jiffies + x->replay_maxage);  in __xfrm_state_insert()
1758 	net->xfrm.state_num++;  in __xfrm_state_insert()
1760 	xfrm_hash_grow_check(net, x->bydst.next != NULL);  in __xfrm_state_insert()
1761 	xfrm_nat_keepalive_state_updated(x);  in __xfrm_state_insert()
1764 /* net->xfrm.xfrm_state_lock is held */
1768 	unsigned short family = xnew->props.family;  in __xfrm_state_bump_genids()
1769 	u32 reqid = xnew->props.reqid;  in __xfrm_state_bump_genids()
1770 	struct xfrm_state *x;  in __xfrm_state_bump_genids()  local
1772 	u32 mark = xnew->mark.v & xnew->mark.m;  in __xfrm_state_bump_genids()
1773 	u32 if_id = xnew->if_id;  in __xfrm_state_bump_genids()
1774 	u32 cpu_id = xnew->pcpu_num;  in __xfrm_state_bump_genids()
1776 	h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);  in __xfrm_state_bump_genids()
1777 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __xfrm_state_bump_genids()
1778 		if (x->props.family	== family &&  in __xfrm_state_bump_genids()
1779 		    x->props.reqid	== reqid &&  in __xfrm_state_bump_genids()
1780 		    x->if_id		== if_id &&  in __xfrm_state_bump_genids()
1781 		    x->pcpu_num		== cpu_id &&  in __xfrm_state_bump_genids()
1782 		    (mark & x->mark.m) == x->mark.v &&  in __xfrm_state_bump_genids()
1783 		    xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&  in __xfrm_state_bump_genids()
1784 		    xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))  in __xfrm_state_bump_genids()
1785 			x->genid++;  in __xfrm_state_bump_genids()
1789 void xfrm_state_insert(struct xfrm_state *x)  in xfrm_state_insert()  argument
1791 	struct net *net = xs_net(x);  in xfrm_state_insert()
1793 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1794 	__xfrm_state_bump_genids(x);  in xfrm_state_insert()
1795 	__xfrm_state_insert(x);  in xfrm_state_insert()
1796 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1800 /* net->xfrm.xfrm_state_lock is held */
1810 	struct xfrm_state *x;  in __find_acq_core()  local
1811 	u32 mark = m->v & m->m;  in __find_acq_core()
1813 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __find_acq_core()
1814 		if (x->props.reqid  != reqid ||  in __find_acq_core()
1815 		    x->props.mode   != mode ||  in __find_acq_core()
1816 		    x->props.family != family ||  in __find_acq_core()
1817 		    x->km.state     != XFRM_STATE_ACQ ||  in __find_acq_core()
1818 		    x->id.spi       != 0 ||  in __find_acq_core()
1819 		    x->id.proto	    != proto ||  in __find_acq_core()
1820 		    (mark & x->mark.m) != x->mark.v ||  in __find_acq_core()
1821 		    x->pcpu_num != pcpu_num ||  in __find_acq_core()
1822 		    !xfrm_addr_equal(&x->id.daddr, daddr, family) ||  in __find_acq_core()
1823 		    !xfrm_addr_equal(&x->props.saddr, saddr, family))  in __find_acq_core()
1826 		xfrm_state_hold(x);  in __find_acq_core()
1827 		return x;  in __find_acq_core()
1833 	x = xfrm_state_alloc(net);  in __find_acq_core()
1834 	if (likely(x)) {  in __find_acq_core()
1837 			x->sel.daddr.a4 = daddr->a4;  in __find_acq_core()
1838 			x->sel.saddr.a4 = saddr->a4;  in __find_acq_core()
1839 			x->sel.prefixlen_d = 32;  in __find_acq_core()
1840 			x->sel.prefixlen_s = 32;  in __find_acq_core()
1841 			x->props.saddr.a4 = saddr->a4;  in __find_acq_core()
1842 			x->id.daddr.a4 = daddr->a4;  in __find_acq_core()
1846 			x->sel.daddr.in6 = daddr->in6;  in __find_acq_core()
1847 			x->sel.saddr.in6 = saddr->in6;  in __find_acq_core()
1848 			x->sel.prefixlen_d = 128;  in __find_acq_core()
1849 			x->sel.prefixlen_s = 128;  in __find_acq_core()
1850 			x->props.saddr.in6 = saddr->in6;  in __find_acq_core()
1851 			x->id.daddr.in6 = daddr->in6;  in __find_acq_core()
1855 		x->pcpu_num = pcpu_num;  in __find_acq_core()
1856 		x->km.state = XFRM_STATE_ACQ;  in __find_acq_core()
1857 		x->id.proto = proto;  in __find_acq_core()
1858 		x->props.family = family;  in __find_acq_core()
1859 		x->props.mode = mode;  in __find_acq_core()
1860 		x->props.reqid = reqid;  in __find_acq_core()
1861 		x->if_id = if_id;  in __find_acq_core()
1862 		x->mark.v = m->v;  in __find_acq_core()
1863 		x->mark.m = m->m;  in __find_acq_core()
1864 		x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in __find_acq_core()
1865 		xfrm_state_hold(x);  in __find_acq_core()
1866 		hrtimer_start(&x->mtimer,  in __find_acq_core()
1867 			      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in __find_acq_core()
1869 		list_add(&x->km.all, &net->xfrm.state_all);  in __find_acq_core()
1870 		XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,  in __find_acq_core()
1871 				  x->xso.type);  in __find_acq_core()
1873 		XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,  in __find_acq_core()
1874 				  x->xso.type);  in __find_acq_core()
1876 		net->xfrm.state_num++;  in __find_acq_core()
1878 		xfrm_hash_grow_check(net, x->bydst.next != NULL);  in __find_acq_core()
1881 	return x;  in __find_acq_core()
1886 int xfrm_state_add(struct xfrm_state *x)  in xfrm_state_add()  argument
1888 	struct net *net = xs_net(x);  in xfrm_state_add()
1892 	u32 mark = x->mark.v & x->mark.m;  in xfrm_state_add()
1893 	int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);  in xfrm_state_add()
1895 	family = x->props.family;  in xfrm_state_add()
1899 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1901 	x1 = __xfrm_state_locate(x, use_spi, family);  in xfrm_state_add()
1905 		err = -EEXIST;  in xfrm_state_add()
1909 	if (use_spi && x->km.seq) {  in xfrm_state_add()
1910 		x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq, x->pcpu_num);  in xfrm_state_add()
1911 		if (x1 && ((x1->id.proto != x->id.proto) ||  in xfrm_state_add()
1912 		    !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {  in xfrm_state_add()
1919 		x1 = __find_acq_core(net, &x->mark, family, x->props.mode,  in xfrm_state_add()
1920 				     x->props.reqid, x->if_id, x->pcpu_num, x->id.proto,  in xfrm_state_add()
1921 				     &x->id.daddr, &x->props.saddr, 0);  in xfrm_state_add()
1923 	__xfrm_state_bump_genids(x);  in xfrm_state_add()
1924 	__xfrm_state_insert(x);  in xfrm_state_add()
1928 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1943 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)  in clone_security()  argument
1946 	int size = sizeof(*uctx) + security->ctx_len;  in clone_security()
1951 		return -ENOMEM;  in clone_security()
1953 	uctx->exttype = XFRMA_SEC_CTX;  in clone_security()
1954 	uctx->len = size;  in clone_security()
1955 	uctx->ctx_doi = security->ctx_doi;  in clone_security()
1956 	uctx->ctx_alg = security->ctx_alg;  in clone_security()
1957 	uctx->ctx_len = security->ctx_len;  in clone_security()
1958 	memcpy(uctx + 1, security->ctx_str, security->ctx_len);  in clone_security()
1959 	err = security_xfrm_state_alloc(x, uctx);  in clone_security()
1972 	struct xfrm_state *x = xfrm_state_alloc(net);  in xfrm_state_clone_and_setup()  local
1973 	if (!x)  in xfrm_state_clone_and_setup()
1976 	memcpy(&x->id, &orig->id, sizeof(x->id));  in xfrm_state_clone_and_setup()
1977 	memcpy(&x->sel, &orig->sel, sizeof(x->sel));  in xfrm_state_clone_and_setup()
1978 	memcpy(&x->lft, &orig->lft, sizeof(x->lft));  in xfrm_state_clone_and_setup()
1979 	x->props.mode = orig->props.mode;  in xfrm_state_clone_and_setup()
1980 	x->props.replay_window = orig->props.replay_window;  in xfrm_state_clone_and_setup()
1981 	x->props.reqid = orig->props.reqid;  in xfrm_state_clone_and_setup()
1982 	x->props.family = orig->props.family;  in xfrm_state_clone_and_setup()
1983 	x->props.saddr = orig->props.saddr;  in xfrm_state_clone_and_setup()
1985 	if (orig->aalg) {  in xfrm_state_clone_and_setup()
1986 		x->aalg = xfrm_algo_auth_clone(orig->aalg);  in xfrm_state_clone_and_setup()
1987 		if (!x->aalg)  in xfrm_state_clone_and_setup()
1990 	x->props.aalgo = orig->props.aalgo;  in xfrm_state_clone_and_setup()
1992 	if (orig->aead) {  in xfrm_state_clone_and_setup()
1993 		x->aead = xfrm_algo_aead_clone(orig->aead);  in xfrm_state_clone_and_setup()
1994 		x->geniv = orig->geniv;  in xfrm_state_clone_and_setup()
1995 		if (!x->aead)  in xfrm_state_clone_and_setup()
1998 	if (orig->ealg) {  in xfrm_state_clone_and_setup()
1999 		x->ealg = xfrm_algo_clone(orig->ealg);  in xfrm_state_clone_and_setup()
2000 		if (!x->ealg)  in xfrm_state_clone_and_setup()
2003 	x->props.ealgo = orig->props.ealgo;  in xfrm_state_clone_and_setup()
2005 	if (orig->calg) {  in xfrm_state_clone_and_setup()
2006 		x->calg = xfrm_algo_clone(orig->calg);  in xfrm_state_clone_and_setup()
2007 		if (!x->calg)  in xfrm_state_clone_and_setup()
2010 	x->props.calgo = orig->props.calgo;  in xfrm_state_clone_and_setup()
2012 	if (encap || orig->encap) {  in xfrm_state_clone_and_setup()
2014 			x->encap = kmemdup(encap, sizeof(*x->encap),  in xfrm_state_clone_and_setup()
2017 			x->encap = kmemdup(orig->encap, sizeof(*x->encap),  in xfrm_state_clone_and_setup()
2020 		if (!x->encap)  in xfrm_state_clone_and_setup()
2024 	if (orig->security)  in xfrm_state_clone_and_setup()
2025 		if (clone_security(x, orig->security))  in xfrm_state_clone_and_setup()
2028 	if (orig->coaddr) {  in xfrm_state_clone_and_setup()
2029 		x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),  in xfrm_state_clone_and_setup()
2031 		if (!x->coaddr)  in xfrm_state_clone_and_setup()
2035 	if (orig->replay_esn) {  in xfrm_state_clone_and_setup()
2036 		if (xfrm_replay_clone(x, orig))  in xfrm_state_clone_and_setup()
2040 	memcpy(&x->mark, &orig->mark, sizeof(x->mark));  in xfrm_state_clone_and_setup()
2041 	memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));  in xfrm_state_clone_and_setup()
2043 	x->props.flags = orig->props.flags;  in xfrm_state_clone_and_setup()
2044 	x->props.extra_flags = orig->props.extra_flags;  in xfrm_state_clone_and_setup()
2046 	x->pcpu_num = orig->pcpu_num;  in xfrm_state_clone_and_setup()
2047 	x->if_id = orig->if_id;  in xfrm_state_clone_and_setup()
2048 	x->tfcpad = orig->tfcpad;  in xfrm_state_clone_and_setup()
2049 	x->replay_maxdiff = orig->replay_maxdiff;  in xfrm_state_clone_and_setup()
2050 	x->replay_maxage = orig->replay_maxage;  in xfrm_state_clone_and_setup()
2051 	memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));  in xfrm_state_clone_and_setup()
2052 	x->km.state = orig->km.state;  in xfrm_state_clone_and_setup()
2053 	x->km.seq = orig->km.seq;  in xfrm_state_clone_and_setup()
2054 	x->replay = orig->replay;  in xfrm_state_clone_and_setup()
2055 	x->preplay = orig->preplay;  in xfrm_state_clone_and_setup()
2056 	x->mapping_maxage = orig->mapping_maxage;  in xfrm_state_clone_and_setup()
2057 	x->lastused = orig->lastused;  in xfrm_state_clone_and_setup()
2058 	x->new_mapping = 0;  in xfrm_state_clone_and_setup()
2059 	x->new_mapping_sport = 0;  in xfrm_state_clone_and_setup()
2060 	x->dir = orig->dir;  in xfrm_state_clone_and_setup()
2062 	x->mode_cbs = orig->mode_cbs;  in xfrm_state_clone_and_setup()
2063 	if (x->mode_cbs && x->mode_cbs->clone_state) {  in xfrm_state_clone_and_setup()
2064 		if (x->mode_cbs->clone_state(x, orig))  in xfrm_state_clone_and_setup()
2069 	x->props.family = m->new_family;  in xfrm_state_clone_and_setup()
2070 	memcpy(&x->id.daddr, &m->new_daddr, sizeof(x->id.daddr));  in xfrm_state_clone_and_setup()
2071 	memcpy(&x->props.saddr, &m->new_saddr, sizeof(x->props.saddr));  in xfrm_state_clone_and_setup()
2073 	return x;  in xfrm_state_clone_and_setup()
2076 	xfrm_state_put(x);  in xfrm_state_clone_and_setup()
2085 	struct xfrm_state *x = NULL;  in xfrm_migrate_state_find()  local
2087 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
2089 	if (m->reqid) {  in xfrm_migrate_state_find()
2090 		h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,  in xfrm_migrate_state_find()
2091 				  m->reqid, m->old_family);  in xfrm_migrate_state_find()
2092 		hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_migrate_state_find()
2093 			if (x->props.mode != m->mode ||  in xfrm_migrate_state_find()
2094 			    x->id.proto != m->proto)  in xfrm_migrate_state_find()
2096 			if (m->reqid && x->props.reqid != m->reqid)  in xfrm_migrate_state_find()
2098 			if (if_id != 0 && x->if_id != if_id)  in xfrm_migrate_state_find()
2100 			if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,  in xfrm_migrate_state_find()
2101 					     m->old_family) ||  in xfrm_migrate_state_find()
2102 			    !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,  in xfrm_migrate_state_find()
2103 					     m->old_family))  in xfrm_migrate_state_find()
2105 			xfrm_state_hold(x);  in xfrm_migrate_state_find()
2109 		h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,  in xfrm_migrate_state_find()
2110 				  m->old_family);  in xfrm_migrate_state_find()
2111 		hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in xfrm_migrate_state_find()
2112 			if (x->props.mode != m->mode ||  in xfrm_migrate_state_find()
2113 			    x->id.proto != m->proto)  in xfrm_migrate_state_find()
2115 			if (if_id != 0 && x->if_id != if_id)  in xfrm_migrate_state_find()
2117 			if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,  in xfrm_migrate_state_find()
2118 					     m->old_family) ||  in xfrm_migrate_state_find()
2119 			    !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,  in xfrm_migrate_state_find()
2120 					     m->old_family))  in xfrm_migrate_state_find()
2122 			xfrm_state_hold(x);  in xfrm_migrate_state_find()
2127 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
2129 	return x;  in xfrm_migrate_state_find()
2133 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,  in xfrm_state_migrate()  argument
2142 	xc = xfrm_state_clone_and_setup(x, encap, m);  in xfrm_state_migrate()
2154 	if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {  in xfrm_state_migrate()
2171 int xfrm_state_update(struct xfrm_state *x)  in xfrm_state_update()  argument
2175 	int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);  in xfrm_state_update()
2176 	struct net *net = xs_net(x);  in xfrm_state_update()
2180 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2181 	x1 = __xfrm_state_locate(x, use_spi, x->props.family);  in xfrm_state_update()
2183 	err = -ESRCH;  in xfrm_state_update()
2189 		err = -EEXIST;  in xfrm_state_update()
2193 	if (x1->km.state == XFRM_STATE_ACQ) {  in xfrm_state_update()
2194 		if (x->dir && x1->dir != x->dir)  in xfrm_state_update()
2197 		__xfrm_state_insert(x);  in xfrm_state_update()
2198 		x = NULL;  in xfrm_state_update()
2200 		if (x1->dir != x->dir)  in xfrm_state_update()
2206 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2214 	if (!x) {  in xfrm_state_update()
2220 	err = -EINVAL;  in xfrm_state_update()
2221 	spin_lock_bh(&x1->lock);  in xfrm_state_update()
2222 	if (likely(x1->km.state == XFRM_STATE_VALID)) {  in xfrm_state_update()
2223 		if (x->encap && x1->encap &&  in xfrm_state_update()
2224 		    x->encap->encap_type == x1->encap->encap_type)  in xfrm_state_update()
2225 			memcpy(x1->encap, x->encap, sizeof(*x1->encap));  in xfrm_state_update()
2226 		else if (x->encap || x1->encap)  in xfrm_state_update()
2229 		if (x->coaddr && x1->coaddr) {  in xfrm_state_update()
2230 			memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));  in xfrm_state_update()
2232 		if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))  in xfrm_state_update()
2233 			memcpy(&x1->sel, &x->sel, sizeof(x1->sel));  in xfrm_state_update()
2234 		memcpy(&x1->lft, &x->lft, sizeof(x1->lft));  in xfrm_state_update()
2235 		x1->km.dying = 0;  in xfrm_state_update()
2237 		hrtimer_start(&x1->mtimer, ktime_set(1, 0),  in xfrm_state_update()
2239 		if (READ_ONCE(x1->curlft.use_time))  in xfrm_state_update()
2242 		if (x->props.smark.m || x->props.smark.v || x->if_id) {  in xfrm_state_update()
2243 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2245 			if (x->props.smark.m || x->props.smark.v)  in xfrm_state_update()
2246 				x1->props.smark = x->props.smark;  in xfrm_state_update()
2248 			if (x->if_id)  in xfrm_state_update()
2249 				x1->if_id = x->if_id;  in xfrm_state_update()
2252 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2256 		x->km.state = XFRM_STATE_DEAD;  in xfrm_state_update()
2257 		__xfrm_state_put(x);  in xfrm_state_update()
2261 	spin_unlock_bh(&x1->lock);  in xfrm_state_update()
2269 int xfrm_state_check_expire(struct xfrm_state *x)  in xfrm_state_check_expire()  argument
2272 	 * are handled by SW for non-packet offload modes. Simply skip  in xfrm_state_check_expire()
2275 	if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)  in xfrm_state_check_expire()
2276 		xfrm_dev_state_update_stats(x);  in xfrm_state_check_expire()
2278 	if (!READ_ONCE(x->curlft.use_time))  in xfrm_state_check_expire()
2279 		WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());  in xfrm_state_check_expire()
2281 	if (x->curlft.bytes >= x->lft.hard_byte_limit ||  in xfrm_state_check_expire()
2282 	    x->curlft.packets >= x->lft.hard_packet_limit) {  in xfrm_state_check_expire()
2283 		x->km.state = XFRM_STATE_EXPIRED;  in xfrm_state_check_expire()
2284 		hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);  in xfrm_state_check_expire()
2285 		return -EINVAL;  in xfrm_state_check_expire()
2288 	if (!x->km.dying &&  in xfrm_state_check_expire()
2289 	    (x->curlft.bytes >= x->lft.soft_byte_limit ||  in xfrm_state_check_expire()
2290 	     x->curlft.packets >= x->lft.soft_packet_limit)) {  in xfrm_state_check_expire()
2291 		x->km.dying = 1;  in xfrm_state_check_expire()
2292 		km_state_expired(x, 0, 0);  in xfrm_state_check_expire()
2300 	struct xfrm_state *x;  in xfrm_state_update_stats()  local
2303 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update_stats()
2304 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_update_stats()
2305 		hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst)  in xfrm_state_update_stats()
2306 			xfrm_dev_state_update_stats(x);  in xfrm_state_update_stats()
2308 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update_stats()
2316 	struct xfrm_state *x;  in xfrm_state_lookup()  local
2321 	x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);  in xfrm_state_lookup()
2323 	return x;  in xfrm_state_lookup()
2333 	struct xfrm_state *x;  in xfrm_state_lookup_byaddr()  local
2339 	x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family);  in xfrm_state_lookup_byaddr()
2341 	return x;  in xfrm_state_lookup_byaddr()
2350 	struct xfrm_state *x;  in xfrm_find_acq()  local
2352 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
2353 	x = __find_acq_core(net, mark, family, mode, reqid, if_id, pcpu_num,  in xfrm_find_acq()
2355 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
2357 	return x;  in xfrm_find_acq()
2380 		count[i] += count[i - 1];  in __xfrm6_sort()
2383 		dst[count[class[i] - 1]++] = src[i];  in __xfrm6_sort()
2400 	switch (v->props.mode) {  in __xfrm6_state_sort_cmp()
2402 		if (v->id.proto != IPPROTO_AH)  in __xfrm6_state_sort_cmp()
2430 	switch (v->mode) {  in __xfrm6_tmpl_sort_cmp()
2494 	struct xfrm_state *x;  in __xfrm_find_acq_byseq()  local
2496 	hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {  in __xfrm_find_acq_byseq()
2497 		if (x->km.seq == seq &&  in __xfrm_find_acq_byseq()
2498 		    (mark & x->mark.m) == x->mark.v &&  in __xfrm_find_acq_byseq()
2499 		    x->pcpu_num == pcpu_num &&  in __xfrm_find_acq_byseq()
2500 		    x->km.state == XFRM_STATE_ACQ) {  in __xfrm_find_acq_byseq()
2501 			xfrm_state_hold(x);  in __xfrm_find_acq_byseq()
2502 			return x;  in __xfrm_find_acq_byseq()
2511 	struct xfrm_state *x;  in xfrm_find_acq_byseq()  local
2513 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2514 	x = __xfrm_find_acq_byseq(net, mark, seq, pcpu_num);  in xfrm_find_acq_byseq()
2515 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2516 	return x;  in xfrm_find_acq_byseq()
2541 		/* IPCOMP spi is 16-bits. */  in verify_spi_info()
2544 			return -EINVAL;  in verify_spi_info()
2550 		return -EINVAL;  in verify_spi_info()
2555 		return -EINVAL;  in verify_spi_info()
2562 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,  in xfrm_alloc_spi()  argument
2565 	struct net *net = xs_net(x);  in xfrm_alloc_spi()
2568 	int err = -ENOENT;  in xfrm_alloc_spi()
2569 	u32 range = high - low + 1;  in xfrm_alloc_spi()
2572 	spin_lock_bh(&x->lock);  in xfrm_alloc_spi()
2573 	if (x->km.state == XFRM_STATE_DEAD) {  in xfrm_alloc_spi()
2579 	if (x->id.spi)  in xfrm_alloc_spi()
2582 	err = -ENOENT;  in xfrm_alloc_spi()
2588 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2589 		x0 = xfrm_state_lookup_spi_proto(net, newspi, x->id.proto);  in xfrm_alloc_spi()
2591 			x->id.spi = newspi;  in xfrm_alloc_spi()
2592 			h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family);  in xfrm_alloc_spi()
2593 			XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, x->xso.type);  in xfrm_alloc_spi()
2594 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2599 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2602 			err = -ERESTARTSYS;  in xfrm_alloc_spi()
2614 	spin_unlock_bh(&x->lock);  in xfrm_alloc_spi()
2620 static bool __xfrm_state_filter_match(struct xfrm_state *x,  in __xfrm_state_filter_match()  argument
2624 		if ((filter->family == AF_INET ||  in __xfrm_state_filter_match()
2625 		     filter->family == AF_INET6) &&  in __xfrm_state_filter_match()
2626 		    x->props.family != filter->family)  in __xfrm_state_filter_match()
2629 		return addr_match(&x->props.saddr, &filter->saddr,  in __xfrm_state_filter_match()
2630 				  filter->splen) &&  in __xfrm_state_filter_match()
2631 		       addr_match(&x->id.daddr, &filter->daddr,  in __xfrm_state_filter_match()
2632 				  filter->dplen);  in __xfrm_state_filter_match()
2642 	struct xfrm_state_walk *x;  in xfrm_state_walk()  local
2645 	if (walk->seq != 0 && list_empty(&walk->all))  in xfrm_state_walk()
2648 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2649 	if (list_empty(&walk->all))  in xfrm_state_walk()
2650 		x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2652 		x = list_first_entry(&walk->all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2653 	list_for_each_entry_from(x, &net->xfrm.state_all, all) {  in xfrm_state_walk()
2654 		if (x->state == XFRM_STATE_DEAD)  in xfrm_state_walk()
2656 		state = container_of(x, struct xfrm_state, km);  in xfrm_state_walk()
2657 		if (!xfrm_id_proto_match(state->id.proto, walk->proto))  in xfrm_state_walk()
2659 		if (!__xfrm_state_filter_match(state, walk->filter))  in xfrm_state_walk()
2661 		err = func(state, walk->seq, data);  in xfrm_state_walk()
2663 			list_move_tail(&walk->all, &x->all);  in xfrm_state_walk()
2666 		walk->seq++;  in xfrm_state_walk()
2668 	if (walk->seq == 0) {  in xfrm_state_walk()
2669 		err = -ENOENT;  in xfrm_state_walk()
2672 	list_del_init(&walk->all);  in xfrm_state_walk()
2674 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2682 	INIT_LIST_HEAD(&walk->all);  in xfrm_state_walk_init()
2683 	walk->proto = proto;  in xfrm_state_walk_init()
2684 	walk->state = XFRM_STATE_DEAD;  in xfrm_state_walk_init()
2685 	walk->seq = 0;  in xfrm_state_walk_init()
2686 	walk->filter = filter;  in xfrm_state_walk_init()
2692 	kfree(walk->filter);  in xfrm_state_walk_done()
2694 	if (list_empty(&walk->all))  in xfrm_state_walk_done()
2697 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2698 	list_del(&walk->all);  in xfrm_state_walk_done()
2699 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2705 	struct xfrm_state *x = timer_container_of(x, t, rtimer);  in xfrm_replay_timer_handler()  local
2707 	spin_lock(&x->lock);  in xfrm_replay_timer_handler()
2709 	if (x->km.state == XFRM_STATE_VALID) {  in xfrm_replay_timer_handler()
2710 		if (xfrm_aevent_is_on(xs_net(x)))  in xfrm_replay_timer_handler()
2711 			xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);  in xfrm_replay_timer_handler()
2713 			x->xflags |= XFRM_TIME_DEFER;  in xfrm_replay_timer_handler()
2716 	spin_unlock(&x->lock);  in xfrm_replay_timer_handler()
2727 		if (km->notify_policy)  in km_policy_notify()
2728 			km->notify_policy(xp, dir, c);  in km_policy_notify()
2732 void km_state_notify(struct xfrm_state *x, const struct km_event *c)  in km_state_notify()  argument
2737 		if (km->notify)  in km_state_notify()
2738 			km->notify(x, c);  in km_state_notify()
2745 void km_state_expired(struct xfrm_state *x, int hard, u32 portid)  in km_state_expired()  argument
2752 	km_state_notify(x, &c);  in km_state_expired()
2760 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)  in km_query()  argument
2762 	int err = -EINVAL, acqret;  in km_query()
2767 		acqret = km->acquire(x, t, pol);  in km_query()
2776 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)  in __km_new_mapping()  argument
2778 	int err = -EINVAL;  in __km_new_mapping()
2783 		if (km->new_mapping)  in __km_new_mapping()
2784 			err = km->new_mapping(x, ipaddr, sport);  in __km_new_mapping()
2792 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)  in km_new_mapping()  argument
2796 	if (x->mapping_maxage) {  in km_new_mapping()
2797 		if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||  in km_new_mapping()
2798 		    x->new_mapping_sport != sport) {  in km_new_mapping()
2799 			x->new_mapping_sport = sport;  in km_new_mapping()
2800 			x->new_mapping = jiffies / HZ;  in km_new_mapping()
2801 			ret = __km_new_mapping(x, ipaddr, sport);  in km_new_mapping()
2804 		ret = __km_new_mapping(x, ipaddr, sport);  in km_new_mapping()
2828 	int err = -EINVAL;  in km_migrate()
2834 		if (km->migrate) {  in km_migrate()
2835 			ret = km->migrate(sel, dir, type, m, num_migrate, k,  in km_migrate()
2849 	int err = -EINVAL;  in km_report()
2855 		if (km->report) {  in km_report()
2856 			ret = km->report(net, proto, sel, addr);  in km_report()
2873 		if (km->is_alive && km->is_alive(c)) {  in km_is_alive()
2895 	if (!try_module_get(xtr->owner))  in xfrm_get_translator()
2905 	module_put(xtr->owner);  in xfrm_put_translator()
2915 		err = -EEXIST;  in xfrm_register_translator()
2931 			err = -EINVAL;  in xfrm_unregister_translator()
2958 		return -EMSGSIZE;  in xfrm_user_policy()
2969 			return -EOPNOTSUPP;  in xfrm_user_policy()
2972 		err = xtr->xlate_user_policy_sockptr(&data, optlen);  in xfrm_user_policy()
2980 	err = -EINVAL;  in xfrm_user_policy()
2983 		pol = km->compile_policy(sk, optname, data,  in xfrm_user_policy()
3007 	list_add_tail_rcu(&km->list, &xfrm_km_list);  in xfrm_register_km()
3015 	list_del_rcu(&km->list);  in xfrm_unregister_km()
3025 	if (WARN_ON(afinfo->family >= NPROTO))  in xfrm_state_register_afinfo()
3026 		return -EAFNOSUPPORT;  in xfrm_state_register_afinfo()
3029 	if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))  in xfrm_state_register_afinfo()
3030 		err = -EEXIST;  in xfrm_state_register_afinfo()
3032 		rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);  in xfrm_state_register_afinfo()
3040 	int err = 0, family = afinfo->family;  in xfrm_state_unregister_afinfo()
3043 		return -EAFNOSUPPORT;  in xfrm_state_unregister_afinfo()
3046 	if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {  in xfrm_state_unregister_afinfo()
3048 			err = -EINVAL;  in xfrm_state_unregister_afinfo()
3050 			RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);  in xfrm_state_unregister_afinfo()
3085 static void xfrm_state_delete_tunnel(struct xfrm_state *x)  in xfrm_state_delete_tunnel()  argument
3087 	if (x->tunnel) {  in xfrm_state_delete_tunnel()
3088 		struct xfrm_state *t = x->tunnel;  in xfrm_state_delete_tunnel()
3090 		if (atomic_dec_return(&t->tunnel_users) == 1)  in xfrm_state_delete_tunnel()
3093 		x->tunnel = NULL;  in xfrm_state_delete_tunnel()
3097 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)  in xfrm_state_mtu()  argument
3099 	const struct xfrm_type *type = READ_ONCE(x->type);  in xfrm_state_mtu()
3103 	if (x->km.state != XFRM_STATE_VALID ||  in xfrm_state_mtu()
3104 	    !type || type->proto != IPPROTO_ESP)  in xfrm_state_mtu()
3105 		return mtu - x->props.header_len;  in xfrm_state_mtu()
3107 	aead = x->data;  in xfrm_state_mtu()
3110 	switch (x->props.mode) {  in xfrm_state_mtu()
3113 		if (x->props.family == AF_INET)  in xfrm_state_mtu()
3115 		else if (x->props.family == AF_INET6)  in xfrm_state_mtu()
3121 		if (x->mode_cbs && x->mode_cbs->get_inner_mtu)  in xfrm_state_mtu()
3122 			return x->mode_cbs->get_inner_mtu(x, mtu);  in xfrm_state_mtu()
3128 	return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -  in xfrm_state_mtu()
3129 		 net_adj) & ~(blksize - 1)) + net_adj - 2;  in xfrm_state_mtu()
3133 int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)  in __xfrm_init_state()  argument
3137 	int family = x->props.family;  in __xfrm_init_state()
3141 	    READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))  in __xfrm_init_state()
3142 		x->props.flags |= XFRM_STATE_NOPMTUDISC;  in __xfrm_init_state()
3144 	err = -EPROTONOSUPPORT;  in __xfrm_init_state()
3146 	if (x->sel.family != AF_UNSPEC) {  in __xfrm_init_state()
3147 		inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);  in __xfrm_init_state()
3153 		if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&  in __xfrm_init_state()
3154 		    family != x->sel.family) {  in __xfrm_init_state()
3159 		x->inner_mode = *inner_mode;  in __xfrm_init_state()
3164 		inner_mode = xfrm_get_mode(x->props.mode, x->props.family);  in __xfrm_init_state()
3170 		x->inner_mode = *inner_mode;  in __xfrm_init_state()
3172 		if (x->props.family == AF_INET)  in __xfrm_init_state()
3175 		inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);  in __xfrm_init_state()
3177 			if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)  in __xfrm_init_state()
3178 				x->inner_mode_iaf = *inner_mode_iaf;  in __xfrm_init_state()
3182 	x->type = xfrm_get_type(x->id.proto, family);  in __xfrm_init_state()
3183 	if (x->type == NULL) {  in __xfrm_init_state()
3188 	err = x->type->init_state(x, extack);  in __xfrm_init_state()
3192 	outer_mode = xfrm_get_mode(x->props.mode, family);  in __xfrm_init_state()
3195 		err = -EPROTONOSUPPORT;  in __xfrm_init_state()
3199 	x->outer_mode = *outer_mode;  in __xfrm_init_state()
3200 	if (x->nat_keepalive_interval) {  in __xfrm_init_state()
3201 		if (x->dir != XFRM_SA_DIR_OUT) {  in __xfrm_init_state()
3203 			err = -EINVAL;  in __xfrm_init_state()
3207 		if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) {  in __xfrm_init_state()
3210 			err = -EINVAL;  in __xfrm_init_state()
3215 	x->mode_cbs = xfrm_get_mode_cbs(x->props.mode);  in __xfrm_init_state()
3216 	if (x->mode_cbs) {  in __xfrm_init_state()
3217 		if (x->mode_cbs->init_state)  in __xfrm_init_state()
3218 			err = x->mode_cbs->init_state(x);  in __xfrm_init_state()
3219 		module_put(x->mode_cbs->owner);  in __xfrm_init_state()
3227 int xfrm_init_state(struct xfrm_state *x)  in xfrm_init_state()  argument
3231 	err = __xfrm_init_state(x, NULL);  in xfrm_init_state()
3235 	err = xfrm_init_replay(x, NULL);  in xfrm_init_state()
3239 	x->km.state = XFRM_STATE_VALID;  in xfrm_init_state()
3253 	INIT_LIST_HEAD(&net->xfrm.state_all);  in xfrm_state_init()
3257 	net->xfrm.state_bydst = xfrm_hash_alloc(sz);  in xfrm_state_init()
3258 	if (!net->xfrm.state_bydst)  in xfrm_state_init()
3260 	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);  in xfrm_state_init()
3261 	if (!net->xfrm.state_bysrc)  in xfrm_state_init()
3263 	net->xfrm.state_byspi = xfrm_hash_alloc(sz);  in xfrm_state_init()
3264 	if (!net->xfrm.state_byspi)  in xfrm_state_init()
3266 	net->xfrm.state_byseq = xfrm_hash_alloc(sz);  in xfrm_state_init()
3267 	if (!net->xfrm.state_byseq)  in xfrm_state_init()
3270 	net->xfrm.state_cache_input = alloc_percpu(struct hlist_head);  in xfrm_state_init()
3271 	if (!net->xfrm.state_cache_input)  in xfrm_state_init()
3274 	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);  in xfrm_state_init()
3276 	net->xfrm.state_num = 0;  in xfrm_state_init()
3277 	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);  in xfrm_state_init()
3278 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_state_init()
3279 	seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,  in xfrm_state_init()
3280 			       &net->xfrm.xfrm_state_lock);  in xfrm_state_init()
3284 	xfrm_hash_free(net->xfrm.state_byseq, sz);  in xfrm_state_init()
3286 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_init()
3288 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_init()
3290 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_init()
3292 	return -ENOMEM;  in xfrm_state_init()
3299 	flush_work(&net->xfrm.state_hash_work);  in xfrm_state_fini()
3303 	WARN_ON(!list_empty(&net->xfrm.state_all));  in xfrm_state_fini()
3305 	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);  in xfrm_state_fini()
3306 	WARN_ON(!hlist_empty(net->xfrm.state_byseq));  in xfrm_state_fini()
3307 	xfrm_hash_free(net->xfrm.state_byseq, sz);  in xfrm_state_fini()
3308 	WARN_ON(!hlist_empty(net->xfrm.state_byspi));  in xfrm_state_fini()
3309 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_fini()
3310 	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));  in xfrm_state_fini()
3311 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_fini()
3312 	WARN_ON(!hlist_empty(net->xfrm.state_bydst));  in xfrm_state_fini()
3313 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_fini()
3314 	free_percpu(net->xfrm.state_cache_input);  in xfrm_state_fini()
3318 static void xfrm_audit_helper_sainfo(struct xfrm_state *x,  in xfrm_audit_helper_sainfo()  argument
3321 	struct xfrm_sec_ctx *ctx = x->security;  in xfrm_audit_helper_sainfo()
3322 	u32 spi = ntohl(x->id.spi);  in xfrm_audit_helper_sainfo()
3326 				 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);  in xfrm_audit_helper_sainfo()
3328 	switch (x->props.family) {  in xfrm_audit_helper_sainfo()
3331 				 &x->props.saddr.a4, &x->id.daddr.a4);  in xfrm_audit_helper_sainfo()
3335 				 x->props.saddr.a6, x->id.daddr.a6);  in xfrm_audit_helper_sainfo()
3339 	audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);  in xfrm_audit_helper_sainfo()
3352 				 &iph4->saddr, &iph4->daddr);  in xfrm_audit_helper_pktinfo()
3357 				 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",  in xfrm_audit_helper_pktinfo()
3358 				 &iph6->saddr, &iph6->daddr,  in xfrm_audit_helper_pktinfo()
3359 				 iph6->flow_lbl[0] & 0x0f,  in xfrm_audit_helper_pktinfo()
3360 				 iph6->flow_lbl[1],  in xfrm_audit_helper_pktinfo()
3361 				 iph6->flow_lbl[2]);  in xfrm_audit_helper_pktinfo()
3366 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)  in xfrm_audit_state_add()  argument
3370 	audit_buf = xfrm_audit_start("SAD-add");  in xfrm_audit_state_add()
3374 	xfrm_audit_helper_sainfo(x, audit_buf);  in xfrm_audit_state_add()
3380 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)  in xfrm_audit_state_delete()  argument
3384 	audit_buf = xfrm_audit_start("SAD-delete");  in xfrm_audit_state_delete()
3388 	xfrm_audit_helper_sainfo(x, audit_buf);  in xfrm_audit_state_delete()
3394 void xfrm_audit_state_replay_overflow(struct xfrm_state *x,  in xfrm_audit_state_replay_overflow()  argument
3400 	audit_buf = xfrm_audit_start("SA-replay-overflow");  in xfrm_audit_state_replay_overflow()
3403 	xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);  in xfrm_audit_state_replay_overflow()
3406 	spi = ntohl(x->id.spi);  in xfrm_audit_state_replay_overflow()
3407 	audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);  in xfrm_audit_state_replay_overflow()
3412 void xfrm_audit_state_replay(struct xfrm_state *x,  in xfrm_audit_state_replay()  argument
3418 	audit_buf = xfrm_audit_start("SA-replayed-pkt");  in xfrm_audit_state_replay()
3421 	xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);  in xfrm_audit_state_replay()
3422 	spi = ntohl(x->id.spi);  in xfrm_audit_state_replay()
3423 	audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",  in xfrm_audit_state_replay()
3433 	audit_buf = xfrm_audit_start("SA-notfound");  in xfrm_audit_state_notfound_simple()
3447 	audit_buf = xfrm_audit_start("SA-notfound");  in xfrm_audit_state_notfound()
3452 	audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",  in xfrm_audit_state_notfound()
3458 void xfrm_audit_state_icvfail(struct xfrm_state *x,  in xfrm_audit_state_icvfail()  argument
3465 	audit_buf = xfrm_audit_start("SA-icv-failure");  in xfrm_audit_state_icvfail()
3468 	xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);  in xfrm_audit_state_icvfail()
3471 		audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",  in xfrm_audit_state_icvfail()