xref: /linux/net/xfrm/xfrm_device.c (revision 1a9239bb4253f9076b5b4b2a1a4e8d7defd77a95)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * xfrm_device.c - IPsec device offloading code.
4  *
5  * Copyright (c) 2015 secunet Security Networks AG
6  *
7  * Author:
8  * Steffen Klassert <steffen.klassert@secunet.com>
9  */
10 
11 #include <linux/errno.h>
12 #include <linux/module.h>
13 #include <linux/netdevice.h>
14 #include <linux/skbuff.h>
15 #include <linux/slab.h>
16 #include <linux/spinlock.h>
17 #include <net/dst.h>
18 #include <net/gso.h>
19 #include <net/xfrm.h>
20 #include <linux/notifier.h>
21 
22 #ifdef CONFIG_XFRM_OFFLOAD
__xfrm_transport_prep(struct xfrm_state * x,struct sk_buff * skb,unsigned int hsize)23 static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
24 				  unsigned int hsize)
25 {
26 	struct xfrm_offload *xo = xfrm_offload(skb);
27 
28 	skb_reset_mac_len(skb);
29 	if (xo->flags & XFRM_GSO_SEGMENT)
30 		skb->transport_header -= x->props.header_len;
31 
32 	pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
33 }
34 
__xfrm_mode_tunnel_prep(struct xfrm_state * x,struct sk_buff * skb,unsigned int hsize)35 static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
36 				    unsigned int hsize)
37 
38 {
39 	struct xfrm_offload *xo = xfrm_offload(skb);
40 
41 	if (xo->flags & XFRM_GSO_SEGMENT)
42 		skb->transport_header = skb->network_header + hsize;
43 
44 	skb_reset_mac_len(skb);
45 	pskb_pull(skb,
46 		  skb->mac_len + x->props.header_len - x->props.enc_hdr_len);
47 }
48 
__xfrm_mode_beet_prep(struct xfrm_state * x,struct sk_buff * skb,unsigned int hsize)49 static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
50 				  unsigned int hsize)
51 {
52 	struct xfrm_offload *xo = xfrm_offload(skb);
53 	int phlen = 0;
54 
55 	if (xo->flags & XFRM_GSO_SEGMENT)
56 		skb->transport_header = skb->network_header + hsize;
57 
58 	skb_reset_mac_len(skb);
59 	if (x->sel.family != AF_INET6) {
60 		phlen = IPV4_BEET_PHMAXLEN;
61 		if (x->outer_mode.family == AF_INET6)
62 			phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
63 	}
64 
65 	pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
66 }
67 
68 /* Adjust pointers into the packet when IPsec is done at layer2 */
xfrm_outer_mode_prep(struct xfrm_state * x,struct sk_buff * skb)69 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
70 {
71 	switch (x->outer_mode.encap) {
72 	case XFRM_MODE_IPTFS:
73 	case XFRM_MODE_TUNNEL:
74 		if (x->outer_mode.family == AF_INET)
75 			return __xfrm_mode_tunnel_prep(x, skb,
76 						       sizeof(struct iphdr));
77 		if (x->outer_mode.family == AF_INET6)
78 			return __xfrm_mode_tunnel_prep(x, skb,
79 						       sizeof(struct ipv6hdr));
80 		break;
81 	case XFRM_MODE_TRANSPORT:
82 		if (x->outer_mode.family == AF_INET)
83 			return __xfrm_transport_prep(x, skb,
84 						     sizeof(struct iphdr));
85 		if (x->outer_mode.family == AF_INET6)
86 			return __xfrm_transport_prep(x, skb,
87 						     sizeof(struct ipv6hdr));
88 		break;
89 	case XFRM_MODE_BEET:
90 		if (x->outer_mode.family == AF_INET)
91 			return __xfrm_mode_beet_prep(x, skb,
92 						     sizeof(struct iphdr));
93 		if (x->outer_mode.family == AF_INET6)
94 			return __xfrm_mode_beet_prep(x, skb,
95 						     sizeof(struct ipv6hdr));
96 		break;
97 	case XFRM_MODE_ROUTEOPTIMIZATION:
98 	case XFRM_MODE_IN_TRIGGER:
99 		break;
100 	}
101 }
102 
xmit_xfrm_check_overflow(struct sk_buff * skb)103 static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb)
104 {
105 	struct xfrm_offload *xo = xfrm_offload(skb);
106 	__u32 seq = xo->seq.low;
107 
108 	seq += skb_shinfo(skb)->gso_segs;
109 	if (unlikely(seq < xo->seq.low))
110 		return true;
111 
112 	return false;
113 }
114 
validate_xmit_xfrm(struct sk_buff * skb,netdev_features_t features,bool * again)115 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
116 {
117 	int err;
118 	unsigned long flags;
119 	struct xfrm_state *x;
120 	struct softnet_data *sd;
121 	struct sk_buff *skb2, *nskb, *pskb = NULL;
122 	netdev_features_t esp_features = features;
123 	struct xfrm_offload *xo = xfrm_offload(skb);
124 	struct net_device *dev = skb->dev;
125 	struct sec_path *sp;
126 
127 	if (!xo || (xo->flags & XFRM_XMIT))
128 		return skb;
129 
130 	if (!(features & NETIF_F_HW_ESP))
131 		esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
132 
133 	sp = skb_sec_path(skb);
134 	x = sp->xvec[sp->len - 1];
135 	if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
136 		return skb;
137 
138 	/* The packet was sent to HW IPsec packet offload engine,
139 	 * but to wrong device. Drop the packet, so it won't skip
140 	 * XFRM stack.
141 	 */
142 	if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) {
143 		kfree_skb(skb);
144 		dev_core_stats_tx_dropped_inc(dev);
145 		return NULL;
146 	}
147 
148 	/* This skb was already validated on the upper/virtual dev */
149 	if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
150 		return skb;
151 
152 	local_irq_save(flags);
153 	sd = this_cpu_ptr(&softnet_data);
154 	err = !skb_queue_empty(&sd->xfrm_backlog);
155 	local_irq_restore(flags);
156 
157 	if (err) {
158 		*again = true;
159 		return skb;
160 	}
161 
162 	if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) ||
163 				unlikely(xmit_xfrm_check_overflow(skb)))) {
164 		struct sk_buff *segs;
165 
166 		/* Packet got rerouted, fixup features and segment it. */
167 		esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP);
168 
169 		segs = skb_gso_segment(skb, esp_features);
170 		if (IS_ERR(segs)) {
171 			kfree_skb(skb);
172 			dev_core_stats_tx_dropped_inc(dev);
173 			return NULL;
174 		} else {
175 			consume_skb(skb);
176 			skb = segs;
177 		}
178 	}
179 
180 	if (!skb->next) {
181 		esp_features |= skb->dev->gso_partial_features;
182 		xfrm_outer_mode_prep(x, skb);
183 
184 		xo->flags |= XFRM_DEV_RESUME;
185 
186 		err = x->type_offload->xmit(x, skb, esp_features);
187 		if (err) {
188 			if (err == -EINPROGRESS)
189 				return NULL;
190 
191 			XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
192 			kfree_skb(skb);
193 			return NULL;
194 		}
195 
196 		skb_push(skb, skb->data - skb_mac_header(skb));
197 
198 		return skb;
199 	}
200 
201 	skb_list_walk_safe(skb, skb2, nskb) {
202 		esp_features |= skb->dev->gso_partial_features;
203 		skb_mark_not_on_list(skb2);
204 
205 		xo = xfrm_offload(skb2);
206 		xo->flags |= XFRM_DEV_RESUME;
207 
208 		xfrm_outer_mode_prep(x, skb2);
209 
210 		err = x->type_offload->xmit(x, skb2, esp_features);
211 		if (!err) {
212 			skb2->next = nskb;
213 		} else if (err != -EINPROGRESS) {
214 			XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
215 			skb2->next = nskb;
216 			kfree_skb_list(skb2);
217 			return NULL;
218 		} else {
219 			if (skb == skb2)
220 				skb = nskb;
221 			else
222 				pskb->next = nskb;
223 
224 			continue;
225 		}
226 
227 		skb_push(skb2, skb2->data - skb_mac_header(skb2));
228 		pskb = skb2;
229 	}
230 
231 	return skb;
232 }
233 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
234 
xfrm_dev_state_add(struct net * net,struct xfrm_state * x,struct xfrm_user_offload * xuo,struct netlink_ext_ack * extack)235 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
236 		       struct xfrm_user_offload *xuo,
237 		       struct netlink_ext_ack *extack)
238 {
239 	int err;
240 	struct dst_entry *dst;
241 	struct net_device *dev;
242 	struct xfrm_dev_offload *xso = &x->xso;
243 	xfrm_address_t *saddr;
244 	xfrm_address_t *daddr;
245 	bool is_packet_offload;
246 
247 	if (xuo->flags &
248 	    ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) {
249 		NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
250 		return -EINVAL;
251 	}
252 
253 	if ((xuo->flags & XFRM_OFFLOAD_INBOUND && x->dir == XFRM_SA_DIR_OUT) ||
254 	    (!(xuo->flags & XFRM_OFFLOAD_INBOUND) && x->dir == XFRM_SA_DIR_IN)) {
255 		NL_SET_ERR_MSG(extack, "Mismatched SA and offload direction");
256 		return -EINVAL;
257 	}
258 
259 	is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET;
260 
261 	/* We don't yet support TFC padding. */
262 	if (x->tfcpad) {
263 		NL_SET_ERR_MSG(extack, "TFC padding can't be offloaded");
264 		return -EINVAL;
265 	}
266 
267 	dev = dev_get_by_index(net, xuo->ifindex);
268 	if (!dev) {
269 		struct xfrm_dst_lookup_params params;
270 
271 		if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
272 			saddr = &x->props.saddr;
273 			daddr = &x->id.daddr;
274 		} else {
275 			saddr = &x->id.daddr;
276 			daddr = &x->props.saddr;
277 		}
278 
279 		memset(&params, 0, sizeof(params));
280 		params.net = net;
281 		params.saddr = saddr;
282 		params.daddr = daddr;
283 		params.mark = xfrm_smark_get(0, x);
284 		dst = __xfrm_dst_lookup(x->props.family, &params);
285 		if (IS_ERR(dst))
286 			return (is_packet_offload) ? -EINVAL : 0;
287 
288 		dev = dst->dev;
289 
290 		dev_hold(dev);
291 		dst_release(dst);
292 	}
293 
294 	if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
295 		xso->dev = NULL;
296 		dev_put(dev);
297 		return (is_packet_offload) ? -EINVAL : 0;
298 	}
299 
300 	if (!is_packet_offload && x->props.flags & XFRM_STATE_ESN &&
301 	    !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
302 		NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN");
303 		xso->dev = NULL;
304 		dev_put(dev);
305 		return -EINVAL;
306 	}
307 
308 	xfrm_set_type_offload(x);
309 	if (!x->type_offload) {
310 		NL_SET_ERR_MSG(extack, "Type doesn't support offload");
311 		dev_put(dev);
312 		return -EINVAL;
313 	}
314 
315 	xso->dev = dev;
316 	netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
317 	xso->real_dev = dev;
318 
319 	if (xuo->flags & XFRM_OFFLOAD_INBOUND)
320 		xso->dir = XFRM_DEV_OFFLOAD_IN;
321 	else
322 		xso->dir = XFRM_DEV_OFFLOAD_OUT;
323 
324 	if (is_packet_offload)
325 		xso->type = XFRM_DEV_OFFLOAD_PACKET;
326 	else
327 		xso->type = XFRM_DEV_OFFLOAD_CRYPTO;
328 
329 	err = dev->xfrmdev_ops->xdo_dev_state_add(x, extack);
330 	if (err) {
331 		xso->dev = NULL;
332 		xso->dir = 0;
333 		xso->real_dev = NULL;
334 		netdev_put(dev, &xso->dev_tracker);
335 		xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
336 
337 		xfrm_unset_type_offload(x);
338 		/* User explicitly requested packet offload mode and configured
339 		 * policy in addition to the XFRM state. So be civil to users,
340 		 * and return an error instead of taking fallback path.
341 		 */
342 		if ((err != -EOPNOTSUPP && !is_packet_offload) || is_packet_offload) {
343 			NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this state");
344 			return err;
345 		}
346 	}
347 
348 	return 0;
349 }
350 EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
351 
xfrm_dev_policy_add(struct net * net,struct xfrm_policy * xp,struct xfrm_user_offload * xuo,u8 dir,struct netlink_ext_ack * extack)352 int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
353 			struct xfrm_user_offload *xuo, u8 dir,
354 			struct netlink_ext_ack *extack)
355 {
356 	struct xfrm_dev_offload *xdo = &xp->xdo;
357 	struct net_device *dev;
358 	int err;
359 
360 	if (!xuo->flags || xuo->flags & ~XFRM_OFFLOAD_PACKET) {
361 		/* We support only packet offload mode and it means
362 		 * that user must set XFRM_OFFLOAD_PACKET bit.
363 		 */
364 		NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
365 		return -EINVAL;
366 	}
367 
368 	dev = dev_get_by_index(net, xuo->ifindex);
369 	if (!dev)
370 		return -EINVAL;
371 
372 	if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_policy_add) {
373 		xdo->dev = NULL;
374 		dev_put(dev);
375 		NL_SET_ERR_MSG(extack, "Policy offload is not supported");
376 		return -EINVAL;
377 	}
378 
379 	xdo->dev = dev;
380 	netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC);
381 	xdo->real_dev = dev;
382 	xdo->type = XFRM_DEV_OFFLOAD_PACKET;
383 	switch (dir) {
384 	case XFRM_POLICY_IN:
385 		xdo->dir = XFRM_DEV_OFFLOAD_IN;
386 		break;
387 	case XFRM_POLICY_OUT:
388 		xdo->dir = XFRM_DEV_OFFLOAD_OUT;
389 		break;
390 	case XFRM_POLICY_FWD:
391 		xdo->dir = XFRM_DEV_OFFLOAD_FWD;
392 		break;
393 	default:
394 		xdo->dev = NULL;
395 		netdev_put(dev, &xdo->dev_tracker);
396 		NL_SET_ERR_MSG(extack, "Unrecognized offload direction");
397 		return -EINVAL;
398 	}
399 
400 	err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack);
401 	if (err) {
402 		xdo->dev = NULL;
403 		xdo->real_dev = NULL;
404 		xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
405 		xdo->dir = 0;
406 		netdev_put(dev, &xdo->dev_tracker);
407 		NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this policy");
408 		return err;
409 	}
410 
411 	return 0;
412 }
413 EXPORT_SYMBOL_GPL(xfrm_dev_policy_add);
414 
xfrm_dev_offload_ok(struct sk_buff * skb,struct xfrm_state * x)415 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
416 {
417 	int mtu;
418 	struct dst_entry *dst = skb_dst(skb);
419 	struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
420 	struct net_device *dev = x->xso.dev;
421 	bool check_tunnel_size;
422 
423 	if (x->xso.type == XFRM_DEV_OFFLOAD_UNSPECIFIED)
424 		return false;
425 
426 	if ((dev == xfrm_dst_path(dst)->dev) && !xdst->child->xfrm) {
427 		mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
428 		if (skb->len <= mtu)
429 			goto ok;
430 
431 		if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
432 			goto ok;
433 	}
434 
435 	return false;
436 
437 ok:
438 	check_tunnel_size = x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
439 			    x->props.mode == XFRM_MODE_TUNNEL;
440 	switch (x->props.family) {
441 	case AF_INET:
442 		/* Check for IPv4 options */
443 		if (ip_hdr(skb)->ihl != 5)
444 			return false;
445 		if (check_tunnel_size && xfrm4_tunnel_check_size(skb))
446 			return false;
447 		break;
448 	case AF_INET6:
449 		/* Check for IPv6 extensions */
450 		if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
451 			return false;
452 		if (check_tunnel_size && xfrm6_tunnel_check_size(skb))
453 			return false;
454 		break;
455 	default:
456 		break;
457 	}
458 
459 	if (dev->xfrmdev_ops->xdo_dev_offload_ok)
460 		return dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
461 
462 	return true;
463 }
464 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
465 
xfrm_dev_resume(struct sk_buff * skb)466 void xfrm_dev_resume(struct sk_buff *skb)
467 {
468 	struct net_device *dev = skb->dev;
469 	int ret = NETDEV_TX_BUSY;
470 	struct netdev_queue *txq;
471 	struct softnet_data *sd;
472 	unsigned long flags;
473 
474 	rcu_read_lock();
475 	txq = netdev_core_pick_tx(dev, skb, NULL);
476 
477 	HARD_TX_LOCK(dev, txq, smp_processor_id());
478 	if (!netif_xmit_frozen_or_stopped(txq))
479 		skb = dev_hard_start_xmit(skb, dev, txq, &ret);
480 	HARD_TX_UNLOCK(dev, txq);
481 
482 	if (!dev_xmit_complete(ret)) {
483 		local_irq_save(flags);
484 		sd = this_cpu_ptr(&softnet_data);
485 		skb_queue_tail(&sd->xfrm_backlog, skb);
486 		raise_softirq_irqoff(NET_TX_SOFTIRQ);
487 		local_irq_restore(flags);
488 	}
489 	rcu_read_unlock();
490 }
491 EXPORT_SYMBOL_GPL(xfrm_dev_resume);
492 
xfrm_dev_backlog(struct softnet_data * sd)493 void xfrm_dev_backlog(struct softnet_data *sd)
494 {
495 	struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
496 	struct sk_buff_head list;
497 	struct sk_buff *skb;
498 
499 	if (skb_queue_empty(xfrm_backlog))
500 		return;
501 
502 	__skb_queue_head_init(&list);
503 
504 	spin_lock(&xfrm_backlog->lock);
505 	skb_queue_splice_init(xfrm_backlog, &list);
506 	spin_unlock(&xfrm_backlog->lock);
507 
508 	while (!skb_queue_empty(&list)) {
509 		skb = __skb_dequeue(&list);
510 		xfrm_dev_resume(skb);
511 	}
512 
513 }
514 #endif
515 
xfrm_api_check(struct net_device * dev)516 static int xfrm_api_check(struct net_device *dev)
517 {
518 #ifdef CONFIG_XFRM_OFFLOAD
519 	if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
520 	    !(dev->features & NETIF_F_HW_ESP))
521 		return NOTIFY_BAD;
522 
523 	if ((dev->features & NETIF_F_HW_ESP) &&
524 	    (!(dev->xfrmdev_ops &&
525 	       dev->xfrmdev_ops->xdo_dev_state_add &&
526 	       dev->xfrmdev_ops->xdo_dev_state_delete)))
527 		return NOTIFY_BAD;
528 #else
529 	if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
530 		return NOTIFY_BAD;
531 #endif
532 
533 	return NOTIFY_DONE;
534 }
535 
xfrm_dev_down(struct net_device * dev)536 static int xfrm_dev_down(struct net_device *dev)
537 {
538 	if (dev->features & NETIF_F_HW_ESP) {
539 		xfrm_dev_state_flush(dev_net(dev), dev, true);
540 		xfrm_dev_policy_flush(dev_net(dev), dev, true);
541 	}
542 
543 	return NOTIFY_DONE;
544 }
545 
xfrm_dev_event(struct notifier_block * this,unsigned long event,void * ptr)546 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
547 {
548 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
549 
550 	switch (event) {
551 	case NETDEV_REGISTER:
552 		return xfrm_api_check(dev);
553 
554 	case NETDEV_FEAT_CHANGE:
555 		return xfrm_api_check(dev);
556 
557 	case NETDEV_DOWN:
558 	case NETDEV_UNREGISTER:
559 		return xfrm_dev_down(dev);
560 	}
561 	return NOTIFY_DONE;
562 }
563 
564 static struct notifier_block xfrm_dev_notifier = {
565 	.notifier_call	= xfrm_dev_event,
566 };
567 
xfrm_dev_init(void)568 void __init xfrm_dev_init(void)
569 {
570 	register_netdevice_notifier(&xfrm_dev_notifier);
571 }
572