1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * xfrm_device.c - IPsec device offloading code.
4 *
5 * Copyright (c) 2015 secunet Security Networks AG
6 *
7 * Author:
8 * Steffen Klassert <steffen.klassert@secunet.com>
9 */
10
11 #include <linux/errno.h>
12 #include <linux/module.h>
13 #include <linux/netdevice.h>
14 #include <linux/skbuff.h>
15 #include <linux/slab.h>
16 #include <linux/spinlock.h>
17 #include <net/dst.h>
18 #include <net/gso.h>
19 #include <net/xfrm.h>
20 #include <linux/notifier.h>
21
22 #ifdef CONFIG_XFRM_OFFLOAD
__xfrm_transport_prep(struct xfrm_state * x,struct sk_buff * skb,unsigned int hsize)23 static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
24 unsigned int hsize)
25 {
26 struct xfrm_offload *xo = xfrm_offload(skb);
27
28 skb_reset_mac_len(skb);
29 if (xo->flags & XFRM_GSO_SEGMENT)
30 skb->transport_header -= x->props.header_len;
31
32 pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
33 }
34
__xfrm_mode_tunnel_prep(struct xfrm_state * x,struct sk_buff * skb,unsigned int hsize)35 static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
36 unsigned int hsize)
37
38 {
39 struct xfrm_offload *xo = xfrm_offload(skb);
40
41 if (xo->flags & XFRM_GSO_SEGMENT)
42 skb->transport_header = skb->network_header + hsize;
43
44 skb_reset_mac_len(skb);
45 pskb_pull(skb,
46 skb->mac_len + x->props.header_len - x->props.enc_hdr_len);
47 }
48
__xfrm_mode_beet_prep(struct xfrm_state * x,struct sk_buff * skb,unsigned int hsize)49 static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
50 unsigned int hsize)
51 {
52 struct xfrm_offload *xo = xfrm_offload(skb);
53 int phlen = 0;
54
55 if (xo->flags & XFRM_GSO_SEGMENT)
56 skb->transport_header = skb->network_header + hsize;
57
58 skb_reset_mac_len(skb);
59 if (x->sel.family != AF_INET6) {
60 phlen = IPV4_BEET_PHMAXLEN;
61 if (x->outer_mode.family == AF_INET6)
62 phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
63 }
64
65 pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
66 }
67
68 /* Adjust pointers into the packet when IPsec is done at layer2 */
xfrm_outer_mode_prep(struct xfrm_state * x,struct sk_buff * skb)69 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
70 {
71 switch (x->outer_mode.encap) {
72 case XFRM_MODE_IPTFS:
73 case XFRM_MODE_TUNNEL:
74 if (x->outer_mode.family == AF_INET)
75 return __xfrm_mode_tunnel_prep(x, skb,
76 sizeof(struct iphdr));
77 if (x->outer_mode.family == AF_INET6)
78 return __xfrm_mode_tunnel_prep(x, skb,
79 sizeof(struct ipv6hdr));
80 break;
81 case XFRM_MODE_TRANSPORT:
82 if (x->outer_mode.family == AF_INET)
83 return __xfrm_transport_prep(x, skb,
84 sizeof(struct iphdr));
85 if (x->outer_mode.family == AF_INET6)
86 return __xfrm_transport_prep(x, skb,
87 sizeof(struct ipv6hdr));
88 break;
89 case XFRM_MODE_BEET:
90 if (x->outer_mode.family == AF_INET)
91 return __xfrm_mode_beet_prep(x, skb,
92 sizeof(struct iphdr));
93 if (x->outer_mode.family == AF_INET6)
94 return __xfrm_mode_beet_prep(x, skb,
95 sizeof(struct ipv6hdr));
96 break;
97 case XFRM_MODE_ROUTEOPTIMIZATION:
98 case XFRM_MODE_IN_TRIGGER:
99 break;
100 }
101 }
102
xmit_xfrm_check_overflow(struct sk_buff * skb)103 static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb)
104 {
105 struct xfrm_offload *xo = xfrm_offload(skb);
106 __u32 seq = xo->seq.low;
107
108 seq += skb_shinfo(skb)->gso_segs;
109 if (unlikely(seq < xo->seq.low))
110 return true;
111
112 return false;
113 }
114
validate_xmit_xfrm(struct sk_buff * skb,netdev_features_t features,bool * again)115 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
116 {
117 int err;
118 unsigned long flags;
119 struct xfrm_state *x;
120 struct softnet_data *sd;
121 struct sk_buff *skb2, *nskb, *pskb = NULL;
122 netdev_features_t esp_features = features;
123 struct xfrm_offload *xo = xfrm_offload(skb);
124 struct net_device *dev = skb->dev;
125 struct sec_path *sp;
126
127 if (!xo || (xo->flags & XFRM_XMIT))
128 return skb;
129
130 if (!(features & NETIF_F_HW_ESP))
131 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
132
133 sp = skb_sec_path(skb);
134 x = sp->xvec[sp->len - 1];
135 if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
136 return skb;
137
138 /* The packet was sent to HW IPsec packet offload engine,
139 * but to wrong device. Drop the packet, so it won't skip
140 * XFRM stack.
141 */
142 if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) {
143 kfree_skb(skb);
144 dev_core_stats_tx_dropped_inc(dev);
145 return NULL;
146 }
147
148 local_irq_save(flags);
149 sd = this_cpu_ptr(&softnet_data);
150 err = !skb_queue_empty(&sd->xfrm_backlog);
151 local_irq_restore(flags);
152
153 if (err) {
154 *again = true;
155 return skb;
156 }
157
158 if (skb_is_gso(skb) && unlikely(xmit_xfrm_check_overflow(skb))) {
159 struct sk_buff *segs;
160
161 /* Packet got rerouted, fixup features and segment it. */
162 esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP);
163
164 segs = skb_gso_segment(skb, esp_features);
165 if (IS_ERR(segs)) {
166 kfree_skb(skb);
167 dev_core_stats_tx_dropped_inc(dev);
168 return NULL;
169 } else {
170 consume_skb(skb);
171 skb = segs;
172 }
173 }
174
175 if (!skb->next) {
176 esp_features |= skb->dev->gso_partial_features;
177 xfrm_outer_mode_prep(x, skb);
178
179 xo->flags |= XFRM_DEV_RESUME;
180
181 err = x->type_offload->xmit(x, skb, esp_features);
182 if (err) {
183 if (err == -EINPROGRESS)
184 return NULL;
185
186 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
187 kfree_skb(skb);
188 return NULL;
189 }
190
191 skb_push(skb, skb->data - skb_mac_header(skb));
192
193 return skb;
194 }
195
196 skb_list_walk_safe(skb, skb2, nskb) {
197 esp_features |= skb->dev->gso_partial_features;
198 skb_mark_not_on_list(skb2);
199
200 xo = xfrm_offload(skb2);
201 xo->flags |= XFRM_DEV_RESUME;
202
203 xfrm_outer_mode_prep(x, skb2);
204
205 err = x->type_offload->xmit(x, skb2, esp_features);
206 if (!err) {
207 skb2->next = nskb;
208 } else if (err != -EINPROGRESS) {
209 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
210 skb2->next = nskb;
211 kfree_skb_list(skb2);
212 return NULL;
213 } else {
214 if (skb == skb2)
215 skb = nskb;
216 else
217 pskb->next = nskb;
218
219 continue;
220 }
221
222 skb_push(skb2, skb2->data - skb_mac_header(skb2));
223 pskb = skb2;
224 }
225
226 return skb;
227 }
228 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
229
xfrm_dev_state_add(struct net * net,struct xfrm_state * x,struct xfrm_user_offload * xuo,struct netlink_ext_ack * extack)230 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
231 struct xfrm_user_offload *xuo,
232 struct netlink_ext_ack *extack)
233 {
234 int err;
235 struct dst_entry *dst;
236 struct net_device *dev;
237 struct xfrm_dev_offload *xso = &x->xso;
238 xfrm_address_t *saddr;
239 xfrm_address_t *daddr;
240 bool is_packet_offload;
241
242 if (xuo->flags &
243 ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) {
244 NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
245 return -EINVAL;
246 }
247
248 if ((xuo->flags & XFRM_OFFLOAD_INBOUND && x->dir == XFRM_SA_DIR_OUT) ||
249 (!(xuo->flags & XFRM_OFFLOAD_INBOUND) && x->dir == XFRM_SA_DIR_IN)) {
250 NL_SET_ERR_MSG(extack, "Mismatched SA and offload direction");
251 return -EINVAL;
252 }
253
254 if (xuo->flags & XFRM_OFFLOAD_INBOUND && x->if_id) {
255 NL_SET_ERR_MSG(extack, "XFRM if_id is not supported in RX path");
256 return -EINVAL;
257 }
258
259 is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET;
260
261 /* We don't yet support TFC padding. */
262 if (x->tfcpad) {
263 NL_SET_ERR_MSG(extack, "TFC padding can't be offloaded");
264 return -EINVAL;
265 }
266
267 dev = dev_get_by_index(net, xuo->ifindex);
268 if (!dev) {
269 struct xfrm_dst_lookup_params params;
270
271 if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
272 saddr = &x->props.saddr;
273 daddr = &x->id.daddr;
274 } else {
275 saddr = &x->id.daddr;
276 daddr = &x->props.saddr;
277 }
278
279 memset(¶ms, 0, sizeof(params));
280 params.net = net;
281 params.saddr = saddr;
282 params.daddr = daddr;
283 params.mark = xfrm_smark_get(0, x);
284 dst = __xfrm_dst_lookup(x->props.family, ¶ms);
285 if (IS_ERR(dst))
286 return (is_packet_offload) ? -EINVAL : 0;
287
288 dev = dst->dev;
289
290 dev_hold(dev);
291 dst_release(dst);
292 }
293
294 if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
295 xso->dev = NULL;
296 dev_put(dev);
297 return (is_packet_offload) ? -EINVAL : 0;
298 }
299
300 if (!is_packet_offload && x->props.flags & XFRM_STATE_ESN &&
301 !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
302 NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN");
303 xso->dev = NULL;
304 dev_put(dev);
305 return -EINVAL;
306 }
307
308 if (!x->type_offload) {
309 NL_SET_ERR_MSG(extack, "Type doesn't support offload");
310 dev_put(dev);
311 return -EINVAL;
312 }
313
314 xso->dev = dev;
315 netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
316
317 if (xuo->flags & XFRM_OFFLOAD_INBOUND)
318 xso->dir = XFRM_DEV_OFFLOAD_IN;
319 else
320 xso->dir = XFRM_DEV_OFFLOAD_OUT;
321
322 if (is_packet_offload)
323 xso->type = XFRM_DEV_OFFLOAD_PACKET;
324 else
325 xso->type = XFRM_DEV_OFFLOAD_CRYPTO;
326
327 err = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, extack);
328 if (err) {
329 xso->dev = NULL;
330 xso->dir = 0;
331 netdev_put(dev, &xso->dev_tracker);
332 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
333
334 xfrm_unset_type_offload(x);
335 /* User explicitly requested packet offload mode and configured
336 * policy in addition to the XFRM state. So be civil to users,
337 * and return an error instead of taking fallback path.
338 */
339 if ((err != -EOPNOTSUPP && !is_packet_offload) || is_packet_offload) {
340 NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this state");
341 return err;
342 }
343 }
344
345 return 0;
346 }
347 EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
348
xfrm_dev_policy_add(struct net * net,struct xfrm_policy * xp,struct xfrm_user_offload * xuo,u8 dir,struct netlink_ext_ack * extack)349 int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
350 struct xfrm_user_offload *xuo, u8 dir,
351 struct netlink_ext_ack *extack)
352 {
353 struct xfrm_dev_offload *xdo = &xp->xdo;
354 struct net_device *dev;
355 int err;
356
357 if (!xuo->flags || xuo->flags & ~XFRM_OFFLOAD_PACKET) {
358 /* We support only packet offload mode and it means
359 * that user must set XFRM_OFFLOAD_PACKET bit.
360 */
361 NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
362 return -EINVAL;
363 }
364
365 dev = dev_get_by_index(net, xuo->ifindex);
366 if (!dev)
367 return -EINVAL;
368
369 if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_policy_add) {
370 xdo->dev = NULL;
371 dev_put(dev);
372 NL_SET_ERR_MSG(extack, "Policy offload is not supported");
373 return -EINVAL;
374 }
375
376 xdo->dev = dev;
377 netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC);
378 xdo->type = XFRM_DEV_OFFLOAD_PACKET;
379 switch (dir) {
380 case XFRM_POLICY_IN:
381 xdo->dir = XFRM_DEV_OFFLOAD_IN;
382 break;
383 case XFRM_POLICY_OUT:
384 xdo->dir = XFRM_DEV_OFFLOAD_OUT;
385 break;
386 case XFRM_POLICY_FWD:
387 xdo->dir = XFRM_DEV_OFFLOAD_FWD;
388 break;
389 default:
390 xdo->dev = NULL;
391 netdev_put(dev, &xdo->dev_tracker);
392 NL_SET_ERR_MSG(extack, "Unrecognized offload direction");
393 return -EINVAL;
394 }
395
396 err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack);
397 if (err) {
398 xdo->dev = NULL;
399 xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
400 xdo->dir = 0;
401 netdev_put(dev, &xdo->dev_tracker);
402 NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this policy");
403 return err;
404 }
405
406 return 0;
407 }
408 EXPORT_SYMBOL_GPL(xfrm_dev_policy_add);
409
xfrm_dev_offload_ok(struct sk_buff * skb,struct xfrm_state * x)410 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
411 {
412 int mtu;
413 struct dst_entry *dst = skb_dst(skb);
414 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
415 struct net_device *dev = x->xso.dev;
416 bool check_tunnel_size;
417
418 if (x->xso.type == XFRM_DEV_OFFLOAD_UNSPECIFIED)
419 return false;
420
421 if ((dev == xfrm_dst_path(dst)->dev) && !xdst->child->xfrm) {
422 mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
423 if (skb->len <= mtu)
424 goto ok;
425
426 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
427 goto ok;
428 }
429
430 return false;
431
432 ok:
433 check_tunnel_size = x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
434 x->props.mode == XFRM_MODE_TUNNEL;
435 switch (x->props.family) {
436 case AF_INET:
437 /* Check for IPv4 options */
438 if (ip_hdr(skb)->ihl != 5)
439 return false;
440 if (check_tunnel_size && xfrm4_tunnel_check_size(skb))
441 return false;
442 break;
443 case AF_INET6:
444 /* Check for IPv6 extensions */
445 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
446 return false;
447 if (check_tunnel_size && xfrm6_tunnel_check_size(skb))
448 return false;
449 break;
450 default:
451 break;
452 }
453
454 if (dev->xfrmdev_ops->xdo_dev_offload_ok)
455 return dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
456
457 return true;
458 }
459 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
460
xfrm_dev_resume(struct sk_buff * skb)461 void xfrm_dev_resume(struct sk_buff *skb)
462 {
463 struct net_device *dev = skb->dev;
464 int ret = NETDEV_TX_BUSY;
465 struct netdev_queue *txq;
466 struct softnet_data *sd;
467 unsigned long flags;
468
469 rcu_read_lock();
470 txq = netdev_core_pick_tx(dev, skb, NULL);
471
472 HARD_TX_LOCK(dev, txq, smp_processor_id());
473 if (!netif_xmit_frozen_or_stopped(txq))
474 skb = dev_hard_start_xmit(skb, dev, txq, &ret);
475 HARD_TX_UNLOCK(dev, txq);
476
477 if (!dev_xmit_complete(ret)) {
478 local_irq_save(flags);
479 sd = this_cpu_ptr(&softnet_data);
480 skb_queue_tail(&sd->xfrm_backlog, skb);
481 raise_softirq_irqoff(NET_TX_SOFTIRQ);
482 local_irq_restore(flags);
483 }
484 rcu_read_unlock();
485 }
486 EXPORT_SYMBOL_GPL(xfrm_dev_resume);
487
xfrm_dev_backlog(struct softnet_data * sd)488 void xfrm_dev_backlog(struct softnet_data *sd)
489 {
490 struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
491 struct sk_buff_head list;
492 struct sk_buff *skb;
493
494 if (skb_queue_empty(xfrm_backlog))
495 return;
496
497 __skb_queue_head_init(&list);
498
499 spin_lock(&xfrm_backlog->lock);
500 skb_queue_splice_init(xfrm_backlog, &list);
501 spin_unlock(&xfrm_backlog->lock);
502
503 while (!skb_queue_empty(&list)) {
504 skb = __skb_dequeue(&list);
505 xfrm_dev_resume(skb);
506 }
507
508 }
509 #endif
510
xfrm_api_check(struct net_device * dev)511 static int xfrm_api_check(struct net_device *dev)
512 {
513 #ifdef CONFIG_XFRM_OFFLOAD
514 if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
515 !(dev->features & NETIF_F_HW_ESP))
516 return NOTIFY_BAD;
517
518 if ((dev->features & NETIF_F_HW_ESP) &&
519 (!(dev->xfrmdev_ops &&
520 dev->xfrmdev_ops->xdo_dev_state_add &&
521 dev->xfrmdev_ops->xdo_dev_state_delete)))
522 return NOTIFY_BAD;
523 #else
524 if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
525 return NOTIFY_BAD;
526 #endif
527
528 return NOTIFY_DONE;
529 }
530
xfrm_dev_down(struct net_device * dev)531 static int xfrm_dev_down(struct net_device *dev)
532 {
533 if (dev->features & NETIF_F_HW_ESP) {
534 xfrm_dev_state_flush(dev_net(dev), dev, true);
535 xfrm_dev_policy_flush(dev_net(dev), dev, true);
536 }
537
538 return NOTIFY_DONE;
539 }
540
xfrm_dev_event(struct notifier_block * this,unsigned long event,void * ptr)541 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
542 {
543 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
544
545 switch (event) {
546 case NETDEV_REGISTER:
547 return xfrm_api_check(dev);
548
549 case NETDEV_FEAT_CHANGE:
550 return xfrm_api_check(dev);
551
552 case NETDEV_DOWN:
553 case NETDEV_UNREGISTER:
554 return xfrm_dev_down(dev);
555 }
556 return NOTIFY_DONE;
557 }
558
559 static struct notifier_block xfrm_dev_notifier = {
560 .notifier_call = xfrm_dev_event,
561 };
562
xfrm_dev_init(void)563 void __init xfrm_dev_init(void)
564 {
565 register_netdevice_notifier(&xfrm_dev_notifier);
566 }
567