1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/module.h>
4 #include <linux/init.h>
5 #include <linux/etherdevice.h>
6 #include <linux/netlink.h>
7 #include <linux/netfilter.h>
8 #include <linux/spinlock.h>
9 #include <linux/netfilter/nf_conntrack_common.h>
10 #include <linux/netfilter/nf_tables.h>
11 #include <net/ip.h>
12 #include <net/inet_dscp.h>
13 #include <net/netfilter/nf_tables.h>
14 #include <net/netfilter/nf_tables_core.h>
15 #include <net/netfilter/nf_conntrack_core.h>
16 #include <net/netfilter/nf_conntrack_extend.h>
17 #include <net/netfilter/nf_flow_table.h>
18
nft_xmit_type(struct dst_entry * dst)19 static enum flow_offload_xmit_type nft_xmit_type(struct dst_entry *dst)
20 {
21 if (dst_xfrm(dst))
22 return FLOW_OFFLOAD_XMIT_XFRM;
23
24 return FLOW_OFFLOAD_XMIT_NEIGH;
25 }
26
nft_default_forward_path(struct nf_flow_route * route,struct dst_entry * dst_cache,enum ip_conntrack_dir dir)27 static void nft_default_forward_path(struct nf_flow_route *route,
28 struct dst_entry *dst_cache,
29 enum ip_conntrack_dir dir)
30 {
31 route->tuple[!dir].in.ifindex = dst_cache->dev->ifindex;
32 route->tuple[dir].dst = dst_cache;
33 route->tuple[dir].xmit_type = nft_xmit_type(dst_cache);
34 }
35
nft_is_valid_ether_device(const struct net_device * dev)36 static bool nft_is_valid_ether_device(const struct net_device *dev)
37 {
38 if (!dev || (dev->flags & IFF_LOOPBACK) || dev->type != ARPHRD_ETHER ||
39 dev->addr_len != ETH_ALEN || !is_valid_ether_addr(dev->dev_addr))
40 return false;
41
42 return true;
43 }
44
nft_dev_fill_forward_path(const struct dst_entry * dst_cache,const struct nf_conn * ct,enum ip_conntrack_dir dir,u8 * ha,__be16 ether_type,struct net_device_path_stack * stack)45 static int nft_dev_fill_forward_path(const struct dst_entry *dst_cache,
46 const struct nf_conn *ct,
47 enum ip_conntrack_dir dir,
48 u8 *ha, __be16 ether_type,
49 struct net_device_path_stack *stack)
50 {
51 const void *daddr = &ct->tuplehash[!dir].tuple.src.u3;
52 struct net_device *dev = dst_cache->dev;
53 struct net_device_path_ctx ctx = {
54 .dev = dev,
55 .ether_type = ether_type,
56 };
57 struct neighbour *n;
58 u8 nud_state;
59
60 if (!nft_is_valid_ether_device(dev)) {
61 eth_zero_addr(ha);
62 goto out;
63 }
64
65 n = dst_neigh_lookup(dst_cache, daddr);
66 if (!n)
67 return -1;
68
69 read_lock_bh(&n->lock);
70 nud_state = n->nud_state;
71 ether_addr_copy(ha, n->ha);
72 read_unlock_bh(&n->lock);
73 neigh_release(n);
74
75 if (!(nud_state & NUD_VALID))
76 return -1;
77
78 out:
79 ether_addr_copy(ctx.daddr, ha);
80
81 return dev_fill_forward_path(&ctx, stack);
82 }
83
84 struct nft_forward_info {
85 const struct net_device *dev;
86 struct id {
87 __u16 id;
88 __be16 proto;
89 } encap[NF_FLOW_TABLE_ENCAP_MAX];
90 u8 num_encaps;
91 struct flow_offload_tunnel tun;
92 struct dst_entry *tun_dst;
93 u8 num_tuns;
94 u8 ingress_vlans;
95 u8 h_source[ETH_ALEN];
96 u8 h_dest[ETH_ALEN];
97 bool needs_gso_segment;
98 enum flow_offload_xmit_type xmit_type;
99 };
100
101 static bool nft_flowtable_find_dev(const struct net_device *dev,
102 struct nft_flowtable *ft);
103
nft_dev_path_info(struct net_device_path_stack * stack,struct nft_forward_info * info,unsigned char * ha,struct nft_flowtable * ft)104 static int nft_dev_path_info(struct net_device_path_stack *stack,
105 struct nft_forward_info *info,
106 unsigned char *ha, struct nft_flowtable *ft)
107 {
108 const struct net_device_path *path;
109 int i;
110
111 memcpy(info->h_dest, ha, ETH_ALEN);
112
113 for (i = 0; i < stack->num_paths; i++) {
114 path = &stack->path[i];
115 switch (path->type) {
116 case DEV_PATH_ETHERNET:
117 case DEV_PATH_DSA:
118 case DEV_PATH_VLAN:
119 case DEV_PATH_PPPOE:
120 case DEV_PATH_TUN:
121 info->dev = path->dev;
122 if (is_zero_ether_addr(info->h_source))
123 memcpy(info->h_source, path->dev->dev_addr, ETH_ALEN);
124
125 if (path->type == DEV_PATH_ETHERNET ||
126 path->type == DEV_PATH_DSA)
127 break;
128
129 /* DEV_PATH_VLAN, DEV_PATH_PPPOE and DEV_PATH_TUN */
130 if (path->type == DEV_PATH_TUN) {
131 if (info->num_tuns)
132 goto err_out;
133
134 info->tun.src_v6 = path->tun.src_v6;
135 info->tun.dst_v6 = path->tun.dst_v6;
136 info->tun.inner_proto = path->tun.inner_proto;
137 info->tun_dst = path->tun.dst;
138 info->num_tuns++;
139 } else {
140 if (info->num_encaps >= NF_FLOW_TABLE_ENCAP_MAX)
141 goto err_out;
142
143 info->encap[info->num_encaps].id =
144 path->encap.id;
145 info->encap[info->num_encaps].proto =
146 path->encap.proto;
147 info->num_encaps++;
148 }
149 if (path->type == DEV_PATH_PPPOE) {
150 memcpy(info->h_dest, path->encap.h_dest, ETH_ALEN);
151 info->xmit_type = FLOW_OFFLOAD_XMIT_DIRECT;
152 info->needs_gso_segment = 1;
153 }
154 break;
155 case DEV_PATH_BRIDGE:
156 if (is_zero_ether_addr(info->h_source))
157 memcpy(info->h_source, path->dev->dev_addr, ETH_ALEN);
158
159 switch (path->bridge.vlan_mode) {
160 case DEV_PATH_BR_VLAN_UNTAG_HW:
161 if (info->num_encaps == 0)
162 goto err_out;
163
164 info->ingress_vlans |= BIT(info->num_encaps - 1);
165 break;
166 case DEV_PATH_BR_VLAN_TAG:
167 if (info->num_encaps >= NF_FLOW_TABLE_ENCAP_MAX)
168 goto err_out;
169
170 info->encap[info->num_encaps].id = path->bridge.vlan_id;
171 info->encap[info->num_encaps].proto = path->bridge.vlan_proto;
172 info->num_encaps++;
173 break;
174 case DEV_PATH_BR_VLAN_UNTAG:
175 if (info->num_encaps == 0)
176 goto err_out;
177
178 info->num_encaps--;
179 break;
180 case DEV_PATH_BR_VLAN_KEEP:
181 break;
182 }
183 info->xmit_type = FLOW_OFFLOAD_XMIT_DIRECT;
184 break;
185 default:
186 goto err_out;
187 }
188 }
189
190 if (nf_flowtable_hw_offload(&ft->data) &&
191 nft_is_valid_ether_device(info->dev))
192 info->xmit_type = FLOW_OFFLOAD_XMIT_DIRECT;
193
194 if (!nft_flowtable_find_dev(info->dev, ft))
195 goto err_out;
196
197 return 0;
198 err_out:
199 dev_fill_forward_path_release(stack);
200
201 return -1;
202 }
203
nft_flowtable_find_dev(const struct net_device * dev,struct nft_flowtable * ft)204 static bool nft_flowtable_find_dev(const struct net_device *dev,
205 struct nft_flowtable *ft)
206 {
207 struct nft_hook *hook;
208 bool found = false;
209
210 list_for_each_entry_rcu(hook, &ft->hook_list, list) {
211 if (!nft_hook_find_ops_rcu(hook, dev))
212 continue;
213
214 found = true;
215 break;
216 }
217
218 return found;
219 }
220
nft_dev_forward_path(const struct nft_pktinfo * pkt,struct nf_flow_route * route,const struct nf_conn * ct,enum ip_conntrack_dir dir,struct nft_flowtable * ft)221 static int nft_dev_forward_path(const struct nft_pktinfo *pkt,
222 struct nf_flow_route *route,
223 const struct nf_conn *ct,
224 enum ip_conntrack_dir dir,
225 struct nft_flowtable *ft)
226 {
227 const struct dst_entry *dst = route->tuple[dir].dst;
228 struct net_device_path_stack stack;
229 struct nft_forward_info info = {};
230 unsigned char ha[ETH_ALEN];
231 int i;
232
233 if (nft_dev_fill_forward_path(dst, ct, dir, ha, pkt->ethertype, &stack) < 0 ||
234 nft_dev_path_info(&stack, &info, ha, ft) < 0)
235 return -ENOENT;
236
237 route->tuple[!dir].in.ifindex = info.dev->ifindex;
238 route->tuple[dir].out.ifindex = info.dev->ifindex;
239
240 for (i = 0; i < info.num_encaps; i++) {
241 route->tuple[!dir].in.encap[i].id = info.encap[i].id;
242 route->tuple[!dir].in.encap[i].proto = info.encap[i].proto;
243 }
244
245 if (info.num_tuns) {
246 route->tuple[!dir].in.tun.src_v6 = info.tun.dst_v6;
247 route->tuple[!dir].in.tun.dst_v6 = info.tun.src_v6;
248 route->tuple[!dir].in.tun.inner_proto = info.tun.inner_proto;
249 route->tuple[!dir].in.num_tuns = info.num_tuns;
250 dst_release(route->tuple[dir].dst);
251 route->tuple[dir].dst = info.tun_dst;
252 }
253
254 route->tuple[!dir].in.num_encaps = info.num_encaps;
255 route->tuple[!dir].in.ingress_vlans = info.ingress_vlans;
256
257 if (info.xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) {
258 memcpy(route->tuple[dir].out.h_source, info.h_source, ETH_ALEN);
259 memcpy(route->tuple[dir].out.h_dest, info.h_dest, ETH_ALEN);
260 route->tuple[dir].xmit_type = info.xmit_type;
261 }
262 route->tuple[dir].out.needs_gso_segment = info.needs_gso_segment;
263
264 return 0;
265 }
266
nft_flow_route(const struct nft_pktinfo * pkt,const struct nf_conn * ct,struct nf_flow_route * route,enum ip_conntrack_dir dir,struct nft_flowtable * ft)267 int nft_flow_route(const struct nft_pktinfo *pkt, const struct nf_conn *ct,
268 struct nf_flow_route *route, enum ip_conntrack_dir dir,
269 struct nft_flowtable *ft)
270 {
271 struct dst_entry *this_dst = skb_dst(pkt->skb);
272 struct dst_entry *other_dst = NULL;
273 struct flowi fl;
274
275 memset(&fl, 0, sizeof(fl));
276 switch (nft_pf(pkt)) {
277 case NFPROTO_IPV4:
278 fl.u.ip4.daddr = ct->tuplehash[dir].tuple.src.u3.ip;
279 fl.u.ip4.saddr = ct->tuplehash[!dir].tuple.src.u3.ip;
280 fl.u.ip4.flowi4_oif = nft_in(pkt)->ifindex;
281 fl.u.ip4.flowi4_iif = this_dst->dev->ifindex;
282 fl.u.ip4.flowi4_dscp = ip4h_dscp(ip_hdr(pkt->skb));
283 fl.u.ip4.flowi4_mark = pkt->skb->mark;
284 fl.u.ip4.flowi4_flags = FLOWI_FLAG_ANYSRC;
285 break;
286 case NFPROTO_IPV6:
287 fl.u.ip6.daddr = ct->tuplehash[dir].tuple.src.u3.in6;
288 fl.u.ip6.saddr = ct->tuplehash[!dir].tuple.src.u3.in6;
289 fl.u.ip6.flowi6_oif = nft_in(pkt)->ifindex;
290 fl.u.ip6.flowi6_iif = this_dst->dev->ifindex;
291 fl.u.ip6.flowlabel = ip6_flowinfo(ipv6_hdr(pkt->skb));
292 fl.u.ip6.flowi6_mark = pkt->skb->mark;
293 fl.u.ip6.flowi6_flags = FLOWI_FLAG_ANYSRC;
294 break;
295 }
296
297 if (!dst_hold_safe(this_dst))
298 return -ENOENT;
299
300 nf_route(nft_net(pkt), &other_dst, &fl, false, nft_pf(pkt));
301 if (!other_dst) {
302 dst_release(this_dst);
303 return -ENOENT;
304 }
305
306 nft_default_forward_path(route, this_dst, dir);
307 nft_default_forward_path(route, other_dst, !dir);
308
309 if (route->tuple[dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH &&
310 nft_dev_forward_path(pkt, route, ct, dir, ft) < 0)
311 goto err_dst_release;
312
313 if (route->tuple[!dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH &&
314 nft_dev_forward_path(pkt, route, ct, !dir, ft) < 0)
315 goto err_dst_release;
316
317 return 0;
318
319 err_dst_release:
320 dst_release(route->tuple[dir].dst);
321 dst_release(route->tuple[!dir].dst);
322 return -ENOENT;
323 }
324 EXPORT_SYMBOL_GPL(nft_flow_route);
325