1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2007-2014 Nicira, Inc.
4 */
5
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/init.h>
9 #include <linux/module.h>
10 #include <linux/if_arp.h>
11 #include <linux/if_vlan.h>
12 #include <linux/in.h>
13 #include <linux/ip.h>
14 #include <linux/jhash.h>
15 #include <linux/delay.h>
16 #include <linux/time.h>
17 #include <linux/etherdevice.h>
18 #include <linux/kernel.h>
19 #include <linux/kthread.h>
20 #include <linux/mutex.h>
21 #include <linux/percpu.h>
22 #include <linux/rcupdate.h>
23 #include <linux/tcp.h>
24 #include <linux/udp.h>
25 #include <linux/ethtool.h>
26 #include <linux/wait.h>
27 #include <asm/div64.h>
28 #include <linux/highmem.h>
29 #include <linux/netfilter_bridge.h>
30 #include <linux/netfilter_ipv4.h>
31 #include <linux/inetdevice.h>
32 #include <linux/list.h>
33 #include <linux/openvswitch.h>
34 #include <linux/rculist.h>
35 #include <linux/dmi.h>
36 #include <net/genetlink.h>
37 #include <net/gso.h>
38 #include <net/net_namespace.h>
39 #include <net/netns/generic.h>
40 #include <net/pkt_cls.h>
41
42 #include "datapath.h"
43 #include "drop.h"
44 #include "flow.h"
45 #include "flow_table.h"
46 #include "flow_netlink.h"
47 #include "meter.h"
48 #include "openvswitch_trace.h"
49 #include "vport-internal_dev.h"
50 #include "vport-netdev.h"
51
52 unsigned int ovs_net_id __read_mostly;
53
54 static struct genl_family dp_packet_genl_family;
55 static struct genl_family dp_flow_genl_family;
56 static struct genl_family dp_datapath_genl_family;
57
58 static const struct nla_policy flow_policy[];
59
60 static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
61 .name = OVS_FLOW_MCGROUP,
62 };
63
64 static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
65 .name = OVS_DATAPATH_MCGROUP,
66 };
67
68 static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
69 .name = OVS_VPORT_MCGROUP,
70 };
71
72 /* Check if need to build a reply message.
73 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
ovs_must_notify(struct genl_family * family,struct genl_info * info,unsigned int group)74 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
75 unsigned int group)
76 {
77 return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
78 genl_has_listeners(family, genl_info_net(info), group);
79 }
80
ovs_notify(struct genl_family * family,struct sk_buff * skb,struct genl_info * info)81 static void ovs_notify(struct genl_family *family,
82 struct sk_buff *skb, struct genl_info *info)
83 {
84 genl_notify(family, skb, info, 0, GFP_KERNEL);
85 }
86
87 /**
88 * DOC: Locking:
89 *
90 * All writes e.g. Writes to device state (add/remove datapath, port, set
91 * operations on vports, etc.), Writes to other state (flow table
92 * modifications, set miscellaneous datapath parameters, etc.) are protected
93 * by ovs_lock.
94 *
95 * Reads are protected by RCU.
96 *
97 * There are a few special cases (mostly stats) that have their own
98 * synchronization but they nest under all of above and don't interact with
99 * each other.
100 *
101 * The RTNL lock nests inside ovs_mutex.
102 */
103
104 static DEFINE_MUTEX(ovs_mutex);
105
ovs_lock(void)106 void ovs_lock(void)
107 {
108 mutex_lock(&ovs_mutex);
109 }
110
ovs_unlock(void)111 void ovs_unlock(void)
112 {
113 mutex_unlock(&ovs_mutex);
114 }
115
116 #ifdef CONFIG_LOCKDEP
lockdep_ovsl_is_held(void)117 int lockdep_ovsl_is_held(void)
118 {
119 if (debug_locks)
120 return lockdep_is_held(&ovs_mutex);
121 else
122 return 1;
123 }
124 #endif
125
126 static struct vport *new_vport(const struct vport_parms *);
127 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
128 const struct sw_flow_key *,
129 const struct dp_upcall_info *,
130 uint32_t cutlen);
131 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
132 const struct sw_flow_key *,
133 const struct dp_upcall_info *,
134 uint32_t cutlen);
135
136 static void ovs_dp_masks_rebalance(struct work_struct *work);
137
138 static int ovs_dp_set_upcall_portids(struct datapath *, const struct nlattr *);
139
140 /* Must be called with rcu_read_lock or ovs_mutex. */
ovs_dp_name(const struct datapath * dp)141 const char *ovs_dp_name(const struct datapath *dp)
142 {
143 struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
144 return ovs_vport_name(vport);
145 }
146
get_dpifindex(const struct datapath * dp)147 static int get_dpifindex(const struct datapath *dp)
148 {
149 struct vport *local;
150 int ifindex;
151
152 rcu_read_lock();
153
154 local = ovs_vport_rcu(dp, OVSP_LOCAL);
155 if (local)
156 ifindex = local->dev->ifindex;
157 else
158 ifindex = 0;
159
160 rcu_read_unlock();
161
162 return ifindex;
163 }
164
destroy_dp_rcu(struct rcu_head * rcu)165 static void destroy_dp_rcu(struct rcu_head *rcu)
166 {
167 struct datapath *dp = container_of(rcu, struct datapath, rcu);
168
169 ovs_flow_tbl_destroy(&dp->table);
170 free_percpu(dp->stats_percpu);
171 kfree(dp->ports);
172 ovs_meters_exit(dp);
173 kfree(rcu_dereference_raw(dp->upcall_portids));
174 kfree(dp);
175 }
176
vport_hash_bucket(const struct datapath * dp,u16 port_no)177 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
178 u16 port_no)
179 {
180 return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
181 }
182
183 /* Called with ovs_mutex or RCU read lock. */
ovs_lookup_vport(const struct datapath * dp,u16 port_no)184 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
185 {
186 struct vport *vport;
187 struct hlist_head *head;
188
189 head = vport_hash_bucket(dp, port_no);
190 hlist_for_each_entry_rcu(vport, head, dp_hash_node,
191 lockdep_ovsl_is_held()) {
192 if (vport->port_no == port_no)
193 return vport;
194 }
195 return NULL;
196 }
197
198 /* Called with ovs_mutex. */
new_vport(const struct vport_parms * parms)199 static struct vport *new_vport(const struct vport_parms *parms)
200 {
201 struct vport *vport;
202
203 vport = ovs_vport_add(parms);
204 if (!IS_ERR(vport)) {
205 struct datapath *dp = parms->dp;
206 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
207
208 hlist_add_head_rcu(&vport->dp_hash_node, head);
209 }
210 return vport;
211 }
212
ovs_vport_update_upcall_stats(struct sk_buff * skb,const struct dp_upcall_info * upcall_info,bool upcall_result)213 static void ovs_vport_update_upcall_stats(struct sk_buff *skb,
214 const struct dp_upcall_info *upcall_info,
215 bool upcall_result)
216 {
217 struct vport *p = OVS_CB(skb)->input_vport;
218 struct vport_upcall_stats_percpu *stats;
219
220 if (upcall_info->cmd != OVS_PACKET_CMD_MISS &&
221 upcall_info->cmd != OVS_PACKET_CMD_ACTION)
222 return;
223
224 stats = this_cpu_ptr(p->upcall_stats);
225 u64_stats_update_begin(&stats->syncp);
226 if (upcall_result)
227 u64_stats_inc(&stats->n_success);
228 else
229 u64_stats_inc(&stats->n_fail);
230 u64_stats_update_end(&stats->syncp);
231 }
232
ovs_dp_detach_port(struct vport * p)233 void ovs_dp_detach_port(struct vport *p)
234 {
235 ASSERT_OVSL();
236
237 /* First drop references to device. */
238 hlist_del_rcu(&p->dp_hash_node);
239
240 /* Then destroy it. */
241 ovs_vport_del(p);
242 }
243
244 /* Must be called with rcu_read_lock. */
ovs_dp_process_packet(struct sk_buff * skb,struct sw_flow_key * key)245 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
246 {
247 struct ovs_pcpu_storage *ovs_pcpu = this_cpu_ptr(ovs_pcpu_storage);
248 const struct vport *p = OVS_CB(skb)->input_vport;
249 struct datapath *dp = p->dp;
250 struct sw_flow *flow;
251 struct sw_flow_actions *sf_acts;
252 struct dp_stats_percpu *stats;
253 bool ovs_pcpu_locked = false;
254 u64 *stats_counter;
255 u32 n_mask_hit;
256 u32 n_cache_hit;
257 int error;
258
259 stats = this_cpu_ptr(dp->stats_percpu);
260
261 /* Look up flow. */
262 flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
263 &n_mask_hit, &n_cache_hit);
264 if (unlikely(!flow)) {
265 struct dp_upcall_info upcall;
266
267 memset(&upcall, 0, sizeof(upcall));
268 upcall.cmd = OVS_PACKET_CMD_MISS;
269
270 if (OVS_CB(skb)->upcall_pid)
271 upcall.portid = OVS_CB(skb)->upcall_pid;
272 else if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU)
273 upcall.portid =
274 ovs_dp_get_upcall_portid(dp, smp_processor_id());
275 else
276 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
277
278 upcall.mru = OVS_CB(skb)->mru;
279 error = ovs_dp_upcall(dp, skb, key, &upcall, U32_MAX);
280 switch (error) {
281 case 0:
282 case -EAGAIN:
283 case -ERESTARTSYS:
284 case -EINTR:
285 consume_skb(skb);
286 break;
287 default:
288 kfree_skb(skb);
289 break;
290 }
291 stats_counter = &stats->n_missed;
292 goto out;
293 }
294
295 ovs_flow_stats_update(flow, key->tp.flags, skb);
296 sf_acts = rcu_dereference(flow->sf_acts);
297 /* This path can be invoked recursively: Use the current task to
298 * identify recursive invocation - the lock must be acquired only once.
299 * Even with disabled bottom halves this can be preempted on PREEMPT_RT.
300 * Limit the locking to RT to avoid assigning `owner' if it can be
301 * avoided.
302 */
303 if (IS_ENABLED(CONFIG_PREEMPT_RT) && ovs_pcpu->owner != current) {
304 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock);
305 ovs_pcpu->owner = current;
306 ovs_pcpu_locked = true;
307 }
308
309 error = ovs_execute_actions(dp, skb, sf_acts, key);
310 if (unlikely(error))
311 net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n",
312 ovs_dp_name(dp), error);
313 if (ovs_pcpu_locked) {
314 ovs_pcpu->owner = NULL;
315 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock);
316 }
317
318 stats_counter = &stats->n_hit;
319
320 out:
321 /* Update datapath statistics. */
322 u64_stats_update_begin(&stats->syncp);
323 (*stats_counter)++;
324 stats->n_mask_hit += n_mask_hit;
325 stats->n_cache_hit += n_cache_hit;
326 u64_stats_update_end(&stats->syncp);
327 }
328
ovs_dp_upcall(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)329 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
330 const struct sw_flow_key *key,
331 const struct dp_upcall_info *upcall_info,
332 uint32_t cutlen)
333 {
334 struct dp_stats_percpu *stats;
335 int err;
336
337 if (trace_ovs_dp_upcall_enabled())
338 trace_ovs_dp_upcall(dp, skb, key, upcall_info);
339
340 if (upcall_info->portid == 0) {
341 err = -ENOTCONN;
342 goto err;
343 }
344
345 if (!skb_is_gso(skb))
346 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
347 else
348 err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
349
350 ovs_vport_update_upcall_stats(skb, upcall_info, !err);
351 if (err)
352 goto err;
353
354 return 0;
355
356 err:
357 stats = this_cpu_ptr(dp->stats_percpu);
358
359 u64_stats_update_begin(&stats->syncp);
360 stats->n_lost++;
361 u64_stats_update_end(&stats->syncp);
362
363 return err;
364 }
365
queue_gso_packets(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)366 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
367 const struct sw_flow_key *key,
368 const struct dp_upcall_info *upcall_info,
369 uint32_t cutlen)
370 {
371 unsigned int gso_type = skb_shinfo(skb)->gso_type;
372 struct sw_flow_key later_key;
373 struct sk_buff *segs, *nskb;
374 int err;
375
376 BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET);
377 segs = __skb_gso_segment(skb, NETIF_F_SG, false);
378 if (IS_ERR(segs))
379 return PTR_ERR(segs);
380 if (segs == NULL)
381 return -EINVAL;
382
383 if (gso_type & SKB_GSO_UDP) {
384 /* The initial flow key extracted by ovs_flow_key_extract()
385 * in this case is for a first fragment, so we need to
386 * properly mark later fragments.
387 */
388 later_key = *key;
389 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
390 }
391
392 /* Queue all of the segments. */
393 skb_list_walk_safe(segs, skb, nskb) {
394 if (gso_type & SKB_GSO_UDP && skb != segs)
395 key = &later_key;
396
397 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
398 if (err)
399 break;
400
401 }
402
403 /* Free all of the segments. */
404 skb_list_walk_safe(segs, skb, nskb) {
405 if (err)
406 kfree_skb(skb);
407 else
408 consume_skb(skb);
409 }
410 return err;
411 }
412
upcall_msg_size(const struct dp_upcall_info * upcall_info,unsigned int hdrlen,int actions_attrlen)413 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
414 unsigned int hdrlen, int actions_attrlen)
415 {
416 size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
417 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
418 + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
419 + nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */
420 + nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */
421
422 /* OVS_PACKET_ATTR_USERDATA */
423 if (upcall_info->userdata)
424 size += NLA_ALIGN(upcall_info->userdata->nla_len);
425
426 /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
427 if (upcall_info->egress_tun_info)
428 size += nla_total_size(ovs_tun_key_attr_size());
429
430 /* OVS_PACKET_ATTR_ACTIONS */
431 if (upcall_info->actions_len)
432 size += nla_total_size(actions_attrlen);
433
434 /* OVS_PACKET_ATTR_MRU */
435 if (upcall_info->mru)
436 size += nla_total_size(sizeof(upcall_info->mru));
437
438 return size;
439 }
440
pad_packet(struct datapath * dp,struct sk_buff * skb)441 static void pad_packet(struct datapath *dp, struct sk_buff *skb)
442 {
443 if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
444 size_t plen = NLA_ALIGN(skb->len) - skb->len;
445
446 if (plen > 0)
447 skb_put_zero(skb, plen);
448 }
449 }
450
queue_userspace_packet(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)451 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
452 const struct sw_flow_key *key,
453 const struct dp_upcall_info *upcall_info,
454 uint32_t cutlen)
455 {
456 struct ovs_header *upcall;
457 struct sk_buff *nskb = NULL;
458 struct sk_buff *user_skb = NULL; /* to be queued to userspace */
459 struct nlattr *nla;
460 size_t msg_size;
461 size_t skb_len;
462 unsigned int hlen;
463 int err, dp_ifindex;
464 u64 hash;
465
466 dp_ifindex = get_dpifindex(dp);
467 if (!dp_ifindex)
468 return -ENODEV;
469
470 if (skb_vlan_tag_present(skb)) {
471 nskb = skb_clone(skb, GFP_ATOMIC);
472 if (!nskb)
473 return -ENOMEM;
474
475 nskb = __vlan_hwaccel_push_inside(nskb);
476 if (!nskb)
477 return -ENOMEM;
478
479 skb = nskb;
480 }
481
482 skb_len = min(skb->len, cutlen);
483 if (nla_attr_size(skb_len) > USHRT_MAX) {
484 err = -EFBIG;
485 goto out;
486 }
487
488 /* Complete checksum if needed */
489 if (skb->ip_summed == CHECKSUM_PARTIAL &&
490 (err = skb_csum_hwoffload_help(skb, 0)))
491 goto out;
492
493 /* Older versions of OVS user space enforce alignment of the last
494 * Netlink attribute to NLA_ALIGNTO which would require extensive
495 * padding logic. Only perform zerocopy if padding is not required.
496 */
497 if (dp->user_features & OVS_DP_F_UNALIGNED)
498 hlen = min(skb_zerocopy_headlen(skb), cutlen);
499 else
500 hlen = skb_len;
501
502 msg_size = upcall_msg_size(upcall_info, hlen,
503 OVS_CB(skb)->acts_origlen);
504 user_skb = genlmsg_new(msg_size, GFP_ATOMIC);
505 if (!user_skb) {
506 err = -ENOMEM;
507 goto out;
508 }
509
510 upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
511 0, upcall_info->cmd);
512 if (!upcall) {
513 err = -EINVAL;
514 goto out;
515 }
516 upcall->dp_ifindex = dp_ifindex;
517
518 err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
519 if (err)
520 goto out;
521
522 if (upcall_info->userdata)
523 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
524 nla_len(upcall_info->userdata),
525 nla_data(upcall_info->userdata));
526
527 if (upcall_info->egress_tun_info) {
528 nla = nla_nest_start_noflag(user_skb,
529 OVS_PACKET_ATTR_EGRESS_TUN_KEY);
530 if (!nla) {
531 err = -EMSGSIZE;
532 goto out;
533 }
534 err = ovs_nla_put_tunnel_info(user_skb,
535 upcall_info->egress_tun_info);
536 if (err)
537 goto out;
538
539 nla_nest_end(user_skb, nla);
540 }
541
542 if (upcall_info->actions_len) {
543 nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS);
544 if (!nla) {
545 err = -EMSGSIZE;
546 goto out;
547 }
548 err = ovs_nla_put_actions(upcall_info->actions,
549 upcall_info->actions_len,
550 user_skb);
551 if (!err)
552 nla_nest_end(user_skb, nla);
553 else
554 nla_nest_cancel(user_skb, nla);
555 }
556
557 /* Add OVS_PACKET_ATTR_MRU */
558 if (upcall_info->mru &&
559 nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) {
560 err = -ENOBUFS;
561 goto out;
562 }
563
564 /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
565 if (skb_len < skb->len &&
566 nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) {
567 err = -ENOBUFS;
568 goto out;
569 }
570
571 /* Add OVS_PACKET_ATTR_HASH */
572 hash = skb_get_hash_raw(skb);
573 if (skb->sw_hash)
574 hash |= OVS_PACKET_HASH_SW_BIT;
575
576 if (skb->l4_hash)
577 hash |= OVS_PACKET_HASH_L4_BIT;
578
579 if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) {
580 err = -ENOBUFS;
581 goto out;
582 }
583
584 /* Only reserve room for attribute header, packet data is added
585 * in skb_zerocopy() */
586 if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
587 err = -ENOBUFS;
588 goto out;
589 }
590 nla->nla_len = nla_attr_size(skb_len);
591
592 err = skb_zerocopy(user_skb, skb, skb_len, hlen);
593 if (err)
594 goto out;
595
596 /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
597 pad_packet(dp, user_skb);
598
599 ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
600
601 err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
602 user_skb = NULL;
603 out:
604 if (err)
605 skb_tx_error(skb);
606 consume_skb(user_skb);
607 consume_skb(nskb);
608
609 return err;
610 }
611
ovs_packet_cmd_execute(struct sk_buff * skb,struct genl_info * info)612 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
613 {
614 struct ovs_header *ovs_header = genl_info_userhdr(info);
615 struct net *net = sock_net(skb->sk);
616 struct nlattr **a = info->attrs;
617 struct sw_flow_actions *acts;
618 struct sk_buff *packet;
619 struct sw_flow *flow;
620 struct sw_flow_actions *sf_acts;
621 struct datapath *dp;
622 struct vport *input_vport;
623 u16 mru = 0;
624 u64 hash;
625 int len;
626 int err;
627 bool log = !a[OVS_PACKET_ATTR_PROBE];
628
629 err = -EINVAL;
630 if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
631 !a[OVS_PACKET_ATTR_ACTIONS])
632 goto err;
633
634 len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
635 packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
636 err = -ENOMEM;
637 if (!packet)
638 goto err;
639 skb_reserve(packet, NET_IP_ALIGN);
640
641 nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
642
643 /* Set packet's mru */
644 if (a[OVS_PACKET_ATTR_MRU]) {
645 mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
646 packet->ignore_df = 1;
647 }
648 OVS_CB(packet)->mru = mru;
649 OVS_CB(packet)->cutlen = U32_MAX;
650
651 if (a[OVS_PACKET_ATTR_HASH]) {
652 hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]);
653
654 __skb_set_hash(packet, hash & 0xFFFFFFFFULL,
655 !!(hash & OVS_PACKET_HASH_SW_BIT),
656 !!(hash & OVS_PACKET_HASH_L4_BIT));
657 }
658
659 OVS_CB(packet)->upcall_pid =
660 nla_get_u32_default(a[OVS_PACKET_ATTR_UPCALL_PID], 0);
661
662 /* Build an sw_flow for sending this packet. */
663 flow = ovs_flow_alloc();
664 err = PTR_ERR(flow);
665 if (IS_ERR(flow))
666 goto err_kfree_skb;
667
668 err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
669 packet, &flow->key, log);
670 if (err)
671 goto err_flow_free;
672
673 err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
674 &flow->key, &acts, log);
675 if (err)
676 goto err_flow_free;
677
678 rcu_assign_pointer(flow->sf_acts, acts);
679 packet->priority = flow->key.phy.priority;
680 packet->mark = flow->key.phy.skb_mark;
681
682 rcu_read_lock();
683 dp = get_dp_rcu(net, ovs_header->dp_ifindex);
684 err = -ENODEV;
685 if (!dp)
686 goto err_unlock;
687
688 input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
689 if (!input_vport)
690 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
691
692 if (!input_vport)
693 goto err_unlock;
694
695 packet->dev = input_vport->dev;
696 OVS_CB(packet)->input_vport = input_vport;
697 sf_acts = rcu_dereference(flow->sf_acts);
698
699 local_bh_disable();
700 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock);
701 if (IS_ENABLED(CONFIG_PREEMPT_RT))
702 this_cpu_write(ovs_pcpu_storage->owner, current);
703 err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
704 if (IS_ENABLED(CONFIG_PREEMPT_RT))
705 this_cpu_write(ovs_pcpu_storage->owner, NULL);
706 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock);
707 local_bh_enable();
708 rcu_read_unlock();
709
710 ovs_flow_free(flow, false);
711 return err;
712
713 err_unlock:
714 rcu_read_unlock();
715 err_flow_free:
716 ovs_flow_free(flow, false);
717 err_kfree_skb:
718 kfree_skb(packet);
719 err:
720 return err;
721 }
722
723 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
724 [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
725 [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
726 [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
727 [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
728 [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
729 [OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 },
730 [OVS_PACKET_ATTR_UPCALL_PID] = { .type = NLA_U32 },
731 };
732
733 static const struct genl_small_ops dp_packet_genl_ops[] = {
734 { .cmd = OVS_PACKET_CMD_EXECUTE,
735 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
736 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
737 .doit = ovs_packet_cmd_execute
738 }
739 };
740
741 static struct genl_family dp_packet_genl_family __ro_after_init = {
742 .hdrsize = sizeof(struct ovs_header),
743 .name = OVS_PACKET_FAMILY,
744 .version = OVS_PACKET_VERSION,
745 .maxattr = OVS_PACKET_ATTR_MAX,
746 .policy = packet_policy,
747 .netnsok = true,
748 .parallel_ops = true,
749 .small_ops = dp_packet_genl_ops,
750 .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops),
751 .resv_start_op = OVS_PACKET_CMD_EXECUTE + 1,
752 .module = THIS_MODULE,
753 };
754
get_dp_stats(const struct datapath * dp,struct ovs_dp_stats * stats,struct ovs_dp_megaflow_stats * mega_stats)755 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
756 struct ovs_dp_megaflow_stats *mega_stats)
757 {
758 int i;
759
760 memset(mega_stats, 0, sizeof(*mega_stats));
761
762 stats->n_flows = ovs_flow_tbl_count(&dp->table);
763 mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
764
765 stats->n_hit = stats->n_missed = stats->n_lost = 0;
766
767 for_each_possible_cpu(i) {
768 const struct dp_stats_percpu *percpu_stats;
769 struct dp_stats_percpu local_stats;
770 unsigned int start;
771
772 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
773
774 do {
775 start = u64_stats_fetch_begin(&percpu_stats->syncp);
776 local_stats = *percpu_stats;
777 } while (u64_stats_fetch_retry(&percpu_stats->syncp, start));
778
779 stats->n_hit += local_stats.n_hit;
780 stats->n_missed += local_stats.n_missed;
781 stats->n_lost += local_stats.n_lost;
782 mega_stats->n_mask_hit += local_stats.n_mask_hit;
783 mega_stats->n_cache_hit += local_stats.n_cache_hit;
784 }
785 }
786
should_fill_key(const struct sw_flow_id * sfid,uint32_t ufid_flags)787 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
788 {
789 return ovs_identifier_is_ufid(sfid) &&
790 !(ufid_flags & OVS_UFID_F_OMIT_KEY);
791 }
792
should_fill_mask(uint32_t ufid_flags)793 static bool should_fill_mask(uint32_t ufid_flags)
794 {
795 return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
796 }
797
should_fill_actions(uint32_t ufid_flags)798 static bool should_fill_actions(uint32_t ufid_flags)
799 {
800 return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
801 }
802
ovs_flow_cmd_msg_size(const struct sw_flow_actions * acts,const struct sw_flow_id * sfid,uint32_t ufid_flags)803 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
804 const struct sw_flow_id *sfid,
805 uint32_t ufid_flags)
806 {
807 size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
808
809 /* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback
810 * see ovs_nla_put_identifier()
811 */
812 if (sfid && ovs_identifier_is_ufid(sfid))
813 len += nla_total_size(sfid->ufid_len);
814 else
815 len += nla_total_size(ovs_key_attr_size());
816
817 /* OVS_FLOW_ATTR_KEY */
818 if (!sfid || should_fill_key(sfid, ufid_flags))
819 len += nla_total_size(ovs_key_attr_size());
820
821 /* OVS_FLOW_ATTR_MASK */
822 if (should_fill_mask(ufid_flags))
823 len += nla_total_size(ovs_key_attr_size());
824
825 /* OVS_FLOW_ATTR_ACTIONS */
826 if (should_fill_actions(ufid_flags))
827 len += nla_total_size(acts->orig_len);
828
829 return len
830 + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
831 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
832 + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
833 }
834
835 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_stats(const struct sw_flow * flow,struct sk_buff * skb)836 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
837 struct sk_buff *skb)
838 {
839 struct ovs_flow_stats stats;
840 __be16 tcp_flags;
841 unsigned long used;
842
843 ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
844
845 if (used &&
846 nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
847 OVS_FLOW_ATTR_PAD))
848 return -EMSGSIZE;
849
850 if (stats.n_packets &&
851 nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
852 sizeof(struct ovs_flow_stats), &stats,
853 OVS_FLOW_ATTR_PAD))
854 return -EMSGSIZE;
855
856 if ((u8)ntohs(tcp_flags) &&
857 nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
858 return -EMSGSIZE;
859
860 return 0;
861 }
862
863 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_actions(const struct sw_flow * flow,struct sk_buff * skb,int skb_orig_len)864 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
865 struct sk_buff *skb, int skb_orig_len)
866 {
867 struct nlattr *start;
868 int err;
869
870 /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
871 * this is the first flow to be dumped into 'skb'. This is unusual for
872 * Netlink but individual action lists can be longer than
873 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
874 * The userspace caller can always fetch the actions separately if it
875 * really wants them. (Most userspace callers in fact don't care.)
876 *
877 * This can only fail for dump operations because the skb is always
878 * properly sized for single flows.
879 */
880 start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS);
881 if (start) {
882 const struct sw_flow_actions *sf_acts;
883
884 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
885 err = ovs_nla_put_actions(sf_acts->actions,
886 sf_acts->actions_len, skb);
887
888 if (!err)
889 nla_nest_end(skb, start);
890 else {
891 if (skb_orig_len)
892 return err;
893
894 nla_nest_cancel(skb, start);
895 }
896 } else if (skb_orig_len) {
897 return -EMSGSIZE;
898 }
899
900 return 0;
901 }
902
903 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_info(const struct sw_flow * flow,int dp_ifindex,struct sk_buff * skb,u32 portid,u32 seq,u32 flags,u8 cmd,u32 ufid_flags)904 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
905 struct sk_buff *skb, u32 portid,
906 u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
907 {
908 const int skb_orig_len = skb->len;
909 struct ovs_header *ovs_header;
910 int err;
911
912 ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
913 flags, cmd);
914 if (!ovs_header)
915 return -EMSGSIZE;
916
917 ovs_header->dp_ifindex = dp_ifindex;
918
919 err = ovs_nla_put_identifier(flow, skb);
920 if (err)
921 goto error;
922
923 if (should_fill_key(&flow->id, ufid_flags)) {
924 err = ovs_nla_put_masked_key(flow, skb);
925 if (err)
926 goto error;
927 }
928
929 if (should_fill_mask(ufid_flags)) {
930 err = ovs_nla_put_mask(flow, skb);
931 if (err)
932 goto error;
933 }
934
935 err = ovs_flow_cmd_fill_stats(flow, skb);
936 if (err)
937 goto error;
938
939 if (should_fill_actions(ufid_flags)) {
940 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
941 if (err)
942 goto error;
943 }
944
945 genlmsg_end(skb, ovs_header);
946 return 0;
947
948 error:
949 genlmsg_cancel(skb, ovs_header);
950 return err;
951 }
952
953 /* May not be called with RCU read lock. */
ovs_flow_cmd_alloc_info(const struct sw_flow_actions * acts,const struct sw_flow_id * sfid,struct genl_info * info,bool always,uint32_t ufid_flags)954 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
955 const struct sw_flow_id *sfid,
956 struct genl_info *info,
957 bool always,
958 uint32_t ufid_flags)
959 {
960 struct sk_buff *skb;
961 size_t len;
962
963 if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
964 return NULL;
965
966 len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
967 skb = genlmsg_new(len, GFP_KERNEL);
968 if (!skb)
969 return ERR_PTR(-ENOMEM);
970
971 return skb;
972 }
973
974 /* Called with ovs_mutex. */
ovs_flow_cmd_build_info(const struct sw_flow * flow,int dp_ifindex,struct genl_info * info,u8 cmd,bool always,u32 ufid_flags)975 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
976 int dp_ifindex,
977 struct genl_info *info, u8 cmd,
978 bool always, u32 ufid_flags)
979 {
980 struct sk_buff *skb;
981 int retval;
982
983 skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
984 &flow->id, info, always, ufid_flags);
985 if (IS_ERR_OR_NULL(skb))
986 return skb;
987
988 retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
989 info->snd_portid, info->snd_seq, 0,
990 cmd, ufid_flags);
991 if (WARN_ON_ONCE(retval < 0)) {
992 kfree_skb(skb);
993 skb = ERR_PTR(retval);
994 }
995 return skb;
996 }
997
ovs_flow_cmd_new(struct sk_buff * skb,struct genl_info * info)998 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
999 {
1000 struct net *net = sock_net(skb->sk);
1001 struct nlattr **a = info->attrs;
1002 struct ovs_header *ovs_header = genl_info_userhdr(info);
1003 struct sw_flow *flow = NULL, *new_flow;
1004 struct sw_flow_mask mask;
1005 struct sk_buff *reply;
1006 struct datapath *dp;
1007 struct sw_flow_key *key;
1008 struct sw_flow_actions *acts;
1009 struct sw_flow_match match;
1010 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1011 int error;
1012 bool log = !a[OVS_FLOW_ATTR_PROBE];
1013
1014 /* Must have key and actions. */
1015 error = -EINVAL;
1016 if (!a[OVS_FLOW_ATTR_KEY]) {
1017 OVS_NLERR(log, "Flow key attr not present in new flow.");
1018 goto error;
1019 }
1020 if (!a[OVS_FLOW_ATTR_ACTIONS]) {
1021 OVS_NLERR(log, "Flow actions attr not present in new flow.");
1022 goto error;
1023 }
1024
1025 /* Most of the time we need to allocate a new flow, do it before
1026 * locking.
1027 */
1028 new_flow = ovs_flow_alloc();
1029 if (IS_ERR(new_flow)) {
1030 error = PTR_ERR(new_flow);
1031 goto error;
1032 }
1033
1034 /* Extract key. */
1035 key = kzalloc_obj(*key);
1036 if (!key) {
1037 error = -ENOMEM;
1038 goto err_kfree_flow;
1039 }
1040
1041 ovs_match_init(&match, key, false, &mask);
1042 error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1043 a[OVS_FLOW_ATTR_MASK], log);
1044 if (error)
1045 goto err_kfree_key;
1046
1047 ovs_flow_mask_key(&new_flow->key, key, true, &mask);
1048
1049 /* Extract flow identifier. */
1050 error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
1051 key, log);
1052 if (error)
1053 goto err_kfree_key;
1054
1055 /* Validate actions. */
1056 error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
1057 &new_flow->key, &acts, log);
1058 if (error) {
1059 OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
1060 goto err_kfree_key;
1061 }
1062
1063 reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
1064 ufid_flags);
1065 if (IS_ERR(reply)) {
1066 error = PTR_ERR(reply);
1067 goto err_kfree_acts;
1068 }
1069
1070 ovs_lock();
1071 dp = get_dp(net, ovs_header->dp_ifindex);
1072 if (unlikely(!dp)) {
1073 error = -ENODEV;
1074 goto err_unlock_ovs;
1075 }
1076
1077 /* Check if this is a duplicate flow */
1078 if (ovs_identifier_is_ufid(&new_flow->id))
1079 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
1080 if (!flow)
1081 flow = ovs_flow_tbl_lookup(&dp->table, key);
1082 if (likely(!flow)) {
1083 rcu_assign_pointer(new_flow->sf_acts, acts);
1084
1085 /* Put flow in bucket. */
1086 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
1087 if (unlikely(error)) {
1088 acts = NULL;
1089 goto err_unlock_ovs;
1090 }
1091
1092 if (unlikely(reply)) {
1093 error = ovs_flow_cmd_fill_info(new_flow,
1094 ovs_header->dp_ifindex,
1095 reply, info->snd_portid,
1096 info->snd_seq, 0,
1097 OVS_FLOW_CMD_NEW,
1098 ufid_flags);
1099 BUG_ON(error < 0);
1100 }
1101 ovs_unlock();
1102 } else {
1103 struct sw_flow_actions *old_acts;
1104
1105 /* Bail out if we're not allowed to modify an existing flow.
1106 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1107 * because Generic Netlink treats the latter as a dump
1108 * request. We also accept NLM_F_EXCL in case that bug ever
1109 * gets fixed.
1110 */
1111 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1112 | NLM_F_EXCL))) {
1113 error = -EEXIST;
1114 goto err_unlock_ovs;
1115 }
1116
1117 /* Look for any overlapping flow. */
1118 if (unlikely(!ovs_flow_cmp(flow, &match))) {
1119 if (ovs_identifier_is_key(&flow->id))
1120 flow = ovs_flow_tbl_lookup_exact(&dp->table,
1121 &match);
1122 else /* UFID matches but key is different */
1123 flow = NULL;
1124 if (!flow) {
1125 error = -ENOENT;
1126 goto err_unlock_ovs;
1127 }
1128 }
1129
1130 if (unlikely(reply)) {
1131 size_t cur, req;
1132
1133 cur = ovs_flow_cmd_msg_size(acts, &new_flow->id,
1134 ufid_flags);
1135 req = ovs_flow_cmd_msg_size(acts, &flow->id,
1136 ufid_flags);
1137 if (cur < req) {
1138 struct sk_buff *resized;
1139
1140 resized = ovs_flow_cmd_alloc_info(acts,
1141 &flow->id,
1142 info, false,
1143 ufid_flags);
1144 if (IS_ERR(resized)) {
1145 error = PTR_ERR(resized);
1146 goto err_unlock_ovs;
1147 }
1148 kfree_skb(reply);
1149 reply = resized;
1150 }
1151 }
1152
1153 /* Update actions. */
1154 old_acts = ovsl_dereference(flow->sf_acts);
1155 rcu_assign_pointer(flow->sf_acts, acts);
1156
1157 if (unlikely(reply)) {
1158 error = ovs_flow_cmd_fill_info(flow,
1159 ovs_header->dp_ifindex,
1160 reply, info->snd_portid,
1161 info->snd_seq, 0,
1162 OVS_FLOW_CMD_NEW,
1163 ufid_flags);
1164 BUG_ON(error < 0);
1165 }
1166 ovs_unlock();
1167
1168 ovs_nla_free_flow_actions_rcu(old_acts);
1169 ovs_flow_free(new_flow, false);
1170 }
1171
1172 if (reply)
1173 ovs_notify(&dp_flow_genl_family, reply, info);
1174
1175 kfree(key);
1176 return 0;
1177
1178 err_unlock_ovs:
1179 ovs_unlock();
1180 kfree_skb(reply);
1181 err_kfree_acts:
1182 ovs_nla_free_flow_actions(acts);
1183 err_kfree_key:
1184 kfree(key);
1185 err_kfree_flow:
1186 ovs_flow_free(new_flow, false);
1187 error:
1188 return error;
1189 }
1190
1191 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1192 static noinline_for_stack
get_flow_actions(struct net * net,const struct nlattr * a,const struct sw_flow_key * key,const struct sw_flow_mask * mask,bool log)1193 struct sw_flow_actions *get_flow_actions(struct net *net,
1194 const struct nlattr *a,
1195 const struct sw_flow_key *key,
1196 const struct sw_flow_mask *mask,
1197 bool log)
1198 {
1199 struct sw_flow_actions *acts;
1200 struct sw_flow_key masked_key;
1201 int error;
1202
1203 ovs_flow_mask_key(&masked_key, key, true, mask);
1204 error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
1205 if (error) {
1206 OVS_NLERR(log,
1207 "Actions may not be safe on all matching packets");
1208 return ERR_PTR(error);
1209 }
1210
1211 return acts;
1212 }
1213
1214 /* Factor out match-init and action-copy to avoid
1215 * "Wframe-larger-than=1024" warning. Because mask is only
1216 * used to get actions, we new a function to save some
1217 * stack space.
1218 *
1219 * If there are not key and action attrs, we return 0
1220 * directly. In the case, the caller will also not use the
1221 * match as before. If there is action attr, we try to get
1222 * actions and save them to *acts. Before returning from
1223 * the function, we reset the match->mask pointer. Because
1224 * we should not to return match object with dangling reference
1225 * to mask.
1226 * */
1227 static noinline_for_stack int
ovs_nla_init_match_and_action(struct net * net,struct sw_flow_match * match,struct sw_flow_key * key,struct nlattr ** a,struct sw_flow_actions ** acts,bool log)1228 ovs_nla_init_match_and_action(struct net *net,
1229 struct sw_flow_match *match,
1230 struct sw_flow_key *key,
1231 struct nlattr **a,
1232 struct sw_flow_actions **acts,
1233 bool log)
1234 {
1235 struct sw_flow_mask mask;
1236 int error = 0;
1237
1238 if (a[OVS_FLOW_ATTR_KEY]) {
1239 ovs_match_init(match, key, true, &mask);
1240 error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1241 a[OVS_FLOW_ATTR_MASK], log);
1242 if (error)
1243 goto error;
1244 }
1245
1246 if (a[OVS_FLOW_ATTR_ACTIONS]) {
1247 if (!a[OVS_FLOW_ATTR_KEY]) {
1248 OVS_NLERR(log,
1249 "Flow key attribute not present in set flow.");
1250 error = -EINVAL;
1251 goto error;
1252 }
1253
1254 *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1255 &mask, log);
1256 if (IS_ERR(*acts)) {
1257 error = PTR_ERR(*acts);
1258 goto error;
1259 }
1260 }
1261
1262 /* On success, error is 0. */
1263 error:
1264 match->mask = NULL;
1265 return error;
1266 }
1267
ovs_flow_cmd_set(struct sk_buff * skb,struct genl_info * info)1268 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1269 {
1270 struct net *net = sock_net(skb->sk);
1271 struct nlattr **a = info->attrs;
1272 struct ovs_header *ovs_header = genl_info_userhdr(info);
1273 struct sw_flow_key key;
1274 struct sw_flow *flow;
1275 struct sk_buff *reply = NULL;
1276 struct datapath *dp;
1277 struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1278 struct sw_flow_match match;
1279 struct sw_flow_id sfid;
1280 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1281 int error = 0;
1282 bool log = !a[OVS_FLOW_ATTR_PROBE];
1283 bool ufid_present;
1284
1285 ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1286 if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
1287 OVS_NLERR(log,
1288 "Flow set message rejected, Key attribute missing.");
1289 return -EINVAL;
1290 }
1291
1292 error = ovs_nla_init_match_and_action(net, &match, &key, a,
1293 &acts, log);
1294 if (error)
1295 goto error;
1296
1297 if (acts) {
1298 /* Can allocate before locking if have acts. */
1299 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1300 ufid_flags);
1301 if (IS_ERR(reply)) {
1302 error = PTR_ERR(reply);
1303 goto err_kfree_acts;
1304 }
1305 }
1306
1307 ovs_lock();
1308 dp = get_dp(net, ovs_header->dp_ifindex);
1309 if (unlikely(!dp)) {
1310 error = -ENODEV;
1311 goto err_unlock_ovs;
1312 }
1313 /* Check that the flow exists. */
1314 if (ufid_present)
1315 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1316 else
1317 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1318 if (unlikely(!flow)) {
1319 error = -ENOENT;
1320 goto err_unlock_ovs;
1321 }
1322
1323 /* Update actions, if present. */
1324 if (likely(acts)) {
1325 old_acts = ovsl_dereference(flow->sf_acts);
1326 rcu_assign_pointer(flow->sf_acts, acts);
1327
1328 if (unlikely(reply)) {
1329 error = ovs_flow_cmd_fill_info(flow,
1330 ovs_header->dp_ifindex,
1331 reply, info->snd_portid,
1332 info->snd_seq, 0,
1333 OVS_FLOW_CMD_SET,
1334 ufid_flags);
1335 BUG_ON(error < 0);
1336 }
1337 } else {
1338 /* Could not alloc without acts before locking. */
1339 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1340 info, OVS_FLOW_CMD_SET, false,
1341 ufid_flags);
1342
1343 if (IS_ERR(reply)) {
1344 error = PTR_ERR(reply);
1345 reply = NULL;
1346 goto err_unlock_ovs;
1347 }
1348 }
1349
1350 /* Clear stats. */
1351 if (a[OVS_FLOW_ATTR_CLEAR])
1352 ovs_flow_stats_clear(flow);
1353 ovs_unlock();
1354
1355 if (reply)
1356 ovs_notify(&dp_flow_genl_family, reply, info);
1357 if (old_acts)
1358 ovs_nla_free_flow_actions_rcu(old_acts);
1359
1360 return 0;
1361
1362 err_unlock_ovs:
1363 ovs_unlock();
1364 kfree_skb(reply);
1365 err_kfree_acts:
1366 ovs_nla_free_flow_actions(acts);
1367 error:
1368 return error;
1369 }
1370
ovs_flow_cmd_get(struct sk_buff * skb,struct genl_info * info)1371 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1372 {
1373 struct nlattr **a = info->attrs;
1374 struct ovs_header *ovs_header = genl_info_userhdr(info);
1375 struct net *net = sock_net(skb->sk);
1376 struct sw_flow_key key;
1377 struct sk_buff *reply;
1378 struct sw_flow *flow;
1379 struct datapath *dp;
1380 struct sw_flow_match match;
1381 struct sw_flow_id ufid;
1382 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1383 int err = 0;
1384 bool log = !a[OVS_FLOW_ATTR_PROBE];
1385 bool ufid_present;
1386
1387 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1388 if (a[OVS_FLOW_ATTR_KEY]) {
1389 ovs_match_init(&match, &key, true, NULL);
1390 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
1391 log);
1392 } else if (!ufid_present) {
1393 OVS_NLERR(log,
1394 "Flow get message rejected, Key attribute missing.");
1395 err = -EINVAL;
1396 }
1397 if (err)
1398 return err;
1399
1400 ovs_lock();
1401 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1402 if (!dp) {
1403 err = -ENODEV;
1404 goto unlock;
1405 }
1406
1407 if (ufid_present)
1408 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1409 else
1410 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1411 if (!flow) {
1412 err = -ENOENT;
1413 goto unlock;
1414 }
1415
1416 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1417 OVS_FLOW_CMD_GET, true, ufid_flags);
1418 if (IS_ERR(reply)) {
1419 err = PTR_ERR(reply);
1420 goto unlock;
1421 }
1422
1423 ovs_unlock();
1424 return genlmsg_reply(reply, info);
1425 unlock:
1426 ovs_unlock();
1427 return err;
1428 }
1429
ovs_flow_cmd_del(struct sk_buff * skb,struct genl_info * info)1430 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1431 {
1432 struct nlattr **a = info->attrs;
1433 struct ovs_header *ovs_header = genl_info_userhdr(info);
1434 struct net *net = sock_net(skb->sk);
1435 struct sw_flow_key key;
1436 struct sk_buff *reply;
1437 struct sw_flow *flow = NULL;
1438 struct datapath *dp;
1439 struct sw_flow_match match;
1440 struct sw_flow_id ufid;
1441 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1442 int err;
1443 bool log = !a[OVS_FLOW_ATTR_PROBE];
1444 bool ufid_present;
1445
1446 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1447 if (a[OVS_FLOW_ATTR_KEY]) {
1448 ovs_match_init(&match, &key, true, NULL);
1449 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1450 NULL, log);
1451 if (unlikely(err))
1452 return err;
1453 }
1454
1455 ovs_lock();
1456 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1457 if (unlikely(!dp)) {
1458 err = -ENODEV;
1459 goto unlock;
1460 }
1461
1462 if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1463 err = ovs_flow_tbl_flush(&dp->table);
1464 goto unlock;
1465 }
1466
1467 if (ufid_present)
1468 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1469 else
1470 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1471 if (unlikely(!flow)) {
1472 err = -ENOENT;
1473 goto unlock;
1474 }
1475
1476 reply = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
1477 &flow->id, info, false, ufid_flags);
1478 if (IS_ERR(reply)) {
1479 netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0,
1480 PTR_ERR(reply));
1481 reply = NULL;
1482 }
1483
1484 if (likely(reply)) {
1485 err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1486 reply, info->snd_portid,
1487 info->snd_seq, 0,
1488 OVS_FLOW_CMD_DEL, ufid_flags);
1489 if (WARN_ON_ONCE(err < 0)) {
1490 kfree_skb(reply);
1491 reply = NULL;
1492 }
1493 }
1494 /* Removal has to happen after ovs_flow_cmd_fill_info(), as it uses
1495 * the flow->mask that can be scheduled to be freed by the
1496 * ovs_flow_tbl_remove() and we're not holding the RCU read lock.
1497 */
1498 ovs_flow_tbl_remove(&dp->table, flow);
1499 ovs_unlock();
1500
1501 if (likely(reply))
1502 ovs_notify(&dp_flow_genl_family, reply, info);
1503
1504 ovs_flow_free(flow, true);
1505 return 0;
1506 unlock:
1507 ovs_unlock();
1508 return err;
1509 }
1510
ovs_flow_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)1511 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1512 {
1513 struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1514 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1515 struct table_instance *ti;
1516 struct datapath *dp;
1517 u32 ufid_flags;
1518 int err;
1519
1520 err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a,
1521 OVS_FLOW_ATTR_MAX, flow_policy, NULL);
1522 if (err)
1523 return err;
1524 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1525
1526 rcu_read_lock();
1527 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1528 if (!dp) {
1529 rcu_read_unlock();
1530 return -ENODEV;
1531 }
1532
1533 ti = rcu_dereference(dp->table.ti);
1534 for (;;) {
1535 struct sw_flow *flow;
1536 u32 bucket, obj;
1537
1538 bucket = cb->args[0];
1539 obj = cb->args[1];
1540 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1541 if (!flow)
1542 break;
1543
1544 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1545 NETLINK_CB(cb->skb).portid,
1546 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1547 OVS_FLOW_CMD_GET, ufid_flags) < 0)
1548 break;
1549
1550 cb->args[0] = bucket;
1551 cb->args[1] = obj;
1552 }
1553 rcu_read_unlock();
1554 return skb->len;
1555 }
1556
1557 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1558 [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1559 [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1560 [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1561 [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1562 [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1563 [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1564 [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1565 };
1566
1567 static const struct genl_small_ops dp_flow_genl_ops[] = {
1568 { .cmd = OVS_FLOW_CMD_NEW,
1569 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1570 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1571 .doit = ovs_flow_cmd_new
1572 },
1573 { .cmd = OVS_FLOW_CMD_DEL,
1574 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1575 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1576 .doit = ovs_flow_cmd_del
1577 },
1578 { .cmd = OVS_FLOW_CMD_GET,
1579 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1580 .flags = 0, /* OK for unprivileged users. */
1581 .doit = ovs_flow_cmd_get,
1582 .dumpit = ovs_flow_cmd_dump
1583 },
1584 { .cmd = OVS_FLOW_CMD_SET,
1585 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1586 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1587 .doit = ovs_flow_cmd_set,
1588 },
1589 };
1590
1591 static struct genl_family dp_flow_genl_family __ro_after_init = {
1592 .hdrsize = sizeof(struct ovs_header),
1593 .name = OVS_FLOW_FAMILY,
1594 .version = OVS_FLOW_VERSION,
1595 .maxattr = OVS_FLOW_ATTR_MAX,
1596 .policy = flow_policy,
1597 .netnsok = true,
1598 .parallel_ops = true,
1599 .small_ops = dp_flow_genl_ops,
1600 .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops),
1601 .resv_start_op = OVS_FLOW_CMD_SET + 1,
1602 .mcgrps = &ovs_dp_flow_multicast_group,
1603 .n_mcgrps = 1,
1604 .module = THIS_MODULE,
1605 };
1606
ovs_dp_cmd_msg_size(void)1607 static size_t ovs_dp_cmd_msg_size(void)
1608 {
1609 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1610
1611 msgsize += nla_total_size(IFNAMSIZ);
1612 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1613 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
1614 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1615 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
1616 msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */
1617
1618 return msgsize;
1619 }
1620
1621 /* Called with ovs_mutex. */
ovs_dp_cmd_fill_info(struct datapath * dp,struct sk_buff * skb,u32 portid,u32 seq,u32 flags,u8 cmd)1622 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1623 u32 portid, u32 seq, u32 flags, u8 cmd)
1624 {
1625 struct ovs_header *ovs_header;
1626 struct ovs_dp_stats dp_stats;
1627 struct ovs_dp_megaflow_stats dp_megaflow_stats;
1628 struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids);
1629 int err, pids_len;
1630
1631 ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1632 flags, cmd);
1633 if (!ovs_header)
1634 goto error;
1635
1636 ovs_header->dp_ifindex = get_dpifindex(dp);
1637
1638 err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1639 if (err)
1640 goto nla_put_failure;
1641
1642 get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1643 if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1644 &dp_stats, OVS_DP_ATTR_PAD))
1645 goto nla_put_failure;
1646
1647 if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1648 sizeof(struct ovs_dp_megaflow_stats),
1649 &dp_megaflow_stats, OVS_DP_ATTR_PAD))
1650 goto nla_put_failure;
1651
1652 if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1653 goto nla_put_failure;
1654
1655 if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE,
1656 ovs_flow_tbl_masks_cache_size(&dp->table)))
1657 goto nla_put_failure;
1658
1659 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
1660 pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32);
1661 if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids))
1662 goto nla_put_failure;
1663 }
1664
1665 genlmsg_end(skb, ovs_header);
1666 return 0;
1667
1668 nla_put_failure:
1669 genlmsg_cancel(skb, ovs_header);
1670 error:
1671 return -EMSGSIZE;
1672 }
1673
ovs_dp_cmd_alloc_info(void)1674 static struct sk_buff *ovs_dp_cmd_alloc_info(void)
1675 {
1676 return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1677 }
1678
1679 /* Called with rcu_read_lock or ovs_mutex. */
lookup_datapath(struct net * net,const struct ovs_header * ovs_header,struct nlattr * a[OVS_DP_ATTR_MAX+1])1680 static struct datapath *lookup_datapath(struct net *net,
1681 const struct ovs_header *ovs_header,
1682 struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1683 {
1684 struct datapath *dp;
1685
1686 if (!a[OVS_DP_ATTR_NAME])
1687 dp = get_dp(net, ovs_header->dp_ifindex);
1688 else {
1689 struct vport *vport;
1690
1691 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1692 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1693 }
1694 return dp ? dp : ERR_PTR(-ENODEV);
1695 }
1696
ovs_dp_reset_user_features(struct sk_buff * skb,struct genl_info * info)1697 static void ovs_dp_reset_user_features(struct sk_buff *skb,
1698 struct genl_info *info)
1699 {
1700 struct datapath *dp;
1701
1702 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1703 info->attrs);
1704 if (IS_ERR(dp))
1705 return;
1706
1707 pr_warn("%s: Dropping previously announced user features\n",
1708 ovs_dp_name(dp));
1709 dp->user_features = 0;
1710 }
1711
ovs_dp_set_upcall_portids(struct datapath * dp,const struct nlattr * ids)1712 static int ovs_dp_set_upcall_portids(struct datapath *dp,
1713 const struct nlattr *ids)
1714 {
1715 struct dp_nlsk_pids *old, *dp_nlsk_pids;
1716
1717 if (!nla_len(ids) || nla_len(ids) % sizeof(u32))
1718 return -EINVAL;
1719
1720 old = ovsl_dereference(dp->upcall_portids);
1721
1722 dp_nlsk_pids = kmalloc(sizeof(*dp_nlsk_pids) + nla_len(ids),
1723 GFP_KERNEL);
1724 if (!dp_nlsk_pids)
1725 return -ENOMEM;
1726
1727 dp_nlsk_pids->n_pids = nla_len(ids) / sizeof(u32);
1728 nla_memcpy(dp_nlsk_pids->pids, ids, nla_len(ids));
1729
1730 rcu_assign_pointer(dp->upcall_portids, dp_nlsk_pids);
1731
1732 kfree_rcu(old, rcu);
1733
1734 return 0;
1735 }
1736
ovs_dp_get_upcall_portid(const struct datapath * dp,uint32_t cpu_id)1737 u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id)
1738 {
1739 struct dp_nlsk_pids *dp_nlsk_pids;
1740
1741 dp_nlsk_pids = rcu_dereference(dp->upcall_portids);
1742
1743 if (dp_nlsk_pids) {
1744 if (cpu_id < dp_nlsk_pids->n_pids) {
1745 return dp_nlsk_pids->pids[cpu_id];
1746 } else if (dp_nlsk_pids->n_pids > 0 &&
1747 cpu_id >= dp_nlsk_pids->n_pids) {
1748 /* If the number of netlink PIDs is mismatched with
1749 * the number of CPUs as seen by the kernel, log this
1750 * and send the upcall to an arbitrary socket (0) in
1751 * order to not drop packets
1752 */
1753 pr_info_ratelimited("cpu_id mismatch with handler threads");
1754 return dp_nlsk_pids->pids[cpu_id %
1755 dp_nlsk_pids->n_pids];
1756 } else {
1757 return 0;
1758 }
1759 } else {
1760 return 0;
1761 }
1762 }
1763
ovs_dp_change(struct datapath * dp,struct nlattr * a[])1764 static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1765 {
1766 u32 user_features = 0, old_features = dp->user_features;
1767 int err;
1768
1769 if (a[OVS_DP_ATTR_USER_FEATURES]) {
1770 user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1771
1772 if (user_features & ~(OVS_DP_F_VPORT_PIDS |
1773 OVS_DP_F_UNALIGNED |
1774 OVS_DP_F_TC_RECIRC_SHARING |
1775 OVS_DP_F_DISPATCH_UPCALL_PER_CPU))
1776 return -EOPNOTSUPP;
1777
1778 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1779 if (user_features & OVS_DP_F_TC_RECIRC_SHARING)
1780 return -EOPNOTSUPP;
1781 #endif
1782 }
1783
1784 if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
1785 int err;
1786 u32 cache_size;
1787
1788 cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
1789 err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size);
1790 if (err)
1791 return err;
1792 }
1793
1794 dp->user_features = user_features;
1795
1796 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU &&
1797 a[OVS_DP_ATTR_PER_CPU_PIDS]) {
1798 /* Upcall Netlink Port IDs have been updated */
1799 err = ovs_dp_set_upcall_portids(dp,
1800 a[OVS_DP_ATTR_PER_CPU_PIDS]);
1801 if (err)
1802 return err;
1803 }
1804
1805 if ((dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) &&
1806 !(old_features & OVS_DP_F_TC_RECIRC_SHARING))
1807 tc_skb_ext_tc_enable();
1808 else if (!(dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) &&
1809 (old_features & OVS_DP_F_TC_RECIRC_SHARING))
1810 tc_skb_ext_tc_disable();
1811
1812 return 0;
1813 }
1814
ovs_dp_stats_init(struct datapath * dp)1815 static int ovs_dp_stats_init(struct datapath *dp)
1816 {
1817 dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1818 if (!dp->stats_percpu)
1819 return -ENOMEM;
1820
1821 return 0;
1822 }
1823
ovs_dp_vport_init(struct datapath * dp)1824 static int ovs_dp_vport_init(struct datapath *dp)
1825 {
1826 int i;
1827
1828 dp->ports = kmalloc_objs(struct hlist_head, DP_VPORT_HASH_BUCKETS);
1829 if (!dp->ports)
1830 return -ENOMEM;
1831
1832 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1833 INIT_HLIST_HEAD(&dp->ports[i]);
1834
1835 return 0;
1836 }
1837
ovs_dp_cmd_new(struct sk_buff * skb,struct genl_info * info)1838 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1839 {
1840 struct nlattr **a = info->attrs;
1841 struct vport_parms parms;
1842 struct sk_buff *reply;
1843 struct datapath *dp;
1844 struct vport *vport;
1845 struct ovs_net *ovs_net;
1846 int err;
1847
1848 err = -EINVAL;
1849 if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1850 goto err;
1851
1852 reply = ovs_dp_cmd_alloc_info();
1853 if (!reply)
1854 return -ENOMEM;
1855
1856 err = -ENOMEM;
1857 dp = kzalloc_obj(*dp);
1858 if (dp == NULL)
1859 goto err_destroy_reply;
1860
1861 ovs_dp_set_net(dp, sock_net(skb->sk));
1862
1863 /* Allocate table. */
1864 err = ovs_flow_tbl_init(&dp->table);
1865 if (err)
1866 goto err_destroy_dp;
1867
1868 err = ovs_dp_stats_init(dp);
1869 if (err)
1870 goto err_destroy_table;
1871
1872 err = ovs_dp_vport_init(dp);
1873 if (err)
1874 goto err_destroy_stats;
1875
1876 err = ovs_meters_init(dp);
1877 if (err)
1878 goto err_destroy_ports;
1879
1880 /* Set up our datapath device. */
1881 parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1882 parms.type = OVS_VPORT_TYPE_INTERNAL;
1883 parms.dp = dp;
1884 parms.port_no = OVSP_LOCAL;
1885 parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1886 parms.desired_ifindex = nla_get_s32_default(a[OVS_DP_ATTR_IFINDEX], 0);
1887
1888 /* So far only local changes have been made, now need the lock. */
1889 ovs_lock();
1890
1891 err = ovs_dp_change(dp, a);
1892 if (err)
1893 goto err_unlock_and_destroy_meters;
1894
1895 vport = new_vport(&parms);
1896 if (IS_ERR(vport)) {
1897 err = PTR_ERR(vport);
1898 if (err == -EBUSY)
1899 err = -EEXIST;
1900
1901 if (err == -EEXIST) {
1902 /* An outdated user space instance that does not understand
1903 * the concept of user_features has attempted to create a new
1904 * datapath and is likely to reuse it. Drop all user features.
1905 */
1906 if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1907 ovs_dp_reset_user_features(skb, info);
1908 }
1909
1910 goto err_destroy_portids;
1911 }
1912
1913 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1914 info->snd_seq, 0, OVS_DP_CMD_NEW);
1915 BUG_ON(err < 0);
1916
1917 ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1918 list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1919
1920 ovs_unlock();
1921
1922 ovs_notify(&dp_datapath_genl_family, reply, info);
1923 return 0;
1924
1925 err_destroy_portids:
1926 kfree(rcu_dereference_raw(dp->upcall_portids));
1927 err_unlock_and_destroy_meters:
1928 ovs_unlock();
1929 ovs_meters_exit(dp);
1930 err_destroy_ports:
1931 kfree(dp->ports);
1932 err_destroy_stats:
1933 free_percpu(dp->stats_percpu);
1934 err_destroy_table:
1935 ovs_flow_tbl_destroy(&dp->table);
1936 err_destroy_dp:
1937 kfree(dp);
1938 err_destroy_reply:
1939 kfree_skb(reply);
1940 err:
1941 return err;
1942 }
1943
1944 /* Called with ovs_mutex. */
__dp_destroy(struct datapath * dp)1945 static void __dp_destroy(struct datapath *dp)
1946 {
1947 struct flow_table *table = &dp->table;
1948 int i;
1949
1950 if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING)
1951 tc_skb_ext_tc_disable();
1952
1953 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1954 struct vport *vport;
1955 struct hlist_node *n;
1956
1957 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1958 if (vport->port_no != OVSP_LOCAL)
1959 ovs_dp_detach_port(vport);
1960 }
1961
1962 list_del_rcu(&dp->list_node);
1963
1964 /* OVSP_LOCAL is datapath internal port. We need to make sure that
1965 * all ports in datapath are destroyed first before freeing datapath.
1966 */
1967 ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1968
1969 /* Flush sw_flow in the tables. RCU cb only releases resource
1970 * such as dp, ports and tables. That may avoid some issues
1971 * such as RCU usage warning.
1972 */
1973 table_instance_flow_flush(table, ovsl_dereference(table->ti),
1974 ovsl_dereference(table->ufid_ti));
1975
1976 /* RCU destroy the ports, meters and flow tables. */
1977 call_rcu(&dp->rcu, destroy_dp_rcu);
1978 }
1979
ovs_dp_cmd_del(struct sk_buff * skb,struct genl_info * info)1980 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1981 {
1982 struct sk_buff *reply;
1983 struct datapath *dp;
1984 int err;
1985
1986 reply = ovs_dp_cmd_alloc_info();
1987 if (!reply)
1988 return -ENOMEM;
1989
1990 ovs_lock();
1991 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1992 info->attrs);
1993 err = PTR_ERR(dp);
1994 if (IS_ERR(dp))
1995 goto err_unlock_free;
1996
1997 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1998 info->snd_seq, 0, OVS_DP_CMD_DEL);
1999 BUG_ON(err < 0);
2000
2001 __dp_destroy(dp);
2002 ovs_unlock();
2003
2004 ovs_notify(&dp_datapath_genl_family, reply, info);
2005
2006 return 0;
2007
2008 err_unlock_free:
2009 ovs_unlock();
2010 kfree_skb(reply);
2011 return err;
2012 }
2013
ovs_dp_cmd_set(struct sk_buff * skb,struct genl_info * info)2014 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
2015 {
2016 struct sk_buff *reply;
2017 struct datapath *dp;
2018 int err;
2019
2020 reply = ovs_dp_cmd_alloc_info();
2021 if (!reply)
2022 return -ENOMEM;
2023
2024 ovs_lock();
2025 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
2026 info->attrs);
2027 err = PTR_ERR(dp);
2028 if (IS_ERR(dp))
2029 goto err_unlock_free;
2030
2031 err = ovs_dp_change(dp, info->attrs);
2032 if (err)
2033 goto err_unlock_free;
2034
2035 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
2036 info->snd_seq, 0, OVS_DP_CMD_SET);
2037 BUG_ON(err < 0);
2038
2039 ovs_unlock();
2040 ovs_notify(&dp_datapath_genl_family, reply, info);
2041
2042 return 0;
2043
2044 err_unlock_free:
2045 ovs_unlock();
2046 kfree_skb(reply);
2047 return err;
2048 }
2049
ovs_dp_cmd_get(struct sk_buff * skb,struct genl_info * info)2050 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
2051 {
2052 struct sk_buff *reply;
2053 struct datapath *dp;
2054 int err;
2055
2056 reply = ovs_dp_cmd_alloc_info();
2057 if (!reply)
2058 return -ENOMEM;
2059
2060 ovs_lock();
2061 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
2062 info->attrs);
2063 if (IS_ERR(dp)) {
2064 err = PTR_ERR(dp);
2065 goto err_unlock_free;
2066 }
2067 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
2068 info->snd_seq, 0, OVS_DP_CMD_GET);
2069 BUG_ON(err < 0);
2070 ovs_unlock();
2071
2072 return genlmsg_reply(reply, info);
2073
2074 err_unlock_free:
2075 ovs_unlock();
2076 kfree_skb(reply);
2077 return err;
2078 }
2079
ovs_dp_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)2080 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2081 {
2082 struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
2083 struct datapath *dp;
2084 int skip = cb->args[0];
2085 int i = 0;
2086
2087 ovs_lock();
2088 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2089 if (i >= skip &&
2090 ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
2091 cb->nlh->nlmsg_seq, NLM_F_MULTI,
2092 OVS_DP_CMD_GET) < 0)
2093 break;
2094 i++;
2095 }
2096 ovs_unlock();
2097
2098 cb->args[0] = i;
2099
2100 return skb->len;
2101 }
2102
2103 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
2104 [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2105 [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2106 [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
2107 [OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0,
2108 PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
2109 [OVS_DP_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0),
2110 };
2111
2112 static const struct genl_small_ops dp_datapath_genl_ops[] = {
2113 { .cmd = OVS_DP_CMD_NEW,
2114 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2115 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2116 .doit = ovs_dp_cmd_new
2117 },
2118 { .cmd = OVS_DP_CMD_DEL,
2119 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2120 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2121 .doit = ovs_dp_cmd_del
2122 },
2123 { .cmd = OVS_DP_CMD_GET,
2124 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2125 .flags = 0, /* OK for unprivileged users. */
2126 .doit = ovs_dp_cmd_get,
2127 .dumpit = ovs_dp_cmd_dump
2128 },
2129 { .cmd = OVS_DP_CMD_SET,
2130 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2131 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2132 .doit = ovs_dp_cmd_set,
2133 },
2134 };
2135
2136 static struct genl_family dp_datapath_genl_family __ro_after_init = {
2137 .hdrsize = sizeof(struct ovs_header),
2138 .name = OVS_DATAPATH_FAMILY,
2139 .version = OVS_DATAPATH_VERSION,
2140 .maxattr = OVS_DP_ATTR_MAX,
2141 .policy = datapath_policy,
2142 .netnsok = true,
2143 .parallel_ops = true,
2144 .small_ops = dp_datapath_genl_ops,
2145 .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops),
2146 .resv_start_op = OVS_DP_CMD_SET + 1,
2147 .mcgrps = &ovs_dp_datapath_multicast_group,
2148 .n_mcgrps = 1,
2149 .module = THIS_MODULE,
2150 };
2151
2152 /* Called with ovs_mutex or RCU read lock. */
ovs_vport_cmd_fill_info(struct vport * vport,struct sk_buff * skb,struct net * net,u32 portid,u32 seq,u32 flags,u8 cmd,gfp_t gfp)2153 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
2154 struct net *net, u32 portid, u32 seq,
2155 u32 flags, u8 cmd, gfp_t gfp)
2156 {
2157 struct ovs_header *ovs_header;
2158 struct ovs_vport_stats vport_stats;
2159 struct net *net_vport;
2160 int err;
2161
2162 ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
2163 flags, cmd);
2164 if (!ovs_header)
2165 return -EMSGSIZE;
2166
2167 ovs_header->dp_ifindex = get_dpifindex(vport->dp);
2168
2169 if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
2170 nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
2171 nla_put_string(skb, OVS_VPORT_ATTR_NAME,
2172 ovs_vport_name(vport)) ||
2173 nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
2174 goto nla_put_failure;
2175
2176 rcu_read_lock();
2177 net_vport = dev_net_rcu(vport->dev);
2178 if (!net_eq(net, net_vport)) {
2179 int id = peernet2id_alloc(net, net_vport, GFP_ATOMIC);
2180
2181 if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
2182 goto nla_put_failure_unlock;
2183 }
2184 rcu_read_unlock();
2185
2186 ovs_vport_get_stats(vport, &vport_stats);
2187 if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
2188 sizeof(struct ovs_vport_stats), &vport_stats,
2189 OVS_VPORT_ATTR_PAD))
2190 goto nla_put_failure;
2191
2192 if (ovs_vport_get_upcall_stats(vport, skb))
2193 goto nla_put_failure;
2194
2195 if (ovs_vport_get_upcall_portids(vport, skb))
2196 goto nla_put_failure;
2197
2198 genlmsg_end(skb, ovs_header);
2199 return 0;
2200
2201 nla_put_failure_unlock:
2202 rcu_read_unlock();
2203 nla_put_failure:
2204 err = -EMSGSIZE;
2205 genlmsg_cancel(skb, ovs_header);
2206 return err;
2207 }
2208
ovs_vport_cmd_msg_size(void)2209 static size_t ovs_vport_cmd_msg_size(void)
2210 {
2211 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
2212
2213 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_PORT_NO */
2214 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_TYPE */
2215 msgsize += nla_total_size(IFNAMSIZ); /* OVS_VPORT_ATTR_NAME */
2216 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_IFINDEX */
2217 msgsize += nla_total_size(sizeof(s32)); /* OVS_VPORT_ATTR_NETNSID */
2218
2219 /* OVS_VPORT_ATTR_STATS */
2220 msgsize += nla_total_size_64bit(sizeof(struct ovs_vport_stats));
2221
2222 /* OVS_VPORT_ATTR_UPCALL_STATS(OVS_VPORT_UPCALL_ATTR_SUCCESS +
2223 * OVS_VPORT_UPCALL_ATTR_FAIL)
2224 */
2225 msgsize += nla_total_size(nla_total_size_64bit(sizeof(u64)) +
2226 nla_total_size_64bit(sizeof(u64)));
2227
2228 /* OVS_VPORT_ATTR_UPCALL_PID */
2229 msgsize += nla_total_size(nr_cpu_ids * sizeof(u32));
2230
2231 return msgsize;
2232 }
2233
ovs_vport_cmd_alloc_info(void)2234 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
2235 {
2236 return genlmsg_new(ovs_vport_cmd_msg_size(), GFP_KERNEL);
2237 }
2238
2239 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
ovs_vport_cmd_build_info(struct vport * vport,struct net * net,u32 portid,u32 seq,u8 cmd)2240 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
2241 u32 portid, u32 seq, u8 cmd)
2242 {
2243 struct sk_buff *skb;
2244 int retval;
2245
2246 skb = ovs_vport_cmd_alloc_info();
2247 if (!skb)
2248 return ERR_PTR(-ENOMEM);
2249
2250 retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd,
2251 GFP_KERNEL);
2252 BUG_ON(retval < 0);
2253
2254 return skb;
2255 }
2256
2257 /* Called with ovs_mutex or RCU read lock. */
lookup_vport(struct net * net,const struct ovs_header * ovs_header,struct nlattr * a[OVS_VPORT_ATTR_MAX+1])2258 static struct vport *lookup_vport(struct net *net,
2259 const struct ovs_header *ovs_header,
2260 struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
2261 {
2262 struct datapath *dp;
2263 struct vport *vport;
2264
2265 if (a[OVS_VPORT_ATTR_IFINDEX])
2266 return ERR_PTR(-EOPNOTSUPP);
2267 if (a[OVS_VPORT_ATTR_NAME]) {
2268 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
2269 if (!vport)
2270 return ERR_PTR(-ENODEV);
2271 if (ovs_header->dp_ifindex &&
2272 ovs_header->dp_ifindex != get_dpifindex(vport->dp))
2273 return ERR_PTR(-ENODEV);
2274 return vport;
2275 } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
2276 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
2277
2278 if (port_no >= DP_MAX_PORTS)
2279 return ERR_PTR(-EFBIG);
2280
2281 dp = get_dp(net, ovs_header->dp_ifindex);
2282 if (!dp)
2283 return ERR_PTR(-ENODEV);
2284
2285 vport = ovs_vport_ovsl_rcu(dp, port_no);
2286 if (!vport)
2287 return ERR_PTR(-ENODEV);
2288 return vport;
2289 } else
2290 return ERR_PTR(-EINVAL);
2291
2292 }
2293
ovs_get_max_headroom(struct datapath * dp)2294 static unsigned int ovs_get_max_headroom(struct datapath *dp)
2295 {
2296 unsigned int dev_headroom, max_headroom = 0;
2297 struct net_device *dev;
2298 struct vport *vport;
2299 int i;
2300
2301 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2302 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2303 lockdep_ovsl_is_held()) {
2304 dev = vport->dev;
2305 dev_headroom = netdev_get_fwd_headroom(dev);
2306 if (dev_headroom > max_headroom)
2307 max_headroom = dev_headroom;
2308 }
2309 }
2310
2311 return max_headroom;
2312 }
2313
2314 /* Called with ovs_mutex */
ovs_update_headroom(struct datapath * dp,unsigned int new_headroom)2315 static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom)
2316 {
2317 struct vport *vport;
2318 int i;
2319
2320 dp->max_headroom = new_headroom;
2321 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2322 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2323 lockdep_ovsl_is_held())
2324 netdev_set_rx_headroom(vport->dev, new_headroom);
2325 }
2326 }
2327
ovs_vport_cmd_new(struct sk_buff * skb,struct genl_info * info)2328 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
2329 {
2330 struct nlattr **a = info->attrs;
2331 struct ovs_header *ovs_header = genl_info_userhdr(info);
2332 struct vport_parms parms;
2333 struct sk_buff *reply;
2334 struct vport *vport;
2335 struct datapath *dp;
2336 unsigned int new_headroom;
2337 u32 port_no;
2338 int err;
2339
2340 if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
2341 !a[OVS_VPORT_ATTR_UPCALL_PID])
2342 return -EINVAL;
2343
2344 parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2345
2346 if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL)
2347 return -EOPNOTSUPP;
2348
2349 port_no = nla_get_u32_default(a[OVS_VPORT_ATTR_PORT_NO], 0);
2350 if (port_no >= DP_MAX_PORTS)
2351 return -EFBIG;
2352
2353 reply = ovs_vport_cmd_alloc_info();
2354 if (!reply)
2355 return -ENOMEM;
2356
2357 ovs_lock();
2358 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2359 err = -ENODEV;
2360 if (!dp)
2361 goto exit_unlock_free;
2362
2363 if (port_no) {
2364 vport = ovs_vport_ovsl(dp, port_no);
2365 err = -EBUSY;
2366 if (vport)
2367 goto exit_unlock_free;
2368 } else {
2369 for (port_no = 1; ; port_no++) {
2370 if (port_no >= DP_MAX_PORTS) {
2371 err = -EFBIG;
2372 goto exit_unlock_free;
2373 }
2374 vport = ovs_vport_ovsl(dp, port_no);
2375 if (!vport)
2376 break;
2377 }
2378 }
2379
2380 parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2381 parms.dp = dp;
2382 parms.port_no = port_no;
2383 parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
2384 parms.desired_ifindex = nla_get_s32_default(a[OVS_VPORT_ATTR_IFINDEX],
2385 0);
2386
2387 vport = new_vport(&parms);
2388 err = PTR_ERR(vport);
2389 if (IS_ERR(vport))
2390 goto exit_unlock_free;
2391
2392 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2393 info->snd_portid, info->snd_seq, 0,
2394 OVS_VPORT_CMD_NEW, GFP_KERNEL);
2395
2396 new_headroom = netdev_get_fwd_headroom(vport->dev);
2397
2398 if (new_headroom > dp->max_headroom)
2399 ovs_update_headroom(dp, new_headroom);
2400 else
2401 netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2402
2403 BUG_ON(err < 0);
2404 ovs_unlock();
2405
2406 ovs_notify(&dp_vport_genl_family, reply, info);
2407 return 0;
2408
2409 exit_unlock_free:
2410 ovs_unlock();
2411 kfree_skb(reply);
2412 return err;
2413 }
2414
ovs_vport_cmd_set(struct sk_buff * skb,struct genl_info * info)2415 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2416 {
2417 struct nlattr **a = info->attrs;
2418 struct sk_buff *reply;
2419 struct vport *vport;
2420 int err;
2421
2422 reply = ovs_vport_cmd_alloc_info();
2423 if (!reply)
2424 return -ENOMEM;
2425
2426 ovs_lock();
2427 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a);
2428 err = PTR_ERR(vport);
2429 if (IS_ERR(vport))
2430 goto exit_unlock_free;
2431
2432 if (a[OVS_VPORT_ATTR_TYPE] &&
2433 nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
2434 err = -EINVAL;
2435 goto exit_unlock_free;
2436 }
2437
2438 if (a[OVS_VPORT_ATTR_OPTIONS]) {
2439 /* There are no vport types that support legacy options. */
2440 err = -EOPNOTSUPP;
2441 goto exit_unlock_free;
2442 }
2443
2444 if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
2445 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2446
2447 err = ovs_vport_set_upcall_portids(vport, ids);
2448 if (err)
2449 goto exit_unlock_free;
2450 }
2451
2452 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2453 info->snd_portid, info->snd_seq, 0,
2454 OVS_VPORT_CMD_SET, GFP_KERNEL);
2455 BUG_ON(err < 0);
2456
2457 ovs_unlock();
2458 ovs_notify(&dp_vport_genl_family, reply, info);
2459 return 0;
2460
2461 exit_unlock_free:
2462 ovs_unlock();
2463 kfree_skb(reply);
2464 return err;
2465 }
2466
ovs_vport_cmd_del(struct sk_buff * skb,struct genl_info * info)2467 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2468 {
2469 bool update_headroom = false;
2470 struct nlattr **a = info->attrs;
2471 struct sk_buff *reply;
2472 struct datapath *dp;
2473 struct vport *vport;
2474 unsigned int new_headroom;
2475 int err;
2476
2477 reply = ovs_vport_cmd_alloc_info();
2478 if (!reply)
2479 return -ENOMEM;
2480
2481 ovs_lock();
2482 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a);
2483 err = PTR_ERR(vport);
2484 if (IS_ERR(vport))
2485 goto exit_unlock_free;
2486
2487 if (vport->port_no == OVSP_LOCAL) {
2488 err = -EINVAL;
2489 goto exit_unlock_free;
2490 }
2491
2492 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2493 info->snd_portid, info->snd_seq, 0,
2494 OVS_VPORT_CMD_DEL, GFP_KERNEL);
2495 BUG_ON(err < 0);
2496
2497 /* the vport deletion may trigger dp headroom update */
2498 dp = vport->dp;
2499 if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2500 update_headroom = true;
2501
2502 netdev_reset_rx_headroom(vport->dev);
2503 ovs_dp_detach_port(vport);
2504
2505 if (update_headroom) {
2506 new_headroom = ovs_get_max_headroom(dp);
2507
2508 if (new_headroom < dp->max_headroom)
2509 ovs_update_headroom(dp, new_headroom);
2510 }
2511 ovs_unlock();
2512
2513 ovs_notify(&dp_vport_genl_family, reply, info);
2514 return 0;
2515
2516 exit_unlock_free:
2517 ovs_unlock();
2518 kfree_skb(reply);
2519 return err;
2520 }
2521
ovs_vport_cmd_get(struct sk_buff * skb,struct genl_info * info)2522 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2523 {
2524 struct nlattr **a = info->attrs;
2525 struct ovs_header *ovs_header = genl_info_userhdr(info);
2526 struct sk_buff *reply;
2527 struct vport *vport;
2528 int err;
2529
2530 reply = ovs_vport_cmd_alloc_info();
2531 if (!reply)
2532 return -ENOMEM;
2533
2534 rcu_read_lock();
2535 vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2536 err = PTR_ERR(vport);
2537 if (IS_ERR(vport))
2538 goto exit_unlock_free;
2539 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2540 info->snd_portid, info->snd_seq, 0,
2541 OVS_VPORT_CMD_GET, GFP_ATOMIC);
2542 BUG_ON(err < 0);
2543 rcu_read_unlock();
2544
2545 return genlmsg_reply(reply, info);
2546
2547 exit_unlock_free:
2548 rcu_read_unlock();
2549 kfree_skb(reply);
2550 return err;
2551 }
2552
ovs_vport_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)2553 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2554 {
2555 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2556 struct datapath *dp;
2557 int bucket = cb->args[0], skip = cb->args[1];
2558 int i, j = 0;
2559
2560 rcu_read_lock();
2561 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2562 if (!dp) {
2563 rcu_read_unlock();
2564 return -ENODEV;
2565 }
2566 for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2567 struct vport *vport;
2568
2569 j = 0;
2570 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2571 if (j >= skip &&
2572 ovs_vport_cmd_fill_info(vport, skb,
2573 sock_net(skb->sk),
2574 NETLINK_CB(cb->skb).portid,
2575 cb->nlh->nlmsg_seq,
2576 NLM_F_MULTI,
2577 OVS_VPORT_CMD_GET,
2578 GFP_ATOMIC) < 0)
2579 goto out;
2580
2581 j++;
2582 }
2583 skip = 0;
2584 }
2585 out:
2586 rcu_read_unlock();
2587
2588 cb->args[0] = i;
2589 cb->args[1] = j;
2590
2591 return skb->len;
2592 }
2593
ovs_dp_masks_rebalance(struct work_struct * work)2594 static void ovs_dp_masks_rebalance(struct work_struct *work)
2595 {
2596 struct ovs_net *ovs_net = container_of(work, struct ovs_net,
2597 masks_rebalance.work);
2598 struct datapath *dp;
2599
2600 ovs_lock();
2601
2602 list_for_each_entry(dp, &ovs_net->dps, list_node)
2603 ovs_flow_masks_rebalance(&dp->table);
2604
2605 ovs_unlock();
2606
2607 schedule_delayed_work(&ovs_net->masks_rebalance,
2608 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2609 }
2610
2611 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2612 [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2613 [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2614 [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2615 [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2616 [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC },
2617 [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED }, /* Unused. */
2618 [OVS_VPORT_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0),
2619 [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
2620 [OVS_VPORT_ATTR_UPCALL_STATS] = { .type = NLA_NESTED },
2621 };
2622
2623 static const struct genl_small_ops dp_vport_genl_ops[] = {
2624 { .cmd = OVS_VPORT_CMD_NEW,
2625 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2626 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2627 .doit = ovs_vport_cmd_new
2628 },
2629 { .cmd = OVS_VPORT_CMD_DEL,
2630 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2631 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2632 .doit = ovs_vport_cmd_del
2633 },
2634 { .cmd = OVS_VPORT_CMD_GET,
2635 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2636 .flags = 0, /* OK for unprivileged users. */
2637 .doit = ovs_vport_cmd_get,
2638 .dumpit = ovs_vport_cmd_dump
2639 },
2640 { .cmd = OVS_VPORT_CMD_SET,
2641 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2642 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2643 .doit = ovs_vport_cmd_set,
2644 },
2645 };
2646
2647 struct genl_family dp_vport_genl_family __ro_after_init = {
2648 .hdrsize = sizeof(struct ovs_header),
2649 .name = OVS_VPORT_FAMILY,
2650 .version = OVS_VPORT_VERSION,
2651 .maxattr = OVS_VPORT_ATTR_MAX,
2652 .policy = vport_policy,
2653 .netnsok = true,
2654 .parallel_ops = true,
2655 .small_ops = dp_vport_genl_ops,
2656 .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops),
2657 .resv_start_op = OVS_VPORT_CMD_SET + 1,
2658 .mcgrps = &ovs_dp_vport_multicast_group,
2659 .n_mcgrps = 1,
2660 .module = THIS_MODULE,
2661 };
2662
2663 static struct genl_family * const dp_genl_families[] = {
2664 &dp_datapath_genl_family,
2665 &dp_vport_genl_family,
2666 &dp_flow_genl_family,
2667 &dp_packet_genl_family,
2668 &dp_meter_genl_family,
2669 #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
2670 &dp_ct_limit_genl_family,
2671 #endif
2672 };
2673
dp_unregister_genl(int n_families)2674 static void dp_unregister_genl(int n_families)
2675 {
2676 int i;
2677
2678 for (i = 0; i < n_families; i++)
2679 genl_unregister_family(dp_genl_families[i]);
2680 }
2681
dp_register_genl(void)2682 static int __init dp_register_genl(void)
2683 {
2684 int err;
2685 int i;
2686
2687 for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2688
2689 err = genl_register_family(dp_genl_families[i]);
2690 if (err)
2691 goto error;
2692 }
2693
2694 return 0;
2695
2696 error:
2697 dp_unregister_genl(i);
2698 return err;
2699 }
2700
ovs_init_net(struct net * net)2701 static int __net_init ovs_init_net(struct net *net)
2702 {
2703 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2704 int err;
2705
2706 INIT_LIST_HEAD(&ovs_net->dps);
2707 INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2708 INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance);
2709
2710 err = ovs_ct_init(net);
2711 if (err)
2712 return err;
2713
2714 schedule_delayed_work(&ovs_net->masks_rebalance,
2715 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2716 return 0;
2717 }
2718
list_vports_from_net(struct net * net,struct net * dnet,struct list_head * head)2719 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2720 struct list_head *head)
2721 {
2722 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2723 struct datapath *dp;
2724
2725 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2726 int i;
2727
2728 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2729 struct vport *vport;
2730
2731 hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2732 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2733 continue;
2734
2735 if (dev_net(vport->dev) == dnet)
2736 list_add(&vport->detach_list, head);
2737 }
2738 }
2739 }
2740 }
2741
ovs_exit_net(struct net * dnet)2742 static void __net_exit ovs_exit_net(struct net *dnet)
2743 {
2744 struct datapath *dp, *dp_next;
2745 struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2746 struct vport *vport, *vport_next;
2747 struct net *net;
2748 LIST_HEAD(head);
2749
2750 ovs_lock();
2751
2752 ovs_ct_exit(dnet);
2753
2754 list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2755 __dp_destroy(dp);
2756
2757 down_read(&net_rwsem);
2758 for_each_net(net)
2759 list_vports_from_net(net, dnet, &head);
2760 up_read(&net_rwsem);
2761
2762 /* Detach all vports from given namespace. */
2763 list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2764 list_del(&vport->detach_list);
2765 ovs_dp_detach_port(vport);
2766 }
2767
2768 ovs_unlock();
2769
2770 cancel_delayed_work_sync(&ovs_net->masks_rebalance);
2771 cancel_work_sync(&ovs_net->dp_notify_work);
2772 }
2773
2774 static struct pernet_operations ovs_net_ops = {
2775 .init = ovs_init_net,
2776 .exit = ovs_exit_net,
2777 .id = &ovs_net_id,
2778 .size = sizeof(struct ovs_net),
2779 };
2780
2781 static const char * const ovs_drop_reasons[] = {
2782 #define S(x) [(x) & ~SKB_DROP_REASON_SUBSYS_MASK] = (#x),
2783 OVS_DROP_REASONS(S)
2784 #undef S
2785 };
2786
2787 static struct drop_reason_list drop_reason_list_ovs = {
2788 .reasons = ovs_drop_reasons,
2789 .n_reasons = ARRAY_SIZE(ovs_drop_reasons),
2790 };
2791
ovs_alloc_percpu_storage(void)2792 static int __init ovs_alloc_percpu_storage(void)
2793 {
2794 unsigned int cpu;
2795
2796 ovs_pcpu_storage = alloc_percpu(*ovs_pcpu_storage);
2797 if (!ovs_pcpu_storage)
2798 return -ENOMEM;
2799
2800 for_each_possible_cpu(cpu) {
2801 struct ovs_pcpu_storage *ovs_pcpu;
2802
2803 ovs_pcpu = per_cpu_ptr(ovs_pcpu_storage, cpu);
2804 local_lock_init(&ovs_pcpu->bh_lock);
2805 }
2806 return 0;
2807 }
2808
ovs_free_percpu_storage(void)2809 static void ovs_free_percpu_storage(void)
2810 {
2811 free_percpu(ovs_pcpu_storage);
2812 }
2813
dp_init(void)2814 static int __init dp_init(void)
2815 {
2816 int err;
2817
2818 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) >
2819 sizeof_field(struct sk_buff, cb));
2820
2821 pr_info("Open vSwitch switching datapath\n");
2822
2823 err = ovs_alloc_percpu_storage();
2824 if (err)
2825 goto error;
2826
2827 err = ovs_internal_dev_rtnl_link_register();
2828 if (err)
2829 goto error;
2830
2831 err = ovs_flow_init();
2832 if (err)
2833 goto error_unreg_rtnl_link;
2834
2835 err = ovs_vport_init();
2836 if (err)
2837 goto error_flow_exit;
2838
2839 err = register_pernet_device(&ovs_net_ops);
2840 if (err)
2841 goto error_vport_exit;
2842
2843 err = register_netdevice_notifier(&ovs_dp_device_notifier);
2844 if (err)
2845 goto error_netns_exit;
2846
2847 err = ovs_netdev_init();
2848 if (err)
2849 goto error_unreg_notifier;
2850
2851 err = dp_register_genl();
2852 if (err < 0)
2853 goto error_unreg_netdev;
2854
2855 drop_reasons_register_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH,
2856 &drop_reason_list_ovs);
2857
2858 return 0;
2859
2860 error_unreg_netdev:
2861 ovs_netdev_exit();
2862 error_unreg_notifier:
2863 unregister_netdevice_notifier(&ovs_dp_device_notifier);
2864 error_netns_exit:
2865 unregister_pernet_device(&ovs_net_ops);
2866 error_vport_exit:
2867 ovs_vport_exit();
2868 error_flow_exit:
2869 ovs_flow_exit();
2870 error_unreg_rtnl_link:
2871 ovs_internal_dev_rtnl_link_unregister();
2872 error:
2873 ovs_free_percpu_storage();
2874 return err;
2875 }
2876
dp_cleanup(void)2877 static void dp_cleanup(void)
2878 {
2879 dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2880 ovs_netdev_exit();
2881 unregister_netdevice_notifier(&ovs_dp_device_notifier);
2882 unregister_pernet_device(&ovs_net_ops);
2883 drop_reasons_unregister_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH);
2884 rcu_barrier();
2885 ovs_vport_exit();
2886 ovs_flow_exit();
2887 ovs_internal_dev_rtnl_link_unregister();
2888 ovs_free_percpu_storage();
2889 }
2890
2891 module_init(dp_init);
2892 module_exit(dp_cleanup);
2893
2894 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2895 MODULE_LICENSE("GPL");
2896 MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2897 MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2898 MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2899 MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
2900 MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
2901 MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
2902