1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2007-2014 Nicira, Inc.
4 */
5
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/init.h>
9 #include <linux/module.h>
10 #include <linux/if_arp.h>
11 #include <linux/if_vlan.h>
12 #include <linux/in.h>
13 #include <linux/ip.h>
14 #include <linux/jhash.h>
15 #include <linux/delay.h>
16 #include <linux/time.h>
17 #include <linux/etherdevice.h>
18 #include <linux/kernel.h>
19 #include <linux/kthread.h>
20 #include <linux/mutex.h>
21 #include <linux/percpu.h>
22 #include <linux/rcupdate.h>
23 #include <linux/tcp.h>
24 #include <linux/udp.h>
25 #include <linux/ethtool.h>
26 #include <linux/wait.h>
27 #include <asm/div64.h>
28 #include <linux/highmem.h>
29 #include <linux/netfilter_bridge.h>
30 #include <linux/netfilter_ipv4.h>
31 #include <linux/inetdevice.h>
32 #include <linux/list.h>
33 #include <linux/openvswitch.h>
34 #include <linux/rculist.h>
35 #include <linux/dmi.h>
36 #include <net/genetlink.h>
37 #include <net/gso.h>
38 #include <net/net_namespace.h>
39 #include <net/netns/generic.h>
40 #include <net/pkt_cls.h>
41
42 #include "datapath.h"
43 #include "drop.h"
44 #include "flow.h"
45 #include "flow_table.h"
46 #include "flow_netlink.h"
47 #include "meter.h"
48 #include "openvswitch_trace.h"
49 #include "vport-internal_dev.h"
50 #include "vport-netdev.h"
51
52 unsigned int ovs_net_id __read_mostly;
53
54 static struct genl_family dp_packet_genl_family;
55 static struct genl_family dp_flow_genl_family;
56 static struct genl_family dp_datapath_genl_family;
57
58 static const struct nla_policy flow_policy[];
59
60 static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
61 .name = OVS_FLOW_MCGROUP,
62 };
63
64 static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
65 .name = OVS_DATAPATH_MCGROUP,
66 };
67
68 static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
69 .name = OVS_VPORT_MCGROUP,
70 };
71
72 /* Check if need to build a reply message.
73 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
ovs_must_notify(struct genl_family * family,struct genl_info * info,unsigned int group)74 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
75 unsigned int group)
76 {
77 return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
78 genl_has_listeners(family, genl_info_net(info), group);
79 }
80
ovs_notify(struct genl_family * family,struct sk_buff * skb,struct genl_info * info)81 static void ovs_notify(struct genl_family *family,
82 struct sk_buff *skb, struct genl_info *info)
83 {
84 genl_notify(family, skb, info, 0, GFP_KERNEL);
85 }
86
87 /**
88 * DOC: Locking:
89 *
90 * All writes e.g. Writes to device state (add/remove datapath, port, set
91 * operations on vports, etc.), Writes to other state (flow table
92 * modifications, set miscellaneous datapath parameters, etc.) are protected
93 * by ovs_lock.
94 *
95 * Reads are protected by RCU.
96 *
97 * There are a few special cases (mostly stats) that have their own
98 * synchronization but they nest under all of above and don't interact with
99 * each other.
100 *
101 * The RTNL lock nests inside ovs_mutex.
102 */
103
104 static DEFINE_MUTEX(ovs_mutex);
105
ovs_lock(void)106 void ovs_lock(void)
107 {
108 mutex_lock(&ovs_mutex);
109 }
110
ovs_unlock(void)111 void ovs_unlock(void)
112 {
113 mutex_unlock(&ovs_mutex);
114 }
115
116 #ifdef CONFIG_LOCKDEP
lockdep_ovsl_is_held(void)117 int lockdep_ovsl_is_held(void)
118 {
119 if (debug_locks)
120 return lockdep_is_held(&ovs_mutex);
121 else
122 return 1;
123 }
124 #endif
125
126 static struct vport *new_vport(const struct vport_parms *);
127 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
128 const struct sw_flow_key *,
129 const struct dp_upcall_info *,
130 uint32_t cutlen);
131 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
132 const struct sw_flow_key *,
133 const struct dp_upcall_info *,
134 uint32_t cutlen);
135
136 static void ovs_dp_masks_rebalance(struct work_struct *work);
137
138 static int ovs_dp_set_upcall_portids(struct datapath *, const struct nlattr *);
139
140 /* Must be called with rcu_read_lock or ovs_mutex. */
ovs_dp_name(const struct datapath * dp)141 const char *ovs_dp_name(const struct datapath *dp)
142 {
143 struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
144 return ovs_vport_name(vport);
145 }
146
get_dpifindex(const struct datapath * dp)147 static int get_dpifindex(const struct datapath *dp)
148 {
149 struct vport *local;
150 int ifindex;
151
152 rcu_read_lock();
153
154 local = ovs_vport_rcu(dp, OVSP_LOCAL);
155 if (local)
156 ifindex = local->dev->ifindex;
157 else
158 ifindex = 0;
159
160 rcu_read_unlock();
161
162 return ifindex;
163 }
164
destroy_dp_rcu(struct rcu_head * rcu)165 static void destroy_dp_rcu(struct rcu_head *rcu)
166 {
167 struct datapath *dp = container_of(rcu, struct datapath, rcu);
168
169 ovs_flow_tbl_destroy(&dp->table);
170 free_percpu(dp->stats_percpu);
171 kfree(dp->ports);
172 ovs_meters_exit(dp);
173 kfree(rcu_dereference_raw(dp->upcall_portids));
174 kfree(dp);
175 }
176
vport_hash_bucket(const struct datapath * dp,u16 port_no)177 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
178 u16 port_no)
179 {
180 return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
181 }
182
183 /* Called with ovs_mutex or RCU read lock. */
ovs_lookup_vport(const struct datapath * dp,u16 port_no)184 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
185 {
186 struct vport *vport;
187 struct hlist_head *head;
188
189 head = vport_hash_bucket(dp, port_no);
190 hlist_for_each_entry_rcu(vport, head, dp_hash_node,
191 lockdep_ovsl_is_held()) {
192 if (vport->port_no == port_no)
193 return vport;
194 }
195 return NULL;
196 }
197
198 /* Called with ovs_mutex. */
new_vport(const struct vport_parms * parms)199 static struct vport *new_vport(const struct vport_parms *parms)
200 {
201 struct vport *vport;
202
203 vport = ovs_vport_add(parms);
204 if (!IS_ERR(vport)) {
205 struct datapath *dp = parms->dp;
206 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
207
208 hlist_add_head_rcu(&vport->dp_hash_node, head);
209 }
210 return vport;
211 }
212
ovs_vport_update_upcall_stats(struct sk_buff * skb,const struct dp_upcall_info * upcall_info,bool upcall_result)213 static void ovs_vport_update_upcall_stats(struct sk_buff *skb,
214 const struct dp_upcall_info *upcall_info,
215 bool upcall_result)
216 {
217 struct vport *p = OVS_CB(skb)->input_vport;
218 struct vport_upcall_stats_percpu *stats;
219
220 if (upcall_info->cmd != OVS_PACKET_CMD_MISS &&
221 upcall_info->cmd != OVS_PACKET_CMD_ACTION)
222 return;
223
224 stats = this_cpu_ptr(p->upcall_stats);
225 u64_stats_update_begin(&stats->syncp);
226 if (upcall_result)
227 u64_stats_inc(&stats->n_success);
228 else
229 u64_stats_inc(&stats->n_fail);
230 u64_stats_update_end(&stats->syncp);
231 }
232
ovs_dp_detach_port(struct vport * p)233 void ovs_dp_detach_port(struct vport *p)
234 {
235 ASSERT_OVSL();
236
237 /* First drop references to device. */
238 hlist_del_rcu(&p->dp_hash_node);
239
240 /* Then destroy it. */
241 ovs_vport_del(p);
242 }
243
244 /* Must be called with rcu_read_lock. */
ovs_dp_process_packet(struct sk_buff * skb,struct sw_flow_key * key)245 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
246 {
247 struct ovs_pcpu_storage *ovs_pcpu = this_cpu_ptr(ovs_pcpu_storage);
248 const struct vport *p = OVS_CB(skb)->input_vport;
249 struct datapath *dp = p->dp;
250 struct sw_flow *flow;
251 struct sw_flow_actions *sf_acts;
252 struct dp_stats_percpu *stats;
253 bool ovs_pcpu_locked = false;
254 u64 *stats_counter;
255 u32 n_mask_hit;
256 u32 n_cache_hit;
257 int error;
258
259 stats = this_cpu_ptr(dp->stats_percpu);
260
261 /* Look up flow. */
262 flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
263 &n_mask_hit, &n_cache_hit);
264 if (unlikely(!flow)) {
265 struct dp_upcall_info upcall;
266
267 memset(&upcall, 0, sizeof(upcall));
268 upcall.cmd = OVS_PACKET_CMD_MISS;
269
270 if (OVS_CB(skb)->upcall_pid)
271 upcall.portid = OVS_CB(skb)->upcall_pid;
272 else if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU)
273 upcall.portid =
274 ovs_dp_get_upcall_portid(dp, smp_processor_id());
275 else
276 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
277
278 upcall.mru = OVS_CB(skb)->mru;
279 error = ovs_dp_upcall(dp, skb, key, &upcall, U32_MAX);
280 switch (error) {
281 case 0:
282 case -EAGAIN:
283 case -ERESTARTSYS:
284 case -EINTR:
285 consume_skb(skb);
286 break;
287 default:
288 skb_tx_error(skb);
289 kfree_skb(skb);
290 break;
291 }
292 stats_counter = &stats->n_missed;
293 goto out;
294 }
295
296 ovs_flow_stats_update(flow, key->tp.flags, skb);
297 sf_acts = rcu_dereference(flow->sf_acts);
298 /* This path can be invoked recursively: Use the current task to
299 * identify recursive invocation - the lock must be acquired only once.
300 * Even with disabled bottom halves this can be preempted on PREEMPT_RT.
301 * Limit the locking to RT to avoid assigning `owner' if it can be
302 * avoided.
303 */
304 if (IS_ENABLED(CONFIG_PREEMPT_RT) && ovs_pcpu->owner != current) {
305 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock);
306 ovs_pcpu->owner = current;
307 ovs_pcpu_locked = true;
308 }
309
310 error = ovs_execute_actions(dp, skb, sf_acts, key);
311 if (unlikely(error))
312 net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n",
313 ovs_dp_name(dp), error);
314 if (ovs_pcpu_locked) {
315 ovs_pcpu->owner = NULL;
316 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock);
317 }
318
319 stats_counter = &stats->n_hit;
320
321 out:
322 /* Update datapath statistics. */
323 u64_stats_update_begin(&stats->syncp);
324 (*stats_counter)++;
325 stats->n_mask_hit += n_mask_hit;
326 stats->n_cache_hit += n_cache_hit;
327 u64_stats_update_end(&stats->syncp);
328 }
329
ovs_dp_upcall(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)330 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
331 const struct sw_flow_key *key,
332 const struct dp_upcall_info *upcall_info,
333 uint32_t cutlen)
334 {
335 struct dp_stats_percpu *stats;
336 int err;
337
338 if (trace_ovs_dp_upcall_enabled())
339 trace_ovs_dp_upcall(dp, skb, key, upcall_info);
340
341 if (upcall_info->portid == 0) {
342 err = -ENOTCONN;
343 goto err;
344 }
345
346 if (!skb_is_gso(skb))
347 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
348 else
349 err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
350
351 ovs_vport_update_upcall_stats(skb, upcall_info, !err);
352 if (err)
353 goto err;
354
355 return 0;
356
357 err:
358 stats = this_cpu_ptr(dp->stats_percpu);
359
360 u64_stats_update_begin(&stats->syncp);
361 stats->n_lost++;
362 u64_stats_update_end(&stats->syncp);
363
364 return err;
365 }
366
queue_gso_packets(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)367 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
368 const struct sw_flow_key *key,
369 const struct dp_upcall_info *upcall_info,
370 uint32_t cutlen)
371 {
372 unsigned int gso_type = skb_shinfo(skb)->gso_type;
373 struct sw_flow_key later_key;
374 struct sk_buff *segs, *nskb;
375 int err;
376
377 BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET);
378 segs = __skb_gso_segment(skb, NETIF_F_SG, false);
379 if (IS_ERR(segs))
380 return PTR_ERR(segs);
381 if (segs == NULL)
382 return -EINVAL;
383
384 if (gso_type & SKB_GSO_UDP) {
385 /* The initial flow key extracted by ovs_flow_key_extract()
386 * in this case is for a first fragment, so we need to
387 * properly mark later fragments.
388 */
389 later_key = *key;
390 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
391 }
392
393 /* Queue all of the segments. */
394 skb_list_walk_safe(segs, skb, nskb) {
395 if (gso_type & SKB_GSO_UDP && skb != segs)
396 key = &later_key;
397
398 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
399 if (err)
400 break;
401
402 }
403
404 /* Free all of the segments. */
405 skb_list_walk_safe(segs, skb, nskb) {
406 if (err)
407 kfree_skb(skb);
408 else
409 consume_skb(skb);
410 }
411 return err;
412 }
413
upcall_msg_size(const struct dp_upcall_info * upcall_info,unsigned int hdrlen,int actions_attrlen)414 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
415 unsigned int hdrlen, int actions_attrlen)
416 {
417 size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
418 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
419 + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
420 + nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */
421 + nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */
422
423 /* OVS_PACKET_ATTR_USERDATA */
424 if (upcall_info->userdata)
425 size += NLA_ALIGN(upcall_info->userdata->nla_len);
426
427 /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
428 if (upcall_info->egress_tun_info)
429 size += nla_total_size(ovs_tun_key_attr_size());
430
431 /* OVS_PACKET_ATTR_ACTIONS */
432 if (upcall_info->actions_len)
433 size += nla_total_size(actions_attrlen);
434
435 /* OVS_PACKET_ATTR_MRU */
436 if (upcall_info->mru)
437 size += nla_total_size(sizeof(upcall_info->mru));
438
439 return size;
440 }
441
pad_packet(struct datapath * dp,struct sk_buff * skb)442 static void pad_packet(struct datapath *dp, struct sk_buff *skb)
443 {
444 if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
445 size_t plen = NLA_ALIGN(skb->len) - skb->len;
446
447 if (plen > 0)
448 skb_put_zero(skb, plen);
449 }
450 }
451
queue_userspace_packet(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)452 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
453 const struct sw_flow_key *key,
454 const struct dp_upcall_info *upcall_info,
455 uint32_t cutlen)
456 {
457 struct ovs_header *upcall;
458 struct sk_buff *nskb = NULL;
459 struct sk_buff *user_skb = NULL; /* to be queued to userspace */
460 struct nlattr *nla;
461 size_t msg_size;
462 size_t skb_len;
463 unsigned int hlen;
464 int err, dp_ifindex;
465 u64 hash;
466
467 dp_ifindex = get_dpifindex(dp);
468 if (!dp_ifindex)
469 return -ENODEV;
470
471 if (!skb_frags_readable(skb))
472 return -EFAULT;
473
474 if (skb_vlan_tag_present(skb)) {
475 nskb = skb_clone(skb, GFP_ATOMIC);
476 if (!nskb)
477 return -ENOMEM;
478
479 nskb = __vlan_hwaccel_push_inside(nskb);
480 if (!nskb)
481 return -ENOMEM;
482
483 skb = nskb;
484 }
485
486 skb_len = min(skb->len, cutlen);
487 if (nla_attr_size(skb_len) > USHRT_MAX) {
488 err = -EFBIG;
489 goto out;
490 }
491
492 /* Complete checksum if needed */
493 if (skb->ip_summed == CHECKSUM_PARTIAL &&
494 (err = skb_csum_hwoffload_help(skb, 0)))
495 goto out;
496
497 /* Older versions of OVS user space enforce alignment of the last
498 * Netlink attribute to NLA_ALIGNTO which would require extensive
499 * padding logic. Only perform zerocopy if padding is not required.
500 */
501 if (dp->user_features & OVS_DP_F_UNALIGNED)
502 hlen = min(skb_zerocopy_headlen(skb), cutlen);
503 else
504 hlen = skb_len;
505
506 msg_size = upcall_msg_size(upcall_info, hlen,
507 OVS_CB(skb)->acts_origlen);
508 user_skb = genlmsg_new(msg_size, GFP_ATOMIC);
509 if (!user_skb) {
510 err = -ENOMEM;
511 goto out;
512 }
513
514 upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
515 0, upcall_info->cmd);
516 if (!upcall) {
517 err = -EINVAL;
518 goto out;
519 }
520 upcall->dp_ifindex = dp_ifindex;
521
522 err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
523 if (err)
524 goto out;
525
526 if (upcall_info->userdata)
527 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
528 nla_len(upcall_info->userdata),
529 nla_data(upcall_info->userdata));
530
531 if (upcall_info->egress_tun_info) {
532 nla = nla_nest_start_noflag(user_skb,
533 OVS_PACKET_ATTR_EGRESS_TUN_KEY);
534 if (!nla) {
535 err = -EMSGSIZE;
536 goto out;
537 }
538 err = ovs_nla_put_tunnel_info(user_skb,
539 upcall_info->egress_tun_info);
540 if (err)
541 goto out;
542
543 nla_nest_end(user_skb, nla);
544 }
545
546 if (upcall_info->actions_len) {
547 nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS);
548 if (!nla) {
549 err = -EMSGSIZE;
550 goto out;
551 }
552 err = ovs_nla_put_actions(upcall_info->actions,
553 upcall_info->actions_len,
554 user_skb);
555 if (!err)
556 nla_nest_end(user_skb, nla);
557 else
558 nla_nest_cancel(user_skb, nla);
559 }
560
561 /* Add OVS_PACKET_ATTR_MRU */
562 if (upcall_info->mru &&
563 nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) {
564 err = -ENOBUFS;
565 goto out;
566 }
567
568 /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
569 if (skb_len < skb->len &&
570 nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) {
571 err = -ENOBUFS;
572 goto out;
573 }
574
575 /* Add OVS_PACKET_ATTR_HASH */
576 hash = skb_get_hash_raw(skb);
577 if (skb->sw_hash)
578 hash |= OVS_PACKET_HASH_SW_BIT;
579
580 if (skb->l4_hash)
581 hash |= OVS_PACKET_HASH_L4_BIT;
582
583 if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) {
584 err = -ENOBUFS;
585 goto out;
586 }
587
588 /* Only reserve room for attribute header, packet data is added
589 * in skb_zerocopy() */
590 if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
591 err = -ENOBUFS;
592 goto out;
593 }
594 nla->nla_len = nla_attr_size(skb_len);
595
596 err = skb_zerocopy(user_skb, skb, skb_len, hlen);
597 if (err)
598 goto out;
599
600 /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
601 pad_packet(dp, user_skb);
602
603 ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
604
605 err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
606 user_skb = NULL;
607 out:
608 consume_skb(user_skb);
609 consume_skb(nskb);
610
611 return err;
612 }
613
ovs_packet_cmd_execute(struct sk_buff * skb,struct genl_info * info)614 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
615 {
616 struct ovs_header *ovs_header = genl_info_userhdr(info);
617 struct net *net = sock_net(skb->sk);
618 struct nlattr **a = info->attrs;
619 struct sw_flow_actions *acts;
620 struct sk_buff *packet;
621 struct sw_flow *flow;
622 struct sw_flow_actions *sf_acts;
623 struct datapath *dp;
624 struct vport *input_vport;
625 u16 mru = 0;
626 u64 hash;
627 int len;
628 int err;
629 bool log = !a[OVS_PACKET_ATTR_PROBE];
630
631 err = -EINVAL;
632 if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
633 !a[OVS_PACKET_ATTR_ACTIONS])
634 goto err;
635
636 len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
637 packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
638 err = -ENOMEM;
639 if (!packet)
640 goto err;
641 skb_reserve(packet, NET_IP_ALIGN);
642
643 nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
644
645 /* Set packet's mru */
646 if (a[OVS_PACKET_ATTR_MRU]) {
647 mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
648 packet->ignore_df = 1;
649 }
650 OVS_CB(packet)->mru = mru;
651 OVS_CB(packet)->cutlen = U32_MAX;
652
653 if (a[OVS_PACKET_ATTR_HASH]) {
654 hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]);
655
656 __skb_set_hash(packet, hash & 0xFFFFFFFFULL,
657 !!(hash & OVS_PACKET_HASH_SW_BIT),
658 !!(hash & OVS_PACKET_HASH_L4_BIT));
659 }
660
661 OVS_CB(packet)->upcall_pid =
662 nla_get_u32_default(a[OVS_PACKET_ATTR_UPCALL_PID], 0);
663
664 /* Build an sw_flow for sending this packet. */
665 flow = ovs_flow_alloc();
666 err = PTR_ERR(flow);
667 if (IS_ERR(flow))
668 goto err_kfree_skb;
669
670 err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
671 packet, &flow->key, log);
672 if (err)
673 goto err_flow_free;
674
675 err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
676 &flow->key, &acts, log);
677 if (err)
678 goto err_flow_free;
679
680 rcu_assign_pointer(flow->sf_acts, acts);
681 packet->priority = flow->key.phy.priority;
682 packet->mark = flow->key.phy.skb_mark;
683
684 rcu_read_lock();
685 dp = get_dp_rcu(net, ovs_header->dp_ifindex);
686 err = -ENODEV;
687 if (!dp)
688 goto err_unlock;
689
690 input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
691 if (!input_vport)
692 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
693
694 if (!input_vport)
695 goto err_unlock;
696
697 packet->dev = input_vport->dev;
698 OVS_CB(packet)->input_vport = input_vport;
699 sf_acts = rcu_dereference(flow->sf_acts);
700
701 local_bh_disable();
702 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock);
703 if (IS_ENABLED(CONFIG_PREEMPT_RT))
704 this_cpu_write(ovs_pcpu_storage->owner, current);
705 err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
706 if (IS_ENABLED(CONFIG_PREEMPT_RT))
707 this_cpu_write(ovs_pcpu_storage->owner, NULL);
708 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock);
709 local_bh_enable();
710 rcu_read_unlock();
711
712 ovs_flow_free(flow, false);
713 return err;
714
715 err_unlock:
716 rcu_read_unlock();
717 err_flow_free:
718 ovs_flow_free(flow, false);
719 err_kfree_skb:
720 kfree_skb(packet);
721 err:
722 return err;
723 }
724
725 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
726 [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
727 [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
728 [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
729 [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
730 [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
731 [OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 },
732 [OVS_PACKET_ATTR_UPCALL_PID] = { .type = NLA_U32 },
733 };
734
735 static const struct genl_small_ops dp_packet_genl_ops[] = {
736 { .cmd = OVS_PACKET_CMD_EXECUTE,
737 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
738 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
739 .doit = ovs_packet_cmd_execute
740 }
741 };
742
743 static struct genl_family dp_packet_genl_family __ro_after_init = {
744 .hdrsize = sizeof(struct ovs_header),
745 .name = OVS_PACKET_FAMILY,
746 .version = OVS_PACKET_VERSION,
747 .maxattr = OVS_PACKET_ATTR_MAX,
748 .policy = packet_policy,
749 .netnsok = true,
750 .parallel_ops = true,
751 .small_ops = dp_packet_genl_ops,
752 .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops),
753 .resv_start_op = OVS_PACKET_CMD_EXECUTE + 1,
754 .module = THIS_MODULE,
755 };
756
get_dp_stats(const struct datapath * dp,struct ovs_dp_stats * stats,struct ovs_dp_megaflow_stats * mega_stats)757 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
758 struct ovs_dp_megaflow_stats *mega_stats)
759 {
760 int i;
761
762 memset(mega_stats, 0, sizeof(*mega_stats));
763
764 stats->n_flows = ovs_flow_tbl_count(&dp->table);
765 mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
766
767 stats->n_hit = stats->n_missed = stats->n_lost = 0;
768
769 for_each_possible_cpu(i) {
770 const struct dp_stats_percpu *percpu_stats;
771 struct dp_stats_percpu local_stats;
772 unsigned int start;
773
774 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
775
776 do {
777 start = u64_stats_fetch_begin(&percpu_stats->syncp);
778 local_stats = *percpu_stats;
779 } while (u64_stats_fetch_retry(&percpu_stats->syncp, start));
780
781 stats->n_hit += local_stats.n_hit;
782 stats->n_missed += local_stats.n_missed;
783 stats->n_lost += local_stats.n_lost;
784 mega_stats->n_mask_hit += local_stats.n_mask_hit;
785 mega_stats->n_cache_hit += local_stats.n_cache_hit;
786 }
787 }
788
should_fill_key(const struct sw_flow_id * sfid,uint32_t ufid_flags)789 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
790 {
791 return ovs_identifier_is_ufid(sfid) &&
792 !(ufid_flags & OVS_UFID_F_OMIT_KEY);
793 }
794
should_fill_mask(uint32_t ufid_flags)795 static bool should_fill_mask(uint32_t ufid_flags)
796 {
797 return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
798 }
799
should_fill_actions(uint32_t ufid_flags)800 static bool should_fill_actions(uint32_t ufid_flags)
801 {
802 return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
803 }
804
ovs_flow_cmd_msg_size(const struct sw_flow_actions * acts,const struct sw_flow_id * sfid,uint32_t ufid_flags)805 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
806 const struct sw_flow_id *sfid,
807 uint32_t ufid_flags)
808 {
809 size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
810
811 /* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback
812 * see ovs_nla_put_identifier()
813 */
814 if (sfid && ovs_identifier_is_ufid(sfid))
815 len += nla_total_size(sfid->ufid_len);
816 else
817 len += nla_total_size(ovs_key_attr_size());
818
819 /* OVS_FLOW_ATTR_KEY */
820 if (!sfid || should_fill_key(sfid, ufid_flags))
821 len += nla_total_size(ovs_key_attr_size());
822
823 /* OVS_FLOW_ATTR_MASK */
824 if (should_fill_mask(ufid_flags))
825 len += nla_total_size(ovs_key_attr_size());
826
827 /* OVS_FLOW_ATTR_ACTIONS */
828 if (should_fill_actions(ufid_flags))
829 len += nla_total_size(acts->orig_len);
830
831 return len
832 + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
833 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
834 + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
835 }
836
837 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_stats(const struct sw_flow * flow,struct sk_buff * skb)838 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
839 struct sk_buff *skb)
840 {
841 struct ovs_flow_stats stats;
842 __be16 tcp_flags;
843 unsigned long used;
844
845 ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
846
847 if (used &&
848 nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
849 OVS_FLOW_ATTR_PAD))
850 return -EMSGSIZE;
851
852 if (stats.n_packets &&
853 nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
854 sizeof(struct ovs_flow_stats), &stats,
855 OVS_FLOW_ATTR_PAD))
856 return -EMSGSIZE;
857
858 if ((u8)ntohs(tcp_flags) &&
859 nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
860 return -EMSGSIZE;
861
862 return 0;
863 }
864
865 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_actions(const struct sw_flow * flow,struct sk_buff * skb,int skb_orig_len)866 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
867 struct sk_buff *skb, int skb_orig_len)
868 {
869 struct nlattr *start;
870 int err;
871
872 /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
873 * this is the first flow to be dumped into 'skb'. This is unusual for
874 * Netlink but individual action lists can be longer than
875 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
876 * The userspace caller can always fetch the actions separately if it
877 * really wants them. (Most userspace callers in fact don't care.)
878 *
879 * This can only fail for dump operations because the skb is always
880 * properly sized for single flows.
881 */
882 start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS);
883 if (start) {
884 const struct sw_flow_actions *sf_acts;
885
886 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
887 err = ovs_nla_put_actions(sf_acts->actions,
888 sf_acts->actions_len, skb);
889
890 if (!err)
891 nla_nest_end(skb, start);
892 else {
893 if (skb_orig_len)
894 return err;
895
896 nla_nest_cancel(skb, start);
897 }
898 } else if (skb_orig_len) {
899 return -EMSGSIZE;
900 }
901
902 return 0;
903 }
904
905 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_info(const struct sw_flow * flow,int dp_ifindex,struct sk_buff * skb,u32 portid,u32 seq,u32 flags,u8 cmd,u32 ufid_flags)906 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
907 struct sk_buff *skb, u32 portid,
908 u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
909 {
910 const int skb_orig_len = skb->len;
911 struct ovs_header *ovs_header;
912 int err;
913
914 ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
915 flags, cmd);
916 if (!ovs_header)
917 return -EMSGSIZE;
918
919 ovs_header->dp_ifindex = dp_ifindex;
920
921 err = ovs_nla_put_identifier(flow, skb);
922 if (err)
923 goto error;
924
925 if (should_fill_key(&flow->id, ufid_flags)) {
926 err = ovs_nla_put_masked_key(flow, skb);
927 if (err)
928 goto error;
929 }
930
931 if (should_fill_mask(ufid_flags)) {
932 err = ovs_nla_put_mask(flow, skb);
933 if (err)
934 goto error;
935 }
936
937 err = ovs_flow_cmd_fill_stats(flow, skb);
938 if (err)
939 goto error;
940
941 if (should_fill_actions(ufid_flags)) {
942 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
943 if (err)
944 goto error;
945 }
946
947 genlmsg_end(skb, ovs_header);
948 return 0;
949
950 error:
951 genlmsg_cancel(skb, ovs_header);
952 return err;
953 }
954
955 /* May not be called with RCU read lock. */
ovs_flow_cmd_alloc_info(const struct sw_flow_actions * acts,const struct sw_flow_id * sfid,struct genl_info * info,bool always,uint32_t ufid_flags)956 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
957 const struct sw_flow_id *sfid,
958 struct genl_info *info,
959 bool always,
960 uint32_t ufid_flags)
961 {
962 struct sk_buff *skb;
963 size_t len;
964
965 if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
966 return NULL;
967
968 len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
969 skb = genlmsg_new(len, GFP_KERNEL);
970 if (!skb)
971 return ERR_PTR(-ENOMEM);
972
973 return skb;
974 }
975
976 /* Called with ovs_mutex. */
ovs_flow_cmd_build_info(const struct sw_flow * flow,int dp_ifindex,struct genl_info * info,u8 cmd,bool always,u32 ufid_flags)977 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
978 int dp_ifindex,
979 struct genl_info *info, u8 cmd,
980 bool always, u32 ufid_flags)
981 {
982 struct sk_buff *skb;
983 int retval;
984
985 skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
986 &flow->id, info, always, ufid_flags);
987 if (IS_ERR_OR_NULL(skb))
988 return skb;
989
990 retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
991 info->snd_portid, info->snd_seq, 0,
992 cmd, ufid_flags);
993 if (WARN_ON_ONCE(retval < 0)) {
994 kfree_skb(skb);
995 skb = ERR_PTR(retval);
996 }
997 return skb;
998 }
999
ovs_flow_cmd_new(struct sk_buff * skb,struct genl_info * info)1000 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
1001 {
1002 struct net *net = sock_net(skb->sk);
1003 struct nlattr **a = info->attrs;
1004 struct ovs_header *ovs_header = genl_info_userhdr(info);
1005 struct sw_flow *flow = NULL, *new_flow;
1006 struct sw_flow_mask mask;
1007 struct sk_buff *reply;
1008 struct datapath *dp;
1009 struct sw_flow_key *key;
1010 struct sw_flow_actions *acts;
1011 struct sw_flow_match match;
1012 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1013 int error;
1014 bool log = !a[OVS_FLOW_ATTR_PROBE];
1015
1016 /* Must have key and actions. */
1017 error = -EINVAL;
1018 if (!a[OVS_FLOW_ATTR_KEY]) {
1019 OVS_NLERR(log, "Flow key attr not present in new flow.");
1020 goto error;
1021 }
1022 if (!a[OVS_FLOW_ATTR_ACTIONS]) {
1023 OVS_NLERR(log, "Flow actions attr not present in new flow.");
1024 goto error;
1025 }
1026
1027 /* Most of the time we need to allocate a new flow, do it before
1028 * locking.
1029 */
1030 new_flow = ovs_flow_alloc();
1031 if (IS_ERR(new_flow)) {
1032 error = PTR_ERR(new_flow);
1033 goto error;
1034 }
1035
1036 /* Extract key. */
1037 key = kzalloc_obj(*key);
1038 if (!key) {
1039 error = -ENOMEM;
1040 goto err_kfree_flow;
1041 }
1042
1043 ovs_match_init(&match, key, false, &mask);
1044 error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1045 a[OVS_FLOW_ATTR_MASK], log);
1046 if (error)
1047 goto err_kfree_key;
1048
1049 ovs_flow_mask_key(&new_flow->key, key, true, &mask);
1050
1051 /* Extract flow identifier. */
1052 error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
1053 key, log);
1054 if (error)
1055 goto err_kfree_key;
1056
1057 /* Validate actions. */
1058 error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
1059 &new_flow->key, &acts, log);
1060 if (error) {
1061 OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
1062 goto err_kfree_key;
1063 }
1064
1065 reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
1066 ufid_flags);
1067 if (IS_ERR(reply)) {
1068 error = PTR_ERR(reply);
1069 goto err_kfree_acts;
1070 }
1071
1072 ovs_lock();
1073 dp = get_dp(net, ovs_header->dp_ifindex);
1074 if (unlikely(!dp)) {
1075 error = -ENODEV;
1076 goto err_unlock_ovs;
1077 }
1078
1079 /* Check if this is a duplicate flow */
1080 if (ovs_identifier_is_ufid(&new_flow->id))
1081 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
1082 if (!flow)
1083 flow = ovs_flow_tbl_lookup(&dp->table, key);
1084 if (likely(!flow)) {
1085 rcu_assign_pointer(new_flow->sf_acts, acts);
1086
1087 /* Put flow in bucket. */
1088 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
1089 if (unlikely(error)) {
1090 acts = NULL;
1091 goto err_unlock_ovs;
1092 }
1093
1094 if (unlikely(reply)) {
1095 error = ovs_flow_cmd_fill_info(new_flow,
1096 ovs_header->dp_ifindex,
1097 reply, info->snd_portid,
1098 info->snd_seq, 0,
1099 OVS_FLOW_CMD_NEW,
1100 ufid_flags);
1101 BUG_ON(error < 0);
1102 }
1103 ovs_unlock();
1104 } else {
1105 struct sw_flow_actions *old_acts;
1106
1107 /* Bail out if we're not allowed to modify an existing flow.
1108 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1109 * because Generic Netlink treats the latter as a dump
1110 * request. We also accept NLM_F_EXCL in case that bug ever
1111 * gets fixed.
1112 */
1113 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1114 | NLM_F_EXCL))) {
1115 error = -EEXIST;
1116 goto err_unlock_ovs;
1117 }
1118
1119 /* Look for any overlapping flow. */
1120 if (unlikely(!ovs_flow_cmp(flow, &match))) {
1121 if (ovs_identifier_is_key(&flow->id))
1122 flow = ovs_flow_tbl_lookup_exact(&dp->table,
1123 &match);
1124 else /* UFID matches but key is different */
1125 flow = NULL;
1126 if (!flow) {
1127 error = -ENOENT;
1128 goto err_unlock_ovs;
1129 }
1130 }
1131
1132 if (unlikely(reply)) {
1133 size_t cur, req;
1134
1135 cur = ovs_flow_cmd_msg_size(acts, &new_flow->id,
1136 ufid_flags);
1137 req = ovs_flow_cmd_msg_size(acts, &flow->id,
1138 ufid_flags);
1139 if (cur < req) {
1140 struct sk_buff *resized;
1141
1142 resized = ovs_flow_cmd_alloc_info(acts,
1143 &flow->id,
1144 info, false,
1145 ufid_flags);
1146 if (IS_ERR(resized)) {
1147 error = PTR_ERR(resized);
1148 goto err_unlock_ovs;
1149 }
1150 kfree_skb(reply);
1151 reply = resized;
1152 }
1153 }
1154
1155 /* Update actions. */
1156 old_acts = ovsl_dereference(flow->sf_acts);
1157 rcu_assign_pointer(flow->sf_acts, acts);
1158
1159 if (unlikely(reply)) {
1160 error = ovs_flow_cmd_fill_info(flow,
1161 ovs_header->dp_ifindex,
1162 reply, info->snd_portid,
1163 info->snd_seq, 0,
1164 OVS_FLOW_CMD_NEW,
1165 ufid_flags);
1166 BUG_ON(error < 0);
1167 }
1168 ovs_unlock();
1169
1170 ovs_nla_free_flow_actions_rcu(old_acts);
1171 ovs_flow_free(new_flow, false);
1172 }
1173
1174 if (reply)
1175 ovs_notify(&dp_flow_genl_family, reply, info);
1176
1177 kfree(key);
1178 return 0;
1179
1180 err_unlock_ovs:
1181 ovs_unlock();
1182 kfree_skb(reply);
1183 err_kfree_acts:
1184 ovs_nla_free_flow_actions(acts);
1185 err_kfree_key:
1186 kfree(key);
1187 err_kfree_flow:
1188 ovs_flow_free(new_flow, false);
1189 error:
1190 return error;
1191 }
1192
1193 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1194 static noinline_for_stack
get_flow_actions(struct net * net,const struct nlattr * a,const struct sw_flow_key * key,const struct sw_flow_mask * mask,bool log)1195 struct sw_flow_actions *get_flow_actions(struct net *net,
1196 const struct nlattr *a,
1197 const struct sw_flow_key *key,
1198 const struct sw_flow_mask *mask,
1199 bool log)
1200 {
1201 struct sw_flow_actions *acts;
1202 struct sw_flow_key masked_key;
1203 int error;
1204
1205 ovs_flow_mask_key(&masked_key, key, true, mask);
1206 error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
1207 if (error) {
1208 OVS_NLERR(log,
1209 "Actions may not be safe on all matching packets");
1210 return ERR_PTR(error);
1211 }
1212
1213 return acts;
1214 }
1215
1216 /* Factor out match-init and action-copy to avoid
1217 * "Wframe-larger-than=1024" warning. Because mask is only
1218 * used to get actions, we new a function to save some
1219 * stack space.
1220 *
1221 * If there are not key and action attrs, we return 0
1222 * directly. In the case, the caller will also not use the
1223 * match as before. If there is action attr, we try to get
1224 * actions and save them to *acts. Before returning from
1225 * the function, we reset the match->mask pointer. Because
1226 * we should not to return match object with dangling reference
1227 * to mask.
1228 * */
1229 static noinline_for_stack int
ovs_nla_init_match_and_action(struct net * net,struct sw_flow_match * match,struct sw_flow_key * key,struct nlattr ** a,struct sw_flow_actions ** acts,bool log)1230 ovs_nla_init_match_and_action(struct net *net,
1231 struct sw_flow_match *match,
1232 struct sw_flow_key *key,
1233 struct nlattr **a,
1234 struct sw_flow_actions **acts,
1235 bool log)
1236 {
1237 struct sw_flow_mask mask;
1238 int error = 0;
1239
1240 if (a[OVS_FLOW_ATTR_KEY]) {
1241 ovs_match_init(match, key, true, &mask);
1242 error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1243 a[OVS_FLOW_ATTR_MASK], log);
1244 if (error)
1245 goto error;
1246 }
1247
1248 if (a[OVS_FLOW_ATTR_ACTIONS]) {
1249 if (!a[OVS_FLOW_ATTR_KEY]) {
1250 OVS_NLERR(log,
1251 "Flow key attribute not present in set flow.");
1252 error = -EINVAL;
1253 goto error;
1254 }
1255
1256 *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1257 &mask, log);
1258 if (IS_ERR(*acts)) {
1259 error = PTR_ERR(*acts);
1260 goto error;
1261 }
1262 }
1263
1264 /* On success, error is 0. */
1265 error:
1266 match->mask = NULL;
1267 return error;
1268 }
1269
ovs_flow_cmd_set(struct sk_buff * skb,struct genl_info * info)1270 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1271 {
1272 struct net *net = sock_net(skb->sk);
1273 struct nlattr **a = info->attrs;
1274 struct ovs_header *ovs_header = genl_info_userhdr(info);
1275 struct sw_flow_key key;
1276 struct sw_flow *flow;
1277 struct sk_buff *reply = NULL;
1278 struct datapath *dp;
1279 struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1280 struct sw_flow_match match;
1281 struct sw_flow_id sfid;
1282 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1283 int error = 0;
1284 bool log = !a[OVS_FLOW_ATTR_PROBE];
1285 bool ufid_present;
1286
1287 ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1288 if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
1289 OVS_NLERR(log,
1290 "Flow set message rejected, Key attribute missing.");
1291 return -EINVAL;
1292 }
1293
1294 error = ovs_nla_init_match_and_action(net, &match, &key, a,
1295 &acts, log);
1296 if (error)
1297 goto error;
1298
1299 if (acts) {
1300 /* Can allocate before locking if have acts. */
1301 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1302 ufid_flags);
1303 if (IS_ERR(reply)) {
1304 error = PTR_ERR(reply);
1305 goto err_kfree_acts;
1306 }
1307 }
1308
1309 ovs_lock();
1310 dp = get_dp(net, ovs_header->dp_ifindex);
1311 if (unlikely(!dp)) {
1312 error = -ENODEV;
1313 goto err_unlock_ovs;
1314 }
1315 /* Check that the flow exists. */
1316 if (ufid_present)
1317 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1318 else
1319 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1320 if (unlikely(!flow)) {
1321 error = -ENOENT;
1322 goto err_unlock_ovs;
1323 }
1324
1325 /* Update actions, if present. */
1326 if (likely(acts)) {
1327 old_acts = ovsl_dereference(flow->sf_acts);
1328 rcu_assign_pointer(flow->sf_acts, acts);
1329
1330 if (unlikely(reply)) {
1331 error = ovs_flow_cmd_fill_info(flow,
1332 ovs_header->dp_ifindex,
1333 reply, info->snd_portid,
1334 info->snd_seq, 0,
1335 OVS_FLOW_CMD_SET,
1336 ufid_flags);
1337 BUG_ON(error < 0);
1338 }
1339 } else {
1340 /* Could not alloc without acts before locking. */
1341 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1342 info, OVS_FLOW_CMD_SET, false,
1343 ufid_flags);
1344
1345 if (IS_ERR(reply)) {
1346 error = PTR_ERR(reply);
1347 reply = NULL;
1348 goto err_unlock_ovs;
1349 }
1350 }
1351
1352 /* Clear stats. */
1353 if (a[OVS_FLOW_ATTR_CLEAR])
1354 ovs_flow_stats_clear(flow);
1355 ovs_unlock();
1356
1357 if (reply)
1358 ovs_notify(&dp_flow_genl_family, reply, info);
1359 if (old_acts)
1360 ovs_nla_free_flow_actions_rcu(old_acts);
1361
1362 return 0;
1363
1364 err_unlock_ovs:
1365 ovs_unlock();
1366 kfree_skb(reply);
1367 err_kfree_acts:
1368 ovs_nla_free_flow_actions(acts);
1369 error:
1370 return error;
1371 }
1372
ovs_flow_cmd_get(struct sk_buff * skb,struct genl_info * info)1373 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1374 {
1375 struct nlattr **a = info->attrs;
1376 struct ovs_header *ovs_header = genl_info_userhdr(info);
1377 struct net *net = sock_net(skb->sk);
1378 struct sw_flow_key key;
1379 struct sk_buff *reply;
1380 struct sw_flow *flow;
1381 struct datapath *dp;
1382 struct sw_flow_match match;
1383 struct sw_flow_id ufid;
1384 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1385 int err = 0;
1386 bool log = !a[OVS_FLOW_ATTR_PROBE];
1387 bool ufid_present;
1388
1389 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1390 if (a[OVS_FLOW_ATTR_KEY]) {
1391 ovs_match_init(&match, &key, true, NULL);
1392 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
1393 log);
1394 } else if (!ufid_present) {
1395 OVS_NLERR(log,
1396 "Flow get message rejected, Key attribute missing.");
1397 err = -EINVAL;
1398 }
1399 if (err)
1400 return err;
1401
1402 ovs_lock();
1403 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1404 if (!dp) {
1405 err = -ENODEV;
1406 goto unlock;
1407 }
1408
1409 if (ufid_present)
1410 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1411 else
1412 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1413 if (!flow) {
1414 err = -ENOENT;
1415 goto unlock;
1416 }
1417
1418 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1419 OVS_FLOW_CMD_GET, true, ufid_flags);
1420 if (IS_ERR(reply)) {
1421 err = PTR_ERR(reply);
1422 goto unlock;
1423 }
1424
1425 ovs_unlock();
1426 return genlmsg_reply(reply, info);
1427 unlock:
1428 ovs_unlock();
1429 return err;
1430 }
1431
ovs_flow_cmd_del(struct sk_buff * skb,struct genl_info * info)1432 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1433 {
1434 struct nlattr **a = info->attrs;
1435 struct ovs_header *ovs_header = genl_info_userhdr(info);
1436 struct net *net = sock_net(skb->sk);
1437 struct sw_flow_key key;
1438 struct sk_buff *reply;
1439 struct sw_flow *flow = NULL;
1440 struct datapath *dp;
1441 struct sw_flow_match match;
1442 struct sw_flow_id ufid;
1443 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1444 int err;
1445 bool log = !a[OVS_FLOW_ATTR_PROBE];
1446 bool ufid_present;
1447
1448 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1449 if (a[OVS_FLOW_ATTR_KEY]) {
1450 ovs_match_init(&match, &key, true, NULL);
1451 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1452 NULL, log);
1453 if (unlikely(err))
1454 return err;
1455 }
1456
1457 ovs_lock();
1458 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1459 if (unlikely(!dp)) {
1460 err = -ENODEV;
1461 goto unlock;
1462 }
1463
1464 if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1465 err = ovs_flow_tbl_flush(&dp->table);
1466 goto unlock;
1467 }
1468
1469 if (ufid_present)
1470 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1471 else
1472 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1473 if (unlikely(!flow)) {
1474 err = -ENOENT;
1475 goto unlock;
1476 }
1477
1478 reply = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
1479 &flow->id, info, false, ufid_flags);
1480 if (IS_ERR(reply)) {
1481 netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0,
1482 PTR_ERR(reply));
1483 reply = NULL;
1484 }
1485
1486 if (likely(reply)) {
1487 err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1488 reply, info->snd_portid,
1489 info->snd_seq, 0,
1490 OVS_FLOW_CMD_DEL, ufid_flags);
1491 if (WARN_ON_ONCE(err < 0)) {
1492 kfree_skb(reply);
1493 reply = NULL;
1494 }
1495 }
1496 /* Removal has to happen after ovs_flow_cmd_fill_info(), as it uses
1497 * the flow->mask that can be scheduled to be freed by the
1498 * ovs_flow_tbl_remove() and we're not holding the RCU read lock.
1499 */
1500 ovs_flow_tbl_remove(&dp->table, flow);
1501 ovs_unlock();
1502
1503 if (likely(reply))
1504 ovs_notify(&dp_flow_genl_family, reply, info);
1505
1506 ovs_flow_free(flow, true);
1507 return 0;
1508 unlock:
1509 ovs_unlock();
1510 return err;
1511 }
1512
ovs_flow_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)1513 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1514 {
1515 struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1516 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1517 struct table_instance *ti;
1518 struct datapath *dp;
1519 u32 ufid_flags;
1520 int err;
1521
1522 err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a,
1523 OVS_FLOW_ATTR_MAX, flow_policy, NULL);
1524 if (err)
1525 return err;
1526 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1527
1528 rcu_read_lock();
1529 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1530 if (!dp) {
1531 rcu_read_unlock();
1532 return -ENODEV;
1533 }
1534
1535 ti = rcu_dereference(dp->table.ti);
1536 for (;;) {
1537 struct sw_flow *flow;
1538 u32 bucket, obj;
1539
1540 bucket = cb->args[0];
1541 obj = cb->args[1];
1542 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1543 if (!flow)
1544 break;
1545
1546 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1547 NETLINK_CB(cb->skb).portid,
1548 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1549 OVS_FLOW_CMD_GET, ufid_flags) < 0)
1550 break;
1551
1552 cb->args[0] = bucket;
1553 cb->args[1] = obj;
1554 }
1555 rcu_read_unlock();
1556 return skb->len;
1557 }
1558
1559 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1560 [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1561 [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1562 [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1563 [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1564 [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1565 [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1566 [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1567 };
1568
1569 static const struct genl_small_ops dp_flow_genl_ops[] = {
1570 { .cmd = OVS_FLOW_CMD_NEW,
1571 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1572 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1573 .doit = ovs_flow_cmd_new
1574 },
1575 { .cmd = OVS_FLOW_CMD_DEL,
1576 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1577 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1578 .doit = ovs_flow_cmd_del
1579 },
1580 { .cmd = OVS_FLOW_CMD_GET,
1581 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1582 .flags = 0, /* OK for unprivileged users. */
1583 .doit = ovs_flow_cmd_get,
1584 .dumpit = ovs_flow_cmd_dump
1585 },
1586 { .cmd = OVS_FLOW_CMD_SET,
1587 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1588 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1589 .doit = ovs_flow_cmd_set,
1590 },
1591 };
1592
1593 static struct genl_family dp_flow_genl_family __ro_after_init = {
1594 .hdrsize = sizeof(struct ovs_header),
1595 .name = OVS_FLOW_FAMILY,
1596 .version = OVS_FLOW_VERSION,
1597 .maxattr = OVS_FLOW_ATTR_MAX,
1598 .policy = flow_policy,
1599 .netnsok = true,
1600 .parallel_ops = true,
1601 .small_ops = dp_flow_genl_ops,
1602 .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops),
1603 .resv_start_op = OVS_FLOW_CMD_SET + 1,
1604 .mcgrps = &ovs_dp_flow_multicast_group,
1605 .n_mcgrps = 1,
1606 .module = THIS_MODULE,
1607 };
1608
ovs_dp_cmd_msg_size(void)1609 static size_t ovs_dp_cmd_msg_size(void)
1610 {
1611 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1612
1613 msgsize += nla_total_size(IFNAMSIZ);
1614 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1615 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
1616 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1617 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
1618 msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */
1619
1620 return msgsize;
1621 }
1622
1623 /* Called with ovs_mutex. */
ovs_dp_cmd_fill_info(struct datapath * dp,struct sk_buff * skb,u32 portid,u32 seq,u32 flags,u8 cmd)1624 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1625 u32 portid, u32 seq, u32 flags, u8 cmd)
1626 {
1627 struct ovs_header *ovs_header;
1628 struct ovs_dp_stats dp_stats;
1629 struct ovs_dp_megaflow_stats dp_megaflow_stats;
1630 struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids);
1631 int err, pids_len;
1632
1633 ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1634 flags, cmd);
1635 if (!ovs_header)
1636 goto error;
1637
1638 ovs_header->dp_ifindex = get_dpifindex(dp);
1639
1640 err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1641 if (err)
1642 goto nla_put_failure;
1643
1644 get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1645 if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1646 &dp_stats, OVS_DP_ATTR_PAD))
1647 goto nla_put_failure;
1648
1649 if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1650 sizeof(struct ovs_dp_megaflow_stats),
1651 &dp_megaflow_stats, OVS_DP_ATTR_PAD))
1652 goto nla_put_failure;
1653
1654 if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1655 goto nla_put_failure;
1656
1657 if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE,
1658 ovs_flow_tbl_masks_cache_size(&dp->table)))
1659 goto nla_put_failure;
1660
1661 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
1662 pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32);
1663 if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids))
1664 goto nla_put_failure;
1665 }
1666
1667 genlmsg_end(skb, ovs_header);
1668 return 0;
1669
1670 nla_put_failure:
1671 genlmsg_cancel(skb, ovs_header);
1672 error:
1673 return -EMSGSIZE;
1674 }
1675
ovs_dp_cmd_alloc_info(void)1676 static struct sk_buff *ovs_dp_cmd_alloc_info(void)
1677 {
1678 return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1679 }
1680
1681 /* Called with rcu_read_lock or ovs_mutex. */
lookup_datapath(struct net * net,const struct ovs_header * ovs_header,struct nlattr * a[OVS_DP_ATTR_MAX+1])1682 static struct datapath *lookup_datapath(struct net *net,
1683 const struct ovs_header *ovs_header,
1684 struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1685 {
1686 struct datapath *dp;
1687
1688 if (!a[OVS_DP_ATTR_NAME])
1689 dp = get_dp(net, ovs_header->dp_ifindex);
1690 else {
1691 struct vport *vport;
1692
1693 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1694 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1695 }
1696 return dp ? dp : ERR_PTR(-ENODEV);
1697 }
1698
ovs_dp_reset_user_features(struct sk_buff * skb,struct genl_info * info)1699 static void ovs_dp_reset_user_features(struct sk_buff *skb,
1700 struct genl_info *info)
1701 {
1702 struct datapath *dp;
1703
1704 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1705 info->attrs);
1706 if (IS_ERR(dp))
1707 return;
1708
1709 pr_warn("%s: Dropping previously announced user features\n",
1710 ovs_dp_name(dp));
1711 dp->user_features = 0;
1712 }
1713
ovs_dp_set_upcall_portids(struct datapath * dp,const struct nlattr * ids)1714 static int ovs_dp_set_upcall_portids(struct datapath *dp,
1715 const struct nlattr *ids)
1716 {
1717 struct dp_nlsk_pids *old, *dp_nlsk_pids;
1718
1719 if (!nla_len(ids) || nla_len(ids) % sizeof(u32))
1720 return -EINVAL;
1721
1722 old = ovsl_dereference(dp->upcall_portids);
1723
1724 dp_nlsk_pids = kmalloc(sizeof(*dp_nlsk_pids) + nla_len(ids),
1725 GFP_KERNEL);
1726 if (!dp_nlsk_pids)
1727 return -ENOMEM;
1728
1729 dp_nlsk_pids->n_pids = nla_len(ids) / sizeof(u32);
1730 nla_memcpy(dp_nlsk_pids->pids, ids, nla_len(ids));
1731
1732 rcu_assign_pointer(dp->upcall_portids, dp_nlsk_pids);
1733
1734 kfree_rcu(old, rcu);
1735
1736 return 0;
1737 }
1738
ovs_dp_get_upcall_portid(const struct datapath * dp,uint32_t cpu_id)1739 u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id)
1740 {
1741 struct dp_nlsk_pids *dp_nlsk_pids;
1742
1743 dp_nlsk_pids = rcu_dereference(dp->upcall_portids);
1744
1745 if (dp_nlsk_pids) {
1746 if (cpu_id < dp_nlsk_pids->n_pids) {
1747 return dp_nlsk_pids->pids[cpu_id];
1748 } else if (dp_nlsk_pids->n_pids > 0 &&
1749 cpu_id >= dp_nlsk_pids->n_pids) {
1750 /* If the number of netlink PIDs is mismatched with
1751 * the number of CPUs as seen by the kernel, log this
1752 * and send the upcall to an arbitrary socket (0) in
1753 * order to not drop packets
1754 */
1755 pr_info_ratelimited("cpu_id mismatch with handler threads");
1756 return dp_nlsk_pids->pids[cpu_id %
1757 dp_nlsk_pids->n_pids];
1758 } else {
1759 return 0;
1760 }
1761 } else {
1762 return 0;
1763 }
1764 }
1765
ovs_dp_change(struct datapath * dp,struct nlattr * a[])1766 static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1767 {
1768 u32 user_features = 0, old_features = dp->user_features;
1769 int err;
1770
1771 if (a[OVS_DP_ATTR_USER_FEATURES]) {
1772 user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1773
1774 if (user_features & ~(OVS_DP_F_VPORT_PIDS |
1775 OVS_DP_F_UNALIGNED |
1776 OVS_DP_F_TC_RECIRC_SHARING |
1777 OVS_DP_F_DISPATCH_UPCALL_PER_CPU))
1778 return -EOPNOTSUPP;
1779
1780 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1781 if (user_features & OVS_DP_F_TC_RECIRC_SHARING)
1782 return -EOPNOTSUPP;
1783 #endif
1784 }
1785
1786 if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
1787 int err;
1788 u32 cache_size;
1789
1790 cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
1791 err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size);
1792 if (err)
1793 return err;
1794 }
1795
1796 dp->user_features = user_features;
1797
1798 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU &&
1799 a[OVS_DP_ATTR_PER_CPU_PIDS]) {
1800 /* Upcall Netlink Port IDs have been updated */
1801 err = ovs_dp_set_upcall_portids(dp,
1802 a[OVS_DP_ATTR_PER_CPU_PIDS]);
1803 if (err)
1804 return err;
1805 }
1806
1807 if ((dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) &&
1808 !(old_features & OVS_DP_F_TC_RECIRC_SHARING))
1809 tc_skb_ext_tc_enable();
1810 else if (!(dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) &&
1811 (old_features & OVS_DP_F_TC_RECIRC_SHARING))
1812 tc_skb_ext_tc_disable();
1813
1814 return 0;
1815 }
1816
ovs_dp_stats_init(struct datapath * dp)1817 static int ovs_dp_stats_init(struct datapath *dp)
1818 {
1819 dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1820 if (!dp->stats_percpu)
1821 return -ENOMEM;
1822
1823 return 0;
1824 }
1825
ovs_dp_vport_init(struct datapath * dp)1826 static int ovs_dp_vport_init(struct datapath *dp)
1827 {
1828 int i;
1829
1830 dp->ports = kmalloc_objs(struct hlist_head, DP_VPORT_HASH_BUCKETS);
1831 if (!dp->ports)
1832 return -ENOMEM;
1833
1834 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1835 INIT_HLIST_HEAD(&dp->ports[i]);
1836
1837 return 0;
1838 }
1839
ovs_dp_cmd_new(struct sk_buff * skb,struct genl_info * info)1840 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1841 {
1842 struct nlattr **a = info->attrs;
1843 struct vport_parms parms;
1844 struct sk_buff *reply;
1845 struct datapath *dp;
1846 struct vport *vport;
1847 struct ovs_net *ovs_net;
1848 int err;
1849
1850 err = -EINVAL;
1851 if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1852 goto err;
1853
1854 reply = ovs_dp_cmd_alloc_info();
1855 if (!reply)
1856 return -ENOMEM;
1857
1858 err = -ENOMEM;
1859 dp = kzalloc_obj(*dp);
1860 if (dp == NULL)
1861 goto err_destroy_reply;
1862
1863 ovs_dp_set_net(dp, sock_net(skb->sk));
1864
1865 /* Allocate table. */
1866 err = ovs_flow_tbl_init(&dp->table);
1867 if (err)
1868 goto err_destroy_dp;
1869
1870 err = ovs_dp_stats_init(dp);
1871 if (err)
1872 goto err_destroy_table;
1873
1874 err = ovs_dp_vport_init(dp);
1875 if (err)
1876 goto err_destroy_stats;
1877
1878 err = ovs_meters_init(dp);
1879 if (err)
1880 goto err_destroy_ports;
1881
1882 /* Set up our datapath device. */
1883 parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1884 parms.type = OVS_VPORT_TYPE_INTERNAL;
1885 parms.dp = dp;
1886 parms.port_no = OVSP_LOCAL;
1887 parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1888 parms.desired_ifindex = nla_get_s32_default(a[OVS_DP_ATTR_IFINDEX], 0);
1889
1890 /* So far only local changes have been made, now need the lock. */
1891 ovs_lock();
1892
1893 err = ovs_dp_change(dp, a);
1894 if (err)
1895 goto err_unlock_and_destroy_meters;
1896
1897 vport = new_vport(&parms);
1898 if (IS_ERR(vport)) {
1899 err = PTR_ERR(vport);
1900 if (err == -EBUSY)
1901 err = -EEXIST;
1902
1903 if (err == -EEXIST) {
1904 /* An outdated user space instance that does not understand
1905 * the concept of user_features has attempted to create a new
1906 * datapath and is likely to reuse it. Drop all user features.
1907 */
1908 if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1909 ovs_dp_reset_user_features(skb, info);
1910 }
1911
1912 goto err_destroy_portids;
1913 }
1914
1915 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1916 info->snd_seq, 0, OVS_DP_CMD_NEW);
1917 BUG_ON(err < 0);
1918
1919 ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1920 list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1921
1922 ovs_unlock();
1923
1924 ovs_notify(&dp_datapath_genl_family, reply, info);
1925 return 0;
1926
1927 err_destroy_portids:
1928 kfree(rcu_dereference_raw(dp->upcall_portids));
1929 err_unlock_and_destroy_meters:
1930 ovs_unlock();
1931 ovs_meters_exit(dp);
1932 err_destroy_ports:
1933 kfree(dp->ports);
1934 err_destroy_stats:
1935 free_percpu(dp->stats_percpu);
1936 err_destroy_table:
1937 ovs_flow_tbl_destroy(&dp->table);
1938 err_destroy_dp:
1939 kfree(dp);
1940 err_destroy_reply:
1941 kfree_skb(reply);
1942 err:
1943 return err;
1944 }
1945
1946 /* Called with ovs_mutex. */
__dp_destroy(struct datapath * dp)1947 static void __dp_destroy(struct datapath *dp)
1948 {
1949 struct flow_table *table = &dp->table;
1950 int i;
1951
1952 if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING)
1953 tc_skb_ext_tc_disable();
1954
1955 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1956 struct vport *vport;
1957 struct hlist_node *n;
1958
1959 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1960 if (vport->port_no != OVSP_LOCAL)
1961 ovs_dp_detach_port(vport);
1962 }
1963
1964 list_del_rcu(&dp->list_node);
1965
1966 /* OVSP_LOCAL is datapath internal port. We need to make sure that
1967 * all ports in datapath are destroyed first before freeing datapath.
1968 */
1969 ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1970
1971 /* Flush sw_flow in the tables. RCU cb only releases resource
1972 * such as dp, ports and tables. That may avoid some issues
1973 * such as RCU usage warning.
1974 */
1975 table_instance_flow_flush(table, ovsl_dereference(table->ti),
1976 ovsl_dereference(table->ufid_ti));
1977
1978 /* RCU destroy the ports, meters and flow tables. */
1979 call_rcu(&dp->rcu, destroy_dp_rcu);
1980 }
1981
ovs_dp_cmd_del(struct sk_buff * skb,struct genl_info * info)1982 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1983 {
1984 struct sk_buff *reply;
1985 struct datapath *dp;
1986 int err;
1987
1988 reply = ovs_dp_cmd_alloc_info();
1989 if (!reply)
1990 return -ENOMEM;
1991
1992 ovs_lock();
1993 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1994 info->attrs);
1995 err = PTR_ERR(dp);
1996 if (IS_ERR(dp))
1997 goto err_unlock_free;
1998
1999 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
2000 info->snd_seq, 0, OVS_DP_CMD_DEL);
2001 BUG_ON(err < 0);
2002
2003 __dp_destroy(dp);
2004 ovs_unlock();
2005
2006 ovs_notify(&dp_datapath_genl_family, reply, info);
2007
2008 return 0;
2009
2010 err_unlock_free:
2011 ovs_unlock();
2012 kfree_skb(reply);
2013 return err;
2014 }
2015
ovs_dp_cmd_set(struct sk_buff * skb,struct genl_info * info)2016 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
2017 {
2018 struct sk_buff *reply;
2019 struct datapath *dp;
2020 int err;
2021
2022 reply = ovs_dp_cmd_alloc_info();
2023 if (!reply)
2024 return -ENOMEM;
2025
2026 ovs_lock();
2027 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
2028 info->attrs);
2029 err = PTR_ERR(dp);
2030 if (IS_ERR(dp))
2031 goto err_unlock_free;
2032
2033 err = ovs_dp_change(dp, info->attrs);
2034 if (err)
2035 goto err_unlock_free;
2036
2037 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
2038 info->snd_seq, 0, OVS_DP_CMD_SET);
2039 BUG_ON(err < 0);
2040
2041 ovs_unlock();
2042 ovs_notify(&dp_datapath_genl_family, reply, info);
2043
2044 return 0;
2045
2046 err_unlock_free:
2047 ovs_unlock();
2048 kfree_skb(reply);
2049 return err;
2050 }
2051
ovs_dp_cmd_get(struct sk_buff * skb,struct genl_info * info)2052 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
2053 {
2054 struct sk_buff *reply;
2055 struct datapath *dp;
2056 int err;
2057
2058 reply = ovs_dp_cmd_alloc_info();
2059 if (!reply)
2060 return -ENOMEM;
2061
2062 ovs_lock();
2063 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
2064 info->attrs);
2065 if (IS_ERR(dp)) {
2066 err = PTR_ERR(dp);
2067 goto err_unlock_free;
2068 }
2069 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
2070 info->snd_seq, 0, OVS_DP_CMD_GET);
2071 BUG_ON(err < 0);
2072 ovs_unlock();
2073
2074 return genlmsg_reply(reply, info);
2075
2076 err_unlock_free:
2077 ovs_unlock();
2078 kfree_skb(reply);
2079 return err;
2080 }
2081
ovs_dp_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)2082 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2083 {
2084 struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
2085 struct datapath *dp;
2086 int skip = cb->args[0];
2087 int i = 0;
2088
2089 ovs_lock();
2090 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2091 if (i >= skip &&
2092 ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
2093 cb->nlh->nlmsg_seq, NLM_F_MULTI,
2094 OVS_DP_CMD_GET) < 0)
2095 break;
2096 i++;
2097 }
2098 ovs_unlock();
2099
2100 cb->args[0] = i;
2101
2102 return skb->len;
2103 }
2104
2105 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
2106 [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2107 [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2108 [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
2109 [OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0,
2110 PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
2111 [OVS_DP_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0),
2112 };
2113
2114 static const struct genl_small_ops dp_datapath_genl_ops[] = {
2115 { .cmd = OVS_DP_CMD_NEW,
2116 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2117 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2118 .doit = ovs_dp_cmd_new
2119 },
2120 { .cmd = OVS_DP_CMD_DEL,
2121 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2122 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2123 .doit = ovs_dp_cmd_del
2124 },
2125 { .cmd = OVS_DP_CMD_GET,
2126 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2127 .flags = 0, /* OK for unprivileged users. */
2128 .doit = ovs_dp_cmd_get,
2129 .dumpit = ovs_dp_cmd_dump
2130 },
2131 { .cmd = OVS_DP_CMD_SET,
2132 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2133 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2134 .doit = ovs_dp_cmd_set,
2135 },
2136 };
2137
2138 static struct genl_family dp_datapath_genl_family __ro_after_init = {
2139 .hdrsize = sizeof(struct ovs_header),
2140 .name = OVS_DATAPATH_FAMILY,
2141 .version = OVS_DATAPATH_VERSION,
2142 .maxattr = OVS_DP_ATTR_MAX,
2143 .policy = datapath_policy,
2144 .netnsok = true,
2145 .parallel_ops = true,
2146 .small_ops = dp_datapath_genl_ops,
2147 .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops),
2148 .resv_start_op = OVS_DP_CMD_SET + 1,
2149 .mcgrps = &ovs_dp_datapath_multicast_group,
2150 .n_mcgrps = 1,
2151 .module = THIS_MODULE,
2152 };
2153
2154 /* Called with ovs_mutex or RCU read lock. */
ovs_vport_cmd_fill_info(struct vport * vport,struct sk_buff * skb,struct net * net,u32 portid,u32 seq,u32 flags,u8 cmd,gfp_t gfp)2155 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
2156 struct net *net, u32 portid, u32 seq,
2157 u32 flags, u8 cmd, gfp_t gfp)
2158 {
2159 struct ovs_header *ovs_header;
2160 struct ovs_vport_stats vport_stats;
2161 struct net *net_vport;
2162 int err;
2163
2164 ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
2165 flags, cmd);
2166 if (!ovs_header)
2167 return -EMSGSIZE;
2168
2169 ovs_header->dp_ifindex = get_dpifindex(vport->dp);
2170
2171 if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
2172 nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
2173 nla_put_string(skb, OVS_VPORT_ATTR_NAME,
2174 ovs_vport_name(vport)) ||
2175 nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
2176 goto nla_put_failure;
2177
2178 rcu_read_lock();
2179 net_vport = dev_net_rcu(vport->dev);
2180 if (!net_eq(net, net_vport)) {
2181 int id = peernet2id_alloc(net, net_vport, GFP_ATOMIC);
2182
2183 if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
2184 goto nla_put_failure_unlock;
2185 }
2186 rcu_read_unlock();
2187
2188 ovs_vport_get_stats(vport, &vport_stats);
2189 if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
2190 sizeof(struct ovs_vport_stats), &vport_stats,
2191 OVS_VPORT_ATTR_PAD))
2192 goto nla_put_failure;
2193
2194 if (ovs_vport_get_upcall_stats(vport, skb))
2195 goto nla_put_failure;
2196
2197 if (ovs_vport_get_upcall_portids(vport, skb))
2198 goto nla_put_failure;
2199
2200 genlmsg_end(skb, ovs_header);
2201 return 0;
2202
2203 nla_put_failure_unlock:
2204 rcu_read_unlock();
2205 nla_put_failure:
2206 err = -EMSGSIZE;
2207 genlmsg_cancel(skb, ovs_header);
2208 return err;
2209 }
2210
ovs_vport_cmd_msg_size(void)2211 static size_t ovs_vport_cmd_msg_size(void)
2212 {
2213 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
2214
2215 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_PORT_NO */
2216 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_TYPE */
2217 msgsize += nla_total_size(IFNAMSIZ); /* OVS_VPORT_ATTR_NAME */
2218 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_IFINDEX */
2219 msgsize += nla_total_size(sizeof(s32)); /* OVS_VPORT_ATTR_NETNSID */
2220
2221 /* OVS_VPORT_ATTR_STATS */
2222 msgsize += nla_total_size_64bit(sizeof(struct ovs_vport_stats));
2223
2224 /* OVS_VPORT_ATTR_UPCALL_STATS(OVS_VPORT_UPCALL_ATTR_SUCCESS +
2225 * OVS_VPORT_UPCALL_ATTR_FAIL)
2226 */
2227 msgsize += nla_total_size(nla_total_size_64bit(sizeof(u64)) +
2228 nla_total_size_64bit(sizeof(u64)));
2229
2230 /* OVS_VPORT_ATTR_UPCALL_PID */
2231 msgsize += nla_total_size(nr_cpu_ids * sizeof(u32));
2232
2233 return msgsize;
2234 }
2235
ovs_vport_cmd_alloc_info(void)2236 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
2237 {
2238 return genlmsg_new(ovs_vport_cmd_msg_size(), GFP_KERNEL);
2239 }
2240
2241 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
ovs_vport_cmd_build_info(struct vport * vport,struct net * net,u32 portid,u32 seq,u8 cmd)2242 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
2243 u32 portid, u32 seq, u8 cmd)
2244 {
2245 struct sk_buff *skb;
2246 int retval;
2247
2248 skb = ovs_vport_cmd_alloc_info();
2249 if (!skb)
2250 return ERR_PTR(-ENOMEM);
2251
2252 retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd,
2253 GFP_KERNEL);
2254 BUG_ON(retval < 0);
2255
2256 return skb;
2257 }
2258
2259 /* Called with ovs_mutex or RCU read lock. */
lookup_vport(struct net * net,const struct ovs_header * ovs_header,struct nlattr * a[OVS_VPORT_ATTR_MAX+1])2260 static struct vport *lookup_vport(struct net *net,
2261 const struct ovs_header *ovs_header,
2262 struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
2263 {
2264 struct datapath *dp;
2265 struct vport *vport;
2266
2267 if (a[OVS_VPORT_ATTR_IFINDEX])
2268 return ERR_PTR(-EOPNOTSUPP);
2269 if (a[OVS_VPORT_ATTR_NAME]) {
2270 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
2271 if (!vport)
2272 return ERR_PTR(-ENODEV);
2273 if (ovs_header->dp_ifindex &&
2274 ovs_header->dp_ifindex != get_dpifindex(vport->dp))
2275 return ERR_PTR(-ENODEV);
2276 return vport;
2277 } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
2278 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
2279
2280 if (port_no >= DP_MAX_PORTS)
2281 return ERR_PTR(-EFBIG);
2282
2283 dp = get_dp(net, ovs_header->dp_ifindex);
2284 if (!dp)
2285 return ERR_PTR(-ENODEV);
2286
2287 vport = ovs_vport_ovsl_rcu(dp, port_no);
2288 if (!vport)
2289 return ERR_PTR(-ENODEV);
2290 return vport;
2291 } else
2292 return ERR_PTR(-EINVAL);
2293
2294 }
2295
ovs_get_max_headroom(struct datapath * dp)2296 static unsigned int ovs_get_max_headroom(struct datapath *dp)
2297 {
2298 unsigned int dev_headroom, max_headroom = 0;
2299 struct net_device *dev;
2300 struct vport *vport;
2301 int i;
2302
2303 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2304 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2305 lockdep_ovsl_is_held()) {
2306 dev = vport->dev;
2307 dev_headroom = netdev_get_fwd_headroom(dev);
2308 if (dev_headroom > max_headroom)
2309 max_headroom = dev_headroom;
2310 }
2311 }
2312
2313 return max_headroom;
2314 }
2315
2316 /* Called with ovs_mutex */
ovs_update_headroom(struct datapath * dp,unsigned int new_headroom)2317 static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom)
2318 {
2319 struct vport *vport;
2320 int i;
2321
2322 dp->max_headroom = new_headroom;
2323 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2324 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2325 lockdep_ovsl_is_held())
2326 netdev_set_rx_headroom(vport->dev, new_headroom);
2327 }
2328 }
2329
ovs_vport_cmd_new(struct sk_buff * skb,struct genl_info * info)2330 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
2331 {
2332 struct nlattr **a = info->attrs;
2333 struct ovs_header *ovs_header = genl_info_userhdr(info);
2334 struct vport_parms parms;
2335 struct sk_buff *reply;
2336 struct vport *vport;
2337 struct datapath *dp;
2338 unsigned int new_headroom;
2339 u32 port_no;
2340 int err;
2341
2342 if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
2343 !a[OVS_VPORT_ATTR_UPCALL_PID])
2344 return -EINVAL;
2345
2346 parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2347
2348 if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL)
2349 return -EOPNOTSUPP;
2350
2351 port_no = nla_get_u32_default(a[OVS_VPORT_ATTR_PORT_NO], 0);
2352 if (port_no >= DP_MAX_PORTS)
2353 return -EFBIG;
2354
2355 reply = ovs_vport_cmd_alloc_info();
2356 if (!reply)
2357 return -ENOMEM;
2358
2359 ovs_lock();
2360 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2361 err = -ENODEV;
2362 if (!dp)
2363 goto exit_unlock_free;
2364
2365 if (port_no) {
2366 vport = ovs_vport_ovsl(dp, port_no);
2367 err = -EBUSY;
2368 if (vport)
2369 goto exit_unlock_free;
2370 } else {
2371 for (port_no = 1; ; port_no++) {
2372 if (port_no >= DP_MAX_PORTS) {
2373 err = -EFBIG;
2374 goto exit_unlock_free;
2375 }
2376 vport = ovs_vport_ovsl(dp, port_no);
2377 if (!vport)
2378 break;
2379 }
2380 }
2381
2382 parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2383 parms.dp = dp;
2384 parms.port_no = port_no;
2385 parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
2386 parms.desired_ifindex = nla_get_s32_default(a[OVS_VPORT_ATTR_IFINDEX],
2387 0);
2388
2389 vport = new_vport(&parms);
2390 err = PTR_ERR(vport);
2391 if (IS_ERR(vport))
2392 goto exit_unlock_free;
2393
2394 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2395 info->snd_portid, info->snd_seq, 0,
2396 OVS_VPORT_CMD_NEW, GFP_KERNEL);
2397
2398 new_headroom = netdev_get_fwd_headroom(vport->dev);
2399
2400 if (new_headroom > dp->max_headroom)
2401 ovs_update_headroom(dp, new_headroom);
2402 else
2403 netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2404
2405 BUG_ON(err < 0);
2406 ovs_unlock();
2407
2408 ovs_notify(&dp_vport_genl_family, reply, info);
2409 return 0;
2410
2411 exit_unlock_free:
2412 ovs_unlock();
2413 kfree_skb(reply);
2414 return err;
2415 }
2416
ovs_vport_cmd_set(struct sk_buff * skb,struct genl_info * info)2417 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2418 {
2419 struct nlattr **a = info->attrs;
2420 struct sk_buff *reply;
2421 struct vport *vport;
2422 int err;
2423
2424 reply = ovs_vport_cmd_alloc_info();
2425 if (!reply)
2426 return -ENOMEM;
2427
2428 ovs_lock();
2429 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a);
2430 err = PTR_ERR(vport);
2431 if (IS_ERR(vport))
2432 goto exit_unlock_free;
2433
2434 if (a[OVS_VPORT_ATTR_TYPE] &&
2435 nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
2436 err = -EINVAL;
2437 goto exit_unlock_free;
2438 }
2439
2440 if (a[OVS_VPORT_ATTR_OPTIONS]) {
2441 /* There are no vport types that support legacy options. */
2442 err = -EOPNOTSUPP;
2443 goto exit_unlock_free;
2444 }
2445
2446 if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
2447 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2448
2449 err = ovs_vport_set_upcall_portids(vport, ids);
2450 if (err)
2451 goto exit_unlock_free;
2452 }
2453
2454 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2455 info->snd_portid, info->snd_seq, 0,
2456 OVS_VPORT_CMD_SET, GFP_KERNEL);
2457 BUG_ON(err < 0);
2458
2459 ovs_unlock();
2460 ovs_notify(&dp_vport_genl_family, reply, info);
2461 return 0;
2462
2463 exit_unlock_free:
2464 ovs_unlock();
2465 kfree_skb(reply);
2466 return err;
2467 }
2468
ovs_vport_cmd_del(struct sk_buff * skb,struct genl_info * info)2469 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2470 {
2471 bool update_headroom = false;
2472 struct nlattr **a = info->attrs;
2473 struct sk_buff *reply;
2474 struct datapath *dp;
2475 struct vport *vport;
2476 unsigned int new_headroom;
2477 int err;
2478
2479 reply = ovs_vport_cmd_alloc_info();
2480 if (!reply)
2481 return -ENOMEM;
2482
2483 ovs_lock();
2484 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a);
2485 err = PTR_ERR(vport);
2486 if (IS_ERR(vport))
2487 goto exit_unlock_free;
2488
2489 if (vport->port_no == OVSP_LOCAL) {
2490 err = -EINVAL;
2491 goto exit_unlock_free;
2492 }
2493
2494 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2495 info->snd_portid, info->snd_seq, 0,
2496 OVS_VPORT_CMD_DEL, GFP_KERNEL);
2497 BUG_ON(err < 0);
2498
2499 /* the vport deletion may trigger dp headroom update */
2500 dp = vport->dp;
2501 if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2502 update_headroom = true;
2503
2504 netdev_reset_rx_headroom(vport->dev);
2505 ovs_dp_detach_port(vport);
2506
2507 if (update_headroom) {
2508 new_headroom = ovs_get_max_headroom(dp);
2509
2510 if (new_headroom < dp->max_headroom)
2511 ovs_update_headroom(dp, new_headroom);
2512 }
2513 ovs_unlock();
2514
2515 ovs_notify(&dp_vport_genl_family, reply, info);
2516 return 0;
2517
2518 exit_unlock_free:
2519 ovs_unlock();
2520 kfree_skb(reply);
2521 return err;
2522 }
2523
ovs_vport_cmd_get(struct sk_buff * skb,struct genl_info * info)2524 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2525 {
2526 struct nlattr **a = info->attrs;
2527 struct ovs_header *ovs_header = genl_info_userhdr(info);
2528 struct sk_buff *reply;
2529 struct vport *vport;
2530 int err;
2531
2532 reply = ovs_vport_cmd_alloc_info();
2533 if (!reply)
2534 return -ENOMEM;
2535
2536 rcu_read_lock();
2537 vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2538 err = PTR_ERR(vport);
2539 if (IS_ERR(vport))
2540 goto exit_unlock_free;
2541 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2542 info->snd_portid, info->snd_seq, 0,
2543 OVS_VPORT_CMD_GET, GFP_ATOMIC);
2544 BUG_ON(err < 0);
2545 rcu_read_unlock();
2546
2547 return genlmsg_reply(reply, info);
2548
2549 exit_unlock_free:
2550 rcu_read_unlock();
2551 kfree_skb(reply);
2552 return err;
2553 }
2554
ovs_vport_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)2555 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2556 {
2557 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2558 struct datapath *dp;
2559 int bucket = cb->args[0], skip = cb->args[1];
2560 int i, j = 0;
2561
2562 rcu_read_lock();
2563 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2564 if (!dp) {
2565 rcu_read_unlock();
2566 return -ENODEV;
2567 }
2568 for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2569 struct vport *vport;
2570
2571 j = 0;
2572 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2573 if (j >= skip &&
2574 ovs_vport_cmd_fill_info(vport, skb,
2575 sock_net(skb->sk),
2576 NETLINK_CB(cb->skb).portid,
2577 cb->nlh->nlmsg_seq,
2578 NLM_F_MULTI,
2579 OVS_VPORT_CMD_GET,
2580 GFP_ATOMIC) < 0)
2581 goto out;
2582
2583 j++;
2584 }
2585 skip = 0;
2586 }
2587 out:
2588 rcu_read_unlock();
2589
2590 cb->args[0] = i;
2591 cb->args[1] = j;
2592
2593 return skb->len;
2594 }
2595
ovs_dp_masks_rebalance(struct work_struct * work)2596 static void ovs_dp_masks_rebalance(struct work_struct *work)
2597 {
2598 struct ovs_net *ovs_net = container_of(work, struct ovs_net,
2599 masks_rebalance.work);
2600 struct datapath *dp;
2601
2602 ovs_lock();
2603
2604 list_for_each_entry(dp, &ovs_net->dps, list_node)
2605 ovs_flow_masks_rebalance(&dp->table);
2606
2607 ovs_unlock();
2608
2609 schedule_delayed_work(&ovs_net->masks_rebalance,
2610 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2611 }
2612
2613 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2614 [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2615 [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2616 [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2617 [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2618 [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC },
2619 [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED }, /* Unused. */
2620 [OVS_VPORT_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0),
2621 [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
2622 [OVS_VPORT_ATTR_UPCALL_STATS] = { .type = NLA_NESTED },
2623 };
2624
2625 static const struct genl_small_ops dp_vport_genl_ops[] = {
2626 { .cmd = OVS_VPORT_CMD_NEW,
2627 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2628 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2629 .doit = ovs_vport_cmd_new
2630 },
2631 { .cmd = OVS_VPORT_CMD_DEL,
2632 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2633 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2634 .doit = ovs_vport_cmd_del
2635 },
2636 { .cmd = OVS_VPORT_CMD_GET,
2637 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2638 .flags = 0, /* OK for unprivileged users. */
2639 .doit = ovs_vport_cmd_get,
2640 .dumpit = ovs_vport_cmd_dump
2641 },
2642 { .cmd = OVS_VPORT_CMD_SET,
2643 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2644 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2645 .doit = ovs_vport_cmd_set,
2646 },
2647 };
2648
2649 struct genl_family dp_vport_genl_family __ro_after_init = {
2650 .hdrsize = sizeof(struct ovs_header),
2651 .name = OVS_VPORT_FAMILY,
2652 .version = OVS_VPORT_VERSION,
2653 .maxattr = OVS_VPORT_ATTR_MAX,
2654 .policy = vport_policy,
2655 .netnsok = true,
2656 .parallel_ops = true,
2657 .small_ops = dp_vport_genl_ops,
2658 .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops),
2659 .resv_start_op = OVS_VPORT_CMD_SET + 1,
2660 .mcgrps = &ovs_dp_vport_multicast_group,
2661 .n_mcgrps = 1,
2662 .module = THIS_MODULE,
2663 };
2664
2665 static struct genl_family * const dp_genl_families[] = {
2666 &dp_datapath_genl_family,
2667 &dp_vport_genl_family,
2668 &dp_flow_genl_family,
2669 &dp_packet_genl_family,
2670 &dp_meter_genl_family,
2671 #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
2672 &dp_ct_limit_genl_family,
2673 #endif
2674 };
2675
dp_unregister_genl(int n_families)2676 static void dp_unregister_genl(int n_families)
2677 {
2678 int i;
2679
2680 for (i = 0; i < n_families; i++)
2681 genl_unregister_family(dp_genl_families[i]);
2682 }
2683
dp_register_genl(void)2684 static int __init dp_register_genl(void)
2685 {
2686 int err;
2687 int i;
2688
2689 for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2690
2691 err = genl_register_family(dp_genl_families[i]);
2692 if (err)
2693 goto error;
2694 }
2695
2696 return 0;
2697
2698 error:
2699 dp_unregister_genl(i);
2700 return err;
2701 }
2702
ovs_init_net(struct net * net)2703 static int __net_init ovs_init_net(struct net *net)
2704 {
2705 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2706 int err;
2707
2708 INIT_LIST_HEAD(&ovs_net->dps);
2709 INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2710 INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance);
2711
2712 err = ovs_ct_init(net);
2713 if (err)
2714 return err;
2715
2716 schedule_delayed_work(&ovs_net->masks_rebalance,
2717 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2718 return 0;
2719 }
2720
list_vports_from_net(struct net * net,struct net * dnet,struct list_head * head)2721 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2722 struct list_head *head)
2723 {
2724 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2725 struct datapath *dp;
2726
2727 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2728 int i;
2729
2730 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2731 struct vport *vport;
2732
2733 hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2734 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2735 continue;
2736
2737 if (dev_net(vport->dev) == dnet)
2738 list_add(&vport->detach_list, head);
2739 }
2740 }
2741 }
2742 }
2743
ovs_pre_exit_net(struct net * dnet)2744 static void __net_exit ovs_pre_exit_net(struct net *dnet)
2745 {
2746 ovs_lock();
2747 ovs_ct_exit_start(dnet);
2748 ovs_unlock();
2749 }
2750
ovs_exit_net(struct net * dnet)2751 static void __net_exit ovs_exit_net(struct net *dnet)
2752 {
2753 struct datapath *dp, *dp_next;
2754 struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2755 struct vport *vport, *vport_next;
2756 struct net *net;
2757 LIST_HEAD(head);
2758
2759 ovs_lock();
2760
2761 ovs_ct_exit_finish(dnet);
2762
2763 list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2764 __dp_destroy(dp);
2765
2766 down_read(&net_rwsem);
2767 for_each_net(net)
2768 list_vports_from_net(net, dnet, &head);
2769 up_read(&net_rwsem);
2770
2771 /* Detach all vports from given namespace. */
2772 list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2773 list_del(&vport->detach_list);
2774 ovs_dp_detach_port(vport);
2775 }
2776
2777 ovs_unlock();
2778
2779 cancel_delayed_work_sync(&ovs_net->masks_rebalance);
2780 cancel_work_sync(&ovs_net->dp_notify_work);
2781 }
2782
2783 static struct pernet_operations ovs_net_ops = {
2784 .init = ovs_init_net,
2785 .pre_exit = ovs_pre_exit_net,
2786 .exit = ovs_exit_net,
2787 .id = &ovs_net_id,
2788 .size = sizeof(struct ovs_net),
2789 };
2790
2791 static const char * const ovs_drop_reasons[] = {
2792 #define S(x) [(x) & ~SKB_DROP_REASON_SUBSYS_MASK] = (#x),
2793 OVS_DROP_REASONS(S)
2794 #undef S
2795 };
2796
2797 static struct drop_reason_list drop_reason_list_ovs = {
2798 .reasons = ovs_drop_reasons,
2799 .n_reasons = ARRAY_SIZE(ovs_drop_reasons),
2800 };
2801
ovs_alloc_percpu_storage(void)2802 static int __init ovs_alloc_percpu_storage(void)
2803 {
2804 unsigned int cpu;
2805
2806 ovs_pcpu_storage = alloc_percpu(*ovs_pcpu_storage);
2807 if (!ovs_pcpu_storage)
2808 return -ENOMEM;
2809
2810 for_each_possible_cpu(cpu) {
2811 struct ovs_pcpu_storage *ovs_pcpu;
2812
2813 ovs_pcpu = per_cpu_ptr(ovs_pcpu_storage, cpu);
2814 local_lock_init(&ovs_pcpu->bh_lock);
2815 }
2816 return 0;
2817 }
2818
ovs_free_percpu_storage(void)2819 static void ovs_free_percpu_storage(void)
2820 {
2821 free_percpu(ovs_pcpu_storage);
2822 }
2823
dp_init(void)2824 static int __init dp_init(void)
2825 {
2826 int err;
2827
2828 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) >
2829 sizeof_field(struct sk_buff, cb));
2830
2831 pr_info("Open vSwitch switching datapath\n");
2832
2833 err = ovs_alloc_percpu_storage();
2834 if (err)
2835 goto error;
2836
2837 err = ovs_internal_dev_rtnl_link_register();
2838 if (err)
2839 goto error;
2840
2841 err = ovs_flow_init();
2842 if (err)
2843 goto error_unreg_rtnl_link;
2844
2845 err = ovs_vport_init();
2846 if (err)
2847 goto error_flow_exit;
2848
2849 err = register_pernet_device(&ovs_net_ops);
2850 if (err)
2851 goto error_vport_exit;
2852
2853 err = register_netdevice_notifier(&ovs_dp_device_notifier);
2854 if (err)
2855 goto error_netns_exit;
2856
2857 err = ovs_netdev_init();
2858 if (err)
2859 goto error_unreg_notifier;
2860
2861 err = dp_register_genl();
2862 if (err < 0)
2863 goto error_unreg_netdev;
2864
2865 drop_reasons_register_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH,
2866 &drop_reason_list_ovs);
2867
2868 return 0;
2869
2870 error_unreg_netdev:
2871 ovs_netdev_exit();
2872 error_unreg_notifier:
2873 unregister_netdevice_notifier(&ovs_dp_device_notifier);
2874 error_netns_exit:
2875 unregister_pernet_device(&ovs_net_ops);
2876 error_vport_exit:
2877 ovs_vport_exit();
2878 error_flow_exit:
2879 ovs_flow_exit();
2880 error_unreg_rtnl_link:
2881 ovs_internal_dev_rtnl_link_unregister();
2882 error:
2883 ovs_free_percpu_storage();
2884 return err;
2885 }
2886
dp_cleanup(void)2887 static void dp_cleanup(void)
2888 {
2889 dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2890 ovs_netdev_exit();
2891 unregister_netdevice_notifier(&ovs_dp_device_notifier);
2892 unregister_pernet_device(&ovs_net_ops);
2893 drop_reasons_unregister_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH);
2894 rcu_barrier();
2895 ovs_vport_exit();
2896 ovs_flow_exit();
2897 ovs_internal_dev_rtnl_link_unregister();
2898 ovs_free_percpu_storage();
2899 }
2900
2901 module_init(dp_init);
2902 module_exit(dp_cleanup);
2903
2904 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2905 MODULE_LICENSE("GPL");
2906 MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2907 MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2908 MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2909 MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
2910 MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
2911 MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
2912